@martian-engineering/lossless-claw 0.6.3 → 0.8.0

package/src/engine.ts

@@ -43,9 +43,16 @@ import {
   generateExplorationSummary,
   parseFileBlocks,
 } from "./large-files.js";
+import { describeLogError } from "./lcm-log.js";
 import { RetrievalEngine } from "./retrieval.js";
 import { compileSessionPatterns, matchesSessionPattern } from "./session-patterns.js";
 import { logStartupBannerOnce } from "./startup-banner-log.js";
+import {
+  CompactionTelemetryStore,
+  type ConversationCompactionTelemetryRecord,
+  type CacheState,
+  type ActivityBand,
+} from "./store/compaction-telemetry-store.js";
 import {
   ConversationStore,
   type ConversationRecord,
@@ -56,6 +63,7 @@ import {
 import { SummaryStore } from "./store/summary-store.js";
 import { createLcmSummarizeFromLegacyParams, LcmProviderAuthError } from "./summarize.js";
 import type { LcmDependencies } from "./types.js";
+import { estimateTokens } from "./estimate-tokens.js";
 
 type AgentMessage = Parameters<ContextEngine["ingest"]>[0]["message"];
 type AssembleResultWithSystemPrompt = AssembleResult & { systemPromptAddition?: string };
@@ -63,6 +71,30 @@ type CircuitBreakerState = {
   failures: number;
   openSince: number | null;
 };
+type PromptCacheSnapshot = {
+  lastObservedCacheRead?: number;
+  lastObservedCacheWrite?: number;
+  cacheState: CacheState;
+  retention?: string;
+  sawExplicitBreak: boolean;
+};
+type IncrementalCompactionDecision = {
+  shouldCompact: boolean;
+  cacheState: CacheState;
+  maxPasses: number;
+  rawTokensOutsideTail: number;
+  threshold: number;
+  leafChunkTokens: number;
+  fallbackLeafChunkTokens: number[];
+  activityBand: ActivityBand;
+  allowCondensedPasses: boolean;
+};
+type DynamicLeafChunkBounds = {
+  floor: number;
+  medium: number;
+  high: number;
+  max: number;
+};
 type TranscriptRewriteReplacement = {
   entryId: string;
   message: AgentMessage;
@@ -83,14 +115,16 @@ type ContextEngineMaintenanceRuntimeContext = Record<string, unknown> & {
 };
 
 const TRANSCRIPT_GC_BATCH_SIZE = 12;
+const HOT_CACHE_HYSTERESIS_TURNS = 2;
+const DYNAMIC_LEAF_CHUNK_MEDIUM_MULTIPLIER = 1.5;
+const DYNAMIC_LEAF_CHUNK_HIGH_MULTIPLIER = 2;
+const DYNAMIC_ACTIVITY_MEDIUM_UPSHIFT_FACTOR = 0.5;
+const DYNAMIC_ACTIVITY_MEDIUM_DOWNSHIFT_FACTOR = 0.35;
+const DYNAMIC_ACTIVITY_HIGH_UPSHIFT_FACTOR = 1.0;
+const DYNAMIC_ACTIVITY_HIGH_DOWNSHIFT_FACTOR = 0.75;
 
 // ── Helpers ──────────────────────────────────────────────────────────────────
 
-/** Rough token estimate: ~4 chars per token. */
-function estimateTokens(text: string): number {
-  return Math.ceil(text.length / 4);
-}
-
 function toJson(value: unknown): string {
   const encoded = JSON.stringify(value);
   return typeof encoded === "string" ? encoded : "";
@@ -100,6 +134,10 @@ function safeString(value: unknown): string | undefined {
   return typeof value === "string" ? value : undefined;
 }
 
+function formatDurationMs(durationMs: number): string {
+  return `${durationMs}ms`;
+}
+
 function asRecord(value: unknown): Record<string, unknown> | undefined {
   return value && typeof value === "object" && !Array.isArray(value)
     ? (value as Record<string, unknown>)
@@ -835,12 +873,25 @@ function isBootstrapMessage(value: unknown): value is AgentMessage {
   return "content" in msg || ("command" in msg && "output" in msg);
 }
 
+function extractCanonicalBootstrapMessage(value: unknown): AgentMessage | null {
+  if (isBootstrapMessage(value)) {
+    return value;
+  }
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    return null;
+  }
+  const entry = value as { type?: unknown; message?: unknown };
+  if ("message" in entry) {
+    if (entry.type !== undefined && entry.type !== "message") {
+      return null;
+    }
+    return isBootstrapMessage(entry.message) ? entry.message : null;
+  }
+  return null;
+}
+
 function extractBootstrapMessageCandidate(value: unknown): AgentMessage | null {
-  const candidate =
-    value && typeof value === "object" && "message" in value
-      ? (value as { message?: unknown }).message
-      : value;
-  return isBootstrapMessage(candidate) ? candidate : null;
+  return extractCanonicalBootstrapMessage(value);
 }
 
 function parseBootstrapJsonl(raw: string, options?: {
@@ -863,9 +914,6 @@ function parseBootstrapJsonl(raw: string, options?: {
         messages.push(candidate);
         continue;
       }
-      if (options?.strict) {
-        hadMalformedLine = true;
-      }
     } catch {
       if (options?.strict) {
         hadMalformedLine = true;
@@ -1018,7 +1066,12 @@ function readFileSegment(sessionFile: string, offset: number): string | null {
   }
 }
 
-function readLastJsonlEntryBeforeOffset(sessionFile: string, offset: number, messageOnly = false): string | null {
+function readLastJsonlEntryBeforeOffset(
+  sessionFile: string,
+  offset: number,
+  messageOnly = false,
+  matcher?: (message: AgentMessage) => boolean,
+): string | null {
   const chunkSize = 16_384;
   let fd: number | null = null;
   try {
@@ -1056,11 +1109,11 @@ function readLastJsonlEntryBeforeOffset(sessionFile: string, offset: number, mes
         const candidate = trimmedEnd.slice(newlineIndex + 1).trim();
         if (candidate) {
           if (messageOnly) {
-            let isMessage = false;
+            let matchedMessage: AgentMessage | null = null;
             try {
-              isMessage = extractBootstrapMessageCandidate(JSON.parse(candidate)) != null;
+              matchedMessage = extractBootstrapMessageCandidate(JSON.parse(candidate));
             } catch { /* not valid JSON, skip */ }
-            if (!isMessage) {
+            if (!matchedMessage || (matcher && !matcher(matchedMessage))) {
               carry = trimmedEnd.slice(0, newlineIndex);
               continue;
             }
@@ -1075,11 +1128,11 @@ function readLastJsonlEntryBeforeOffset(sessionFile: string, offset: number, mes
       if (reachedStart) {
         const firstLine = trimmedEnd.trim() || null;
         if (firstLine && messageOnly) {
-          let isMessage = false;
+          let matchedMessage: AgentMessage | null = null;
           try {
-            isMessage = extractBootstrapMessageCandidate(JSON.parse(firstLine)) != null;
+            matchedMessage = extractBootstrapMessageCandidate(JSON.parse(firstLine));
           } catch { /* not valid JSON */ }
-          if (!isMessage) return null;
+          if (!matchedMessage || (matcher && !matcher(matchedMessage))) return null;
         }
         return firstLine;
       }
@@ -1155,6 +1208,7 @@ export class LcmContextEngine implements ContextEngine {
 
   private conversationStore: ConversationStore;
   private summaryStore: SummaryStore;
+  private compactionTelemetryStore: CompactionTelemetryStore;
   private assembler: ContextAssembler;
   private compaction: CompactionEngine;
   private retrieval: RetrievalEngine;
@@ -1181,13 +1235,14 @@ export class LcmContextEngine implements ContextEngine {
     this.statelessSessionPatterns = compileSessionPatterns(this.config.statelessSessionPatterns);
     this.db = database;
 
-    this.fts5Available = getLcmDbFeatures(this.db).fts5Available;
-
     // Run migrations eagerly at construction time so the schema exists
     // before any lifecycle hook fires.
     let migrationOk = false;
+    const migrationStartedAt = Date.now();
     try {
-      runLcmMigrations(this.db, { fts5Available: this.fts5Available });
+      runLcmMigrations(this.db, {
+        log: this.deps.log,
+      });
       this.migrated = true;
 
       // Verify tables were actually created
@@ -1200,16 +1255,21 @@ export class LcmContextEngine implements ContextEngine {
         );
       } else {
         migrationOk = true;
+        this.deps.log.info(
+          `[lcm] Migration run completed during engine init: duration=${formatDurationMs(Date.now() - migrationStartedAt)} fts5=${this.fts5Available}`,
+        );
         this.deps.log.debug(
           `[lcm] Migration successful — ${tables.length} tables: ${tables.map((t) => t.name).join(", ")}`,
         );
       }
     } catch (err) {
       this.deps.log.error(
-        `[lcm] Migration failed: ${err instanceof Error ? err.message : String(err)}`,
+        `[lcm] Migration failed after ${formatDurationMs(Date.now() - migrationStartedAt)}: ${err instanceof Error ? err.message : String(err)}`,
       );
     }
 
+    this.fts5Available = getLcmDbFeatures(this.db).fts5Available;
+
     // Only claim ownership of compaction when the DB is operational.
     // Without a working schema, ownsCompaction would disable the runtime's
     // built-in compaction safeguard and inflate the context budget.
@@ -1224,6 +1284,7 @@ export class LcmContextEngine implements ContextEngine {
       fts5Available: this.fts5Available,
     });
     this.summaryStore = new SummaryStore(this.db, { fts5Available: this.fts5Available });
+    this.compactionTelemetryStore = new CompactionTelemetryStore(this.db);
 
     if (!this.fts5Available) {
       this.deps.log.warn(
@@ -1269,6 +1330,7 @@ export class LcmContextEngine implements ContextEngine {
       this.conversationStore,
       this.summaryStore,
       compactionConfig,
+      this.deps.log,
     );
 
     this.retrieval = new RetrievalEngine(this.conversationStore, this.summaryStore);
@@ -1335,10 +1397,17 @@ export class LcmContextEngine implements ContextEngine {
   private recordCompactionAuthFailure(key: string): void {
     const state = this.getCircuitBreakerState(key);
     state.failures++;
+    const halfThreshold = Math.ceil(this.config.circuitBreakerThreshold / 2);
+    if (state.failures === halfThreshold && state.failures < this.config.circuitBreakerThreshold) {
+      this.deps.log.warn(
+        `[lcm] WARNING: compaction degraded — ${state.failures}/${this.config.circuitBreakerThreshold} consecutive auth failures for ${key}`,
+      );
+    }
     if (state.failures >= this.config.circuitBreakerThreshold) {
       state.openSince = Date.now();
-      console.error(
-        `[lcm] compaction circuit breaker OPEN: ${state.failures} consecutive auth failures for ${key}. Compaction halted. Will auto-retry after ${Math.round(this.config.circuitBreakerCooldownMs / 60000)}m or gateway restart.`,
+      const cooldownMin = Math.round(this.config.circuitBreakerCooldownMs / 60000);
+      this.deps.log.warn(
+        `[lcm] CIRCUIT BREAKER OPEN: compaction disabled for ${key}. Auto-retry in ${cooldownMin}m. LCM is operating in degraded mode.`,
       );
     }
   }
@@ -1349,7 +1418,7 @@ export class LcmContextEngine implements ContextEngine {
       return;
     }
     if (state.failures > 0 || state.openSince !== null) {
-      console.error(
+      this.deps.log.info(
         `[lcm] compaction circuit breaker CLOSED: successful compaction for ${key} after ${state.failures} prior failures.`,
       );
     }
@@ -1365,17 +1434,29 @@ export class LcmContextEngine implements ContextEngine {
     if (this.migrated) {
       return;
     }
-    runLcmMigrations(this.db, { fts5Available: this.fts5Available });
+    const migrationStartedAt = Date.now();
+    this.deps.log.info("[lcm] ensureMigrated: running migrations lazily");
+    runLcmMigrations(this.db, {
+      log: this.deps.log,
+    });
     this.migrated = true;
+    this.deps.log.info(
+      `[lcm] ensureMigrated: completed in ${formatDurationMs(Date.now() - migrationStartedAt)}`,
+    );
   }
 
   /**
    * Serialize mutating operations per stable session identity to prevent
    * ingest/compaction races across runtime UUID recycling.
    */
-  private async withSessionQueue<T>(queueKey: string, operation: () => Promise<T>): Promise<T> {
+  private async withSessionQueue<T>(
+    queueKey: string,
+    operation: () => Promise<T>,
+    options?: { operationName?: string; context?: string },
+  ): Promise<T> {
     const entry = this.sessionOperationQueues.get(queueKey);
     const previous = entry?.promise ?? Promise.resolve();
+    const queuedAhead = entry?.refCount ?? 0;
     let releaseQueue: () => void = () => {};
     const current = new Promise<void>((resolve) => {
       releaseQueue = resolve;
@@ -1389,7 +1470,15 @@ export class LcmContextEngine implements ContextEngine {
       this.sessionOperationQueues.set(queueKey, { promise: next, refCount: 1 });
     }
 
+    const waitStartedAt = Date.now();
     await previous.catch(() => {});
+    const waitMs = Date.now() - waitStartedAt;
+    if (options?.operationName) {
+      const detail = options.context ? ` ${options.context}` : "";
+      this.deps.log.info(
+        `[lcm] ${options.operationName}: session queue acquired queueKey=${queueKey} queuedAhead=${queuedAhead} wait=${formatDurationMs(waitMs)}${detail}`,
+      );
+    }
     try {
       return await operation();
     } finally {
@@ -1446,6 +1535,486 @@ export class LcmContextEngine implements ContextEngine {
     return cap != null && cap > 0 ? Math.min(budget, cap) : budget;
   }
 
+  /** Normalize token counters that may legitimately be zero. */
+  private normalizeOptionalCount(value: unknown): number | undefined {
+    if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
+      return undefined;
+    }
+    return Math.floor(value);
+  }
+
+  /** Treat a recent cache hit as still-hot for a couple of turns unless telemetry observed a later break. */
+  private shouldApplyHotCacheHysteresis(
+    telemetry: ConversationCompactionTelemetryRecord | null,
+  ): boolean {
+    if (!telemetry?.lastObservedCacheHitAt) {
+      return false;
+    }
+    if (
+      telemetry.lastObservedCacheBreakAt
+      && telemetry.lastObservedCacheBreakAt >= telemetry.lastObservedCacheHitAt
+    ) {
+      return false;
+    }
+    return telemetry.turnsSinceLeafCompaction <= HOT_CACHE_HYSTERESIS_TURNS;
+  }
+
+  /** Resolve the effective cache state the incremental compaction policy should react to. */
+  private resolveCacheAwareState(
+    telemetry: ConversationCompactionTelemetryRecord | null,
+  ): CacheState {
+    if (!telemetry) {
+      return "unknown";
+    }
+    if (telemetry.cacheState === "hot") {
+      return "hot";
+    }
+    if (this.shouldApplyHotCacheHysteresis(telemetry)) {
+      return "hot";
+    }
+    return telemetry.cacheState;
+  }
+
+  /** Decide whether a hot cache still has enough real token-budget headroom to skip incremental maintenance. */
+  private isComfortablyUnderTokenBudget(params: {
+    currentTokenCount?: number;
+    tokenBudget: number;
+  }): boolean {
+    if (
+      typeof params.currentTokenCount !== "number"
+      || !Number.isFinite(params.currentTokenCount)
+      || params.currentTokenCount < 0
+    ) {
+      return false;
+    }
+    const budget = Math.max(1, Math.floor(params.tokenBudget));
+    const safeBudget = Math.floor(
+      budget * (1 - this.config.cacheAwareCompaction.hotCacheBudgetHeadroomRatio),
+    );
+    return params.currentTokenCount <= safeBudget;
+  }
+
+  /** Resolve bounded dynamic leaf chunk sizes from config and the active token budget. */
+  private resolveDynamicLeafChunkBounds(tokenBudget?: number): DynamicLeafChunkBounds {
+    const floor = Math.max(1, Math.floor(this.config.leafChunkTokens));
+    const configuredMax = this.config.dynamicLeafChunkTokens.enabled
+      ? Math.max(floor, Math.floor(this.config.dynamicLeafChunkTokens.max))
+      : floor;
+    const budgetCap =
+      typeof tokenBudget === "number" &&
+      Number.isFinite(tokenBudget) &&
+      tokenBudget > 0
+        ? Math.max(floor, Math.floor(tokenBudget * this.config.contextThreshold))
+        : configuredMax;
+    const max = Math.max(floor, Math.min(configuredMax, budgetCap));
+    const medium = Math.max(
+      floor,
+      Math.min(max, Math.floor(floor * DYNAMIC_LEAF_CHUNK_MEDIUM_MULTIPLIER)),
+    );
+    const high = Math.max(
+      floor,
+      Math.min(max, Math.floor(floor * DYNAMIC_LEAF_CHUNK_HIGH_MULTIPLIER)),
+    );
+    return { floor, medium, high, max };
+  }
+
+  /** Classify the current refill rate into a simple step band with downshift hysteresis. */
+  private classifyDynamicLeafActivityBand(params: {
+    lastActivityBand?: ActivityBand;
+    tokensAccumulatedSinceLeafCompaction: number;
+    turnsSinceLeafCompaction: number;
+    floor: number;
+  }): ActivityBand {
+    const turns = Math.max(1, params.turnsSinceLeafCompaction);
+    const tokensPerTurn = params.tokensAccumulatedSinceLeafCompaction / turns;
+    const mediumUpshift = params.floor * DYNAMIC_ACTIVITY_MEDIUM_UPSHIFT_FACTOR;
+    const mediumDownshift = params.floor * DYNAMIC_ACTIVITY_MEDIUM_DOWNSHIFT_FACTOR;
+    const highUpshift = params.floor * DYNAMIC_ACTIVITY_HIGH_UPSHIFT_FACTOR;
+    const highDownshift = params.floor * DYNAMIC_ACTIVITY_HIGH_DOWNSHIFT_FACTOR;
+    const lastBand = params.lastActivityBand ?? "low";
+
+    if (lastBand === "high") {
+      if (tokensPerTurn >= highDownshift) {
+        return "high";
+      }
+      return tokensPerTurn >= mediumDownshift ? "medium" : "low";
+    }
+    if (lastBand === "medium") {
+      if (tokensPerTurn >= highUpshift) {
+        return "high";
+      }
+      if (tokensPerTurn < mediumDownshift) {
+        return "low";
+      }
+      return "medium";
+    }
+    if (tokensPerTurn >= highUpshift) {
+      return "high";
+    }
+    if (tokensPerTurn >= mediumUpshift) {
+      return "medium";
+    }
+    return "low";
+  }
+
+  /** Map an activity band to the corresponding working leaf chunk size. */
+  private resolveLeafChunkTokensForBand(
+    band: ActivityBand,
+    bounds: DynamicLeafChunkBounds,
+  ): number {
+    switch (band) {
+      case "high":
+        return bounds.high;
+      case "medium":
+        return bounds.medium;
+      default:
+        return bounds.floor;
+    }
+  }
+
+  /** Build descending fallback chunk sizes used when a provider rejects a larger chunk. */
+  private buildLeafChunkFallbacks(params: {
+    preferred: number;
+    bounds: DynamicLeafChunkBounds;
+  }): number[] {
+    const ordered = [params.preferred, params.bounds.max, params.bounds.high, params.bounds.medium, params.bounds.floor];
+    const seen = new Set<number>();
+    const fallbacks: number[] = [];
+    for (const value of ordered) {
+      const normalized = Math.max(params.bounds.floor, Math.floor(value));
+      if (seen.has(normalized)) {
+        continue;
+      }
+      seen.add(normalized);
+      fallbacks.push(normalized);
+    }
+    return fallbacks.sort((a, b) => b - a);
+  }
+
+  /** Detect provider/model token-limit failures that should trigger a lower chunk retry. */
+  private isRecoverableLeafChunkOverflowError(error: unknown): boolean {
+    const message = (error instanceof Error ? error.message : String(error)).toLowerCase();
+    if (!message) {
+      return false;
+    }
+    return [
+      "context length",
+      "context window",
+      "maximum context",
+      "max context",
+      "too many tokens",
+      "too many input tokens",
+      "input tokens",
+      "token limit",
+      "context limit",
+      "input is too large",
+      "input too large",
+      "prompt is too long",
+      "request too large",
+      "exceeds the model",
+      "exceeds context",
+    ].some((fragment) => message.includes(fragment));
+  }
+
+  /** Extract the current prompt-cache snapshot from runtime context, if present. */
+  private readPromptCacheSnapshot(runtimeContext?: Record<string, unknown>): PromptCacheSnapshot | null {
+    const promptCache = asRecord(runtimeContext?.promptCache);
+    if (!promptCache) {
+      return null;
+    }
+
+    const lastCallUsage = asRecord(promptCache.lastCallUsage);
+    const observation = asRecord(promptCache.observation);
+    const cacheRead = this.normalizeOptionalCount(lastCallUsage?.cacheRead);
+    const cacheWrite = this.normalizeOptionalCount(lastCallUsage?.cacheWrite);
+    const sawExplicitBreak = safeBoolean(observation?.broke) === true;
+    const retention = safeString(promptCache.retention)?.trim();
+    const hasUsageSignal = cacheRead !== undefined || cacheWrite !== undefined;
+    const hasObservationSignal =
+      typeof observation?.cacheRead === "number"
+      || typeof observation?.previousCacheRead === "number"
+      || sawExplicitBreak;
+
+    let cacheState: CacheState = "unknown";
+    if (sawExplicitBreak) {
+      cacheState = "cold";
+    } else if (typeof cacheRead === "number" && cacheRead > 0) {
+      cacheState = "hot";
+    } else if (hasUsageSignal || hasObservationSignal) {
+      cacheState = "cold";
+    }
+
+    return {
+      ...(cacheRead !== undefined ? { lastObservedCacheRead: cacheRead } : {}),
+      ...(cacheWrite !== undefined ? { lastObservedCacheWrite: cacheWrite } : {}),
+      cacheState,
+      ...(retention ? { retention } : {}),
+      sawExplicitBreak,
+    };
+  }
+
+  /** Persist the current turn's compaction telemetry for later policy decisions. */
+  private async updateCompactionTelemetry(params: {
+    conversationId: number;
+    runtimeContext?: Record<string, unknown>;
+    tokenBudget?: number;
+    rawTokensOutsideTail?: number;
+  }): Promise<ConversationCompactionTelemetryRecord | null> {
+    const snapshot = this.readPromptCacheSnapshot(params.runtimeContext);
+    const existing = await this.compactionTelemetryStore.getConversationCompactionTelemetry(
+      params.conversationId,
+    );
+    if (!snapshot && params.rawTokensOutsideTail === undefined) {
+      return existing;
+    }
+
+    const now = new Date();
+    const bounds = this.resolveDynamicLeafChunkBounds(params.tokenBudget);
+    const turnsSinceLeafCompaction =
+      (existing?.turnsSinceLeafCompaction ?? 0) + 1;
+    const tokensAccumulatedSinceLeafCompaction =
+      params.rawTokensOutsideTail ?? existing?.tokensAccumulatedSinceLeafCompaction ?? 0;
+    const lastActivityBand = this.classifyDynamicLeafActivityBand({
+      lastActivityBand: existing?.lastActivityBand,
+      tokensAccumulatedSinceLeafCompaction,
+      turnsSinceLeafCompaction,
+      floor: bounds.floor,
+    });
+    await this.compactionTelemetryStore.upsertConversationCompactionTelemetry({
+      conversationId: params.conversationId,
+      lastObservedCacheRead: snapshot?.lastObservedCacheRead ?? existing?.lastObservedCacheRead ?? null,
+      lastObservedCacheWrite:
+        snapshot?.lastObservedCacheWrite ?? existing?.lastObservedCacheWrite ?? null,
+      lastObservedCacheHitAt:
+        snapshot?.cacheState === "hot"
+          ? now
+          : existing?.lastObservedCacheHitAt ?? null,
+      lastObservedCacheBreakAt:
+        snapshot?.sawExplicitBreak
+          ? now
+          : existing?.lastObservedCacheBreakAt ?? null,
+      cacheState: snapshot?.cacheState ?? existing?.cacheState ?? "unknown",
+      retention: snapshot?.retention ?? existing?.retention ?? null,
+      lastLeafCompactionAt: existing?.lastLeafCompactionAt ?? null,
+      turnsSinceLeafCompaction,
+      tokensAccumulatedSinceLeafCompaction,
+      lastActivityBand,
+    });
+    const updated = await this.compactionTelemetryStore.getConversationCompactionTelemetry(
+      params.conversationId,
+    );
+    if (updated) {
+      this.deps.log.debug(
+        `[lcm] compaction telemetry updated: conversation=${params.conversationId} cacheState=${updated.cacheState} cacheRead=${updated.lastObservedCacheRead ?? "null"} cacheWrite=${updated.lastObservedCacheWrite ?? "null"} retention=${updated.retention ?? "null"} turnsSinceLeafCompaction=${updated.turnsSinceLeafCompaction} tokensSinceLeafCompaction=${updated.tokensAccumulatedSinceLeafCompaction} activityBand=${updated.lastActivityBand} rawTokensOutsideTail=${params.rawTokensOutsideTail ?? "null"} tokenBudget=${params.tokenBudget ?? "null"}`,
+      );
+    }
+    return updated;
+  }
+
+  /** Reset refill counters after any successful leaf-producing compaction. */
+  private async markLeafCompactionTelemetrySuccess(params: {
+    conversationId: number;
+    activityBand?: ActivityBand;
+  }): Promise<void> {
+    const existing = await this.compactionTelemetryStore.getConversationCompactionTelemetry(
+      params.conversationId,
+    );
+    await this.compactionTelemetryStore.upsertConversationCompactionTelemetry({
+      conversationId: params.conversationId,
+      lastObservedCacheRead: existing?.lastObservedCacheRead ?? null,
+      lastObservedCacheWrite: existing?.lastObservedCacheWrite ?? null,
+      lastObservedCacheHitAt: existing?.lastObservedCacheHitAt ?? null,
+      lastObservedCacheBreakAt: existing?.lastObservedCacheBreakAt ?? null,
+      cacheState: existing?.cacheState ?? "unknown",
+      retention: existing?.retention ?? null,
+      lastLeafCompactionAt: new Date(),
+      turnsSinceLeafCompaction: 0,
+      tokensAccumulatedSinceLeafCompaction: 0,
+      lastActivityBand: params.activityBand ?? existing?.lastActivityBand ?? "low",
+    });
+    this.deps.log.debug(
+      `[lcm] compaction telemetry reset after leaf compaction: conversation=${params.conversationId} cacheState=${existing?.cacheState ?? "unknown"} activityBand=${params.activityBand ?? existing?.lastActivityBand ?? "low"}`,
+    );
+  }
+
+  /** Emit an operational trace for the incremental compaction policy decision. */
+  private logIncrementalCompactionDecision(params: {
+    conversationId: number;
+    cacheState: CacheState;
+    activityBand: ActivityBand;
+    triggerLeafChunkTokens: number;
+    preferredLeafChunkTokens: number;
+    fallbackLeafChunkTokens: number[];
+    rawTokensOutsideTail: number;
+    threshold: number;
+    shouldCompact: boolean;
+    maxPasses: number;
+    allowCondensedPasses: boolean;
+    reason: string;
+  }): IncrementalCompactionDecision {
+    this.deps.log.info(
+      `[lcm] incremental compaction decision: conversation=${params.conversationId} cacheState=${params.cacheState} activityBand=${params.activityBand} triggerLeafChunkTokens=${params.triggerLeafChunkTokens} preferredLeafChunkTokens=${params.preferredLeafChunkTokens} fallbackLeafChunkTokens=${params.fallbackLeafChunkTokens.join(",")} rawTokensOutsideTail=${params.rawTokensOutsideTail} threshold=${params.threshold} shouldCompact=${params.shouldCompact} maxPasses=${params.maxPasses} allowCondensedPasses=${params.allowCondensedPasses} reason=${params.reason}`,
+    );
+    return {
+      shouldCompact: params.shouldCompact,
+      cacheState: params.cacheState,
+      maxPasses: params.maxPasses,
+      rawTokensOutsideTail: params.rawTokensOutsideTail,
+      threshold: params.threshold,
+      leafChunkTokens: params.preferredLeafChunkTokens,
+      fallbackLeafChunkTokens: params.fallbackLeafChunkTokens,
+      activityBand: params.activityBand,
+      allowCondensedPasses: params.allowCondensedPasses,
+    };
+  }
+
+  /** Resolve the cache-aware incremental-compaction policy for the current session. */
+  private async evaluateIncrementalCompaction(params: {
+    conversationId: number;
+    tokenBudget: number;
+    currentTokenCount?: number;
+  }): Promise<IncrementalCompactionDecision> {
+    const telemetry = await this.compactionTelemetryStore.getConversationCompactionTelemetry(
+      params.conversationId,
+    );
+    const cacheState =
+      this.config.cacheAwareCompaction.enabled
+        ? this.resolveCacheAwareState(telemetry)
+        : "unknown";
+    const bounds = this.resolveDynamicLeafChunkBounds(params.tokenBudget);
+    const activityBand =
+      this.config.dynamicLeafChunkTokens.enabled
+        ? this.classifyDynamicLeafActivityBand({
+          lastActivityBand: telemetry?.lastActivityBand,
+          tokensAccumulatedSinceLeafCompaction:
+            telemetry?.tokensAccumulatedSinceLeafCompaction ?? 0,
+          turnsSinceLeafCompaction: telemetry?.turnsSinceLeafCompaction ?? 0,
+          floor: bounds.floor,
+        })
+        : "low";
+    const triggerLeafChunkTokens =
+      this.config.dynamicLeafChunkTokens.enabled && cacheState === "hot"
+        ? bounds.max
+        : this.config.dynamicLeafChunkTokens.enabled
+          ? this.resolveLeafChunkTokensForBand(activityBand, bounds)
+          : bounds.floor;
+    const preferredLeafChunkTokens =
+      this.config.cacheAwareCompaction.enabled && (cacheState === "cold" || cacheState === "hot")
+        ? bounds.max
+        : triggerLeafChunkTokens;
+    const fallbackLeafChunkTokens = this.buildLeafChunkFallbacks({
+      preferred: preferredLeafChunkTokens,
+      bounds,
+    });
+    const leafTrigger = await this.compaction.evaluateLeafTrigger(
+      params.conversationId,
+      triggerLeafChunkTokens,
+    );
+    if (!leafTrigger.shouldCompact) {
+      return this.logIncrementalCompactionDecision({
+        conversationId: params.conversationId,
+        cacheState,
+        activityBand,
+        triggerLeafChunkTokens,
+        preferredLeafChunkTokens,
+        fallbackLeafChunkTokens,
+        rawTokensOutsideTail: leafTrigger.rawTokensOutsideTail,
+        threshold: leafTrigger.threshold,
+        shouldCompact: false,
+        maxPasses: 1,
+        allowCondensedPasses: false,
+        reason: "below-leaf-trigger",
+      });
+    }
+
+    const budgetDecision = await this.compaction.evaluate(
+      params.conversationId,
+      params.tokenBudget,
+      params.currentTokenCount,
+    );
+    if (budgetDecision.shouldCompact) {
+      return this.logIncrementalCompactionDecision({
+        conversationId: params.conversationId,
+        cacheState,
+        activityBand,
+        triggerLeafChunkTokens,
+        preferredLeafChunkTokens,
+        fallbackLeafChunkTokens,
+        rawTokensOutsideTail: leafTrigger.rawTokensOutsideTail,
+        threshold: leafTrigger.threshold,
+        shouldCompact: true,
+        maxPasses: 1,
+        allowCondensedPasses: true,
+        reason: "budget-trigger",
+      });
+    }
+
+    if (
+      cacheState === "hot"
+      && this.isComfortablyUnderTokenBudget({
+        currentTokenCount: params.currentTokenCount,
+        tokenBudget: params.tokenBudget,
+      })
+    ) {
+      return this.logIncrementalCompactionDecision({
+        conversationId: params.conversationId,
+        cacheState,
+        activityBand,
+        triggerLeafChunkTokens,
+        preferredLeafChunkTokens,
+        fallbackLeafChunkTokens,
+        rawTokensOutsideTail: leafTrigger.rawTokensOutsideTail,
+        threshold: leafTrigger.threshold,
+        shouldCompact: false,
+        maxPasses: 1,
+        allowCondensedPasses: false,
+        reason: "hot-cache-budget-headroom",
+      });
+    }
+
+    if (
+      cacheState === "hot"
+      && leafTrigger.rawTokensOutsideTail
+        < Math.floor(
+          leafTrigger.threshold * this.config.cacheAwareCompaction.hotCachePressureFactor,
+        )
+    ) {
+      return this.logIncrementalCompactionDecision({
+        conversationId: params.conversationId,
+        cacheState,
+        activityBand,
+        triggerLeafChunkTokens,
+        preferredLeafChunkTokens,
+        fallbackLeafChunkTokens,
+        rawTokensOutsideTail: leafTrigger.rawTokensOutsideTail,
+        threshold: leafTrigger.threshold,
+        shouldCompact: false,
+        maxPasses: 1,
+        allowCondensedPasses: false,
+        reason: "hot-cache-defer",
+      });
+    }
+
+    const maxPasses =
+      cacheState === "cold"
+        ? Math.max(1, this.config.cacheAwareCompaction.maxColdCacheCatchupPasses)
+        : 1;
+    return this.logIncrementalCompactionDecision({
+      conversationId: params.conversationId,
+      cacheState,
+      activityBand,
+      triggerLeafChunkTokens,
+      preferredLeafChunkTokens,
+      fallbackLeafChunkTokens,
+      rawTokensOutsideTail: leafTrigger.rawTokensOutsideTail,
+      threshold: leafTrigger.threshold,
+      shouldCompact: true,
+      maxPasses,
+      allowCondensedPasses: cacheState !== "hot",
+      reason: cacheState === "cold" ? "cold-cache-catchup" : "leaf-trigger",
+    });
+  }
+
   /** Resolve an LCM conversation id from a session key via the session store. */
   private async resolveConversationIdForSessionKey(
     sessionKey: string,
@@ -1475,6 +2044,23 @@ export class LcmContextEngine implements ContextEngine {
     }
   }
 
+  /** Format stable session identifiers for LCM diagnostic logs. */
+  private formatSessionLogContext(params: {
+    conversationId: number;
+    sessionId: string;
+    sessionKey?: string;
+  }): string {
+    const parts = [
+      `conversation=${params.conversationId}`,
+      `session=${params.sessionId}`,
+    ];
+    const trimmedSessionKey = params.sessionKey?.trim();
+    if (trimmedSessionKey) {
+      parts.push(`sessionKey=${trimmedSessionKey}`);
+    }
+    return parts.join(" ");
+  }
+
   /** Build a summarize callback with runtime provider fallback handling. */
   private async resolveSummarize(params: {
     legacyParams?: Record<string, unknown>;
@@ -1510,11 +2096,13 @@ export class LcmContextEngine implements ContextEngine {
           breakerKey: runtimeSummarizer.breakerKey,
         };
       }
-      console.error(`[lcm] resolveSummarize: createLcmSummarizeFromLegacyParams returned undefined`);
+      this.deps.log.error(`[lcm] resolveSummarize: createLcmSummarizeFromLegacyParams returned undefined`);
     } catch (err) {
-      console.error(`[lcm] resolveSummarize failed, using emergency fallback:`, err instanceof Error ? err.message : err);
+      this.deps.log.error(
+        `[lcm] resolveSummarize failed, using emergency fallback: ${describeLogError(err)}`,
+      );
     }
-    console.error(`[lcm] resolveSummarize: FALLING BACK TO EMERGENCY TRUNCATION`);
+    this.deps.log.error(`[lcm] resolveSummarize: FALLING BACK TO EMERGENCY TRUNCATION`);
     return { summarize: createEmergencyFallbackSummarize(), summaryModel: "unknown" };
   }
 
@@ -1858,12 +2446,24 @@ export class LcmContextEngine implements ContextEngine {
     hasOverlap: boolean;
   }> {
     const { sessionId, conversationId, historicalMessages } = params;
+    const startedAt = Date.now();
+    const sessionContext = this.formatSessionLogContext({
+      conversationId,
+      sessionId,
+      sessionKey: params.sessionKey,
+    });
     if (historicalMessages.length === 0) {
+      this.deps.log.info(
+        `[lcm] reconcileSessionTail: skipped for ${sessionContext} duration=${formatDurationMs(Date.now() - startedAt)} historicalMessages=0 reason=empty-history`,
+      );
       return { blockedByImportCap: false, importedMessages: 0, hasOverlap: false };
     }
 
     const latestDbMessage = await this.conversationStore.getLastMessage(conversationId);
     if (!latestDbMessage) {
+      this.deps.log.info(
+        `[lcm] reconcileSessionTail: skipped for ${sessionContext} duration=${formatDurationMs(Date.now() - startedAt)} historicalMessages=${historicalMessages.length} reason=no-db-tail`,
+      );
       return { blockedByImportCap: false, importedMessages: 0, hasOverlap: false };
     }
 
@@ -1885,6 +2485,9 @@ export class LcmContextEngine implements ContextEngine {
         }
       }
       if (dbOccurrences === historicalOccurrences) {
+        this.deps.log.info(
+          `[lcm] reconcileSessionTail: fast path for ${sessionContext} duration=${formatDurationMs(Date.now() - startedAt)} historicalMessages=${historicalMessages.length} importedMessages=0 overlap=true`,
+        );
         return { blockedByImportCap: false, importedMessages: 0, hasOverlap: true };
       }
     }
@@ -1937,9 +2540,15 @@ export class LcmContextEngine implements ContextEngine {
     }
 
     if (anchorIndex < 0) {
+      this.deps.log.info(
+        `[lcm] reconcileSessionTail: no anchor for ${sessionContext} duration=${formatDurationMs(Date.now() - startedAt)} historicalMessages=${historicalMessages.length} importedMessages=0 overlap=false`,
+      );
       return { blockedByImportCap: false, importedMessages: 0, hasOverlap: false };
     }
     if (anchorIndex >= historicalMessages.length - 1) {
+      this.deps.log.info(
+        `[lcm] reconcileSessionTail: anchor at tip for ${sessionContext} duration=${formatDurationMs(Date.now() - startedAt)} historicalMessages=${historicalMessages.length} importedMessages=0 overlap=true`,
+      );
       return { blockedByImportCap: false, importedMessages: 0, hasOverlap: true };
     }
 
@@ -1947,7 +2556,12 @@ export class LcmContextEngine implements ContextEngine {
 
     const existingDbCount = await this.conversationStore.getMessageCount(conversationId);
     if (existingDbCount > 0 && missingTail.length > Math.max(existingDbCount * 0.2, 50)) {
-      console.error(`[lcm] reconcileSessionTail: import cap exceeded — would import ${missingTail.length} messages (existing: ${existingDbCount}). Aborting to prevent flood.`);
+      this.deps.log.warn(
+        `[lcm] reconcileSessionTail: import cap exceeded for ${sessionContext} — would import ${missingTail.length} messages (existing: ${existingDbCount}). Aborting to prevent flood.`,
+      );
+      this.deps.log.info(
+        `[lcm] reconcileSessionTail: blocked for ${sessionContext} duration=${formatDurationMs(Date.now() - startedAt)} historicalMessages=${historicalMessages.length} missingTail=${missingTail.length} existingDbCount=${existingDbCount}`,
+      );
       return { blockedByImportCap: true, importedMessages: 0, hasOverlap: true };
     }
 
@@ -1959,9 +2573,43 @@ export class LcmContextEngine implements ContextEngine {
       }
     }
 
+    this.deps.log.info(
+      `[lcm] reconcileSessionTail: slow path for ${sessionContext} duration=${formatDurationMs(Date.now() - startedAt)} historicalMessages=${historicalMessages.length} anchorIndex=${anchorIndex} missingTail=${missingTail.length} importedMessages=${importedMessages}`,
+    );
     return { blockedByImportCap: false, importedMessages, hasOverlap: true };
   }
 
+  /**
+   * Persist bootstrap checkpoint metadata anchored to the current DB frontier.
+   *
+   * We intentionally checkpoint the session file's current EOF while hashing the
+   * latest persisted DB message. This keeps append-only recovery aligned with the
+   * canonical LCM frontier even when trailing transcript entries are pruned or
+   * otherwise noncanonical.
+   */
+  private async refreshBootstrapState(params: {
+    conversationId: number;
+    sessionFile: string;
+    fileStats?: { size: number; mtimeMs: number };
+  }): Promise<void> {
+    const latestDbMessage = await this.conversationStore.getLastMessage(params.conversationId);
+    const fileStats = params.fileStats ?? statSync(params.sessionFile);
+    await this.summaryStore.upsertConversationBootstrapState({
+      conversationId: params.conversationId,
+      sessionFilePath: params.sessionFile,
+      lastSeenSize: fileStats.size,
+      lastSeenMtimeMs: Math.trunc(fileStats.mtimeMs),
+      lastProcessedOffset: fileStats.size,
+      lastProcessedEntryHash: latestDbMessage
+        ? createBootstrapEntryHash({
+            role: latestDbMessage.role,
+            content: latestDbMessage.content,
+            tokenCount: latestDbMessage.tokenCount,
+          })
+        : null,
+    });
+  }
+
   async bootstrap(params: {
     sessionId: string;
     sessionFile: string;
@@ -1982,6 +2630,11 @@ export class LcmContextEngine implements ContextEngine {
       };
     }
     this.ensureMigrated();
+    const startedAt = Date.now();
+    const sessionLabel = [
+      `session=${params.sessionId}`,
+      ...(params.sessionKey?.trim() ? [`sessionKey=${params.sessionKey.trim()}`] : []),
+    ].join(" ");
     const sessionFileStats = statSync(params.sessionFile);
     const sessionFileSize = sessionFileStats.size;
     const sessionFileMtimeMs = Math.trunc(sessionFileStats.mtimeMs);
@@ -1992,19 +2645,14 @@ export class LcmContextEngine implements ContextEngine {
         this.conversationStore.withTransaction(async () => {
           const persistBootstrapState = async (
             conversationId: number,
-            historicalMessages: AgentMessage[],
           ): Promise<void> => {
-            const lastMessage =
-              historicalMessages.length > 0
-                ? toStoredMessage(historicalMessages[historicalMessages.length - 1]!)
-                : null;
-            await this.summaryStore.upsertConversationBootstrapState({
+            await this.refreshBootstrapState({
               conversationId,
-              sessionFilePath: params.sessionFile,
-              lastSeenSize: sessionFileSize,
-              lastSeenMtimeMs: sessionFileMtimeMs,
-              lastProcessedOffset: sessionFileSize,
-              lastProcessedEntryHash: createBootstrapEntryHash(lastMessage),
+              sessionFile: params.sessionFile,
+              fileStats: {
+                size: sessionFileSize,
+                mtimeMs: sessionFileMtimeMs,
+              },
             });
           };
 
@@ -2029,6 +2677,9 @@ export class LcmContextEngine implements ContextEngine {
             if (!conversation.bootstrappedAt) {
               await this.conversationStore.markConversationBootstrapped(conversationId);
             }
+            this.deps.log.info(
+              `[lcm] bootstrap: checkpoint hit conversation=${conversationId} ${sessionLabel} existingCount=${existingCount} duration=${formatDurationMs(Date.now() - startedAt)}`,
+            );
             return {
               bootstrapped: false,
               importedMessages: 0,
@@ -2055,6 +2706,7 @@ export class LcmContextEngine implements ContextEngine {
               params.sessionFile,
               bootstrapState.lastProcessedOffset,
               true,
+              (message) => createBootstrapEntryHash(toStoredMessage(message)) === latestDbHash,
             );
             const tailEntryMessage = readBootstrapMessageFromJsonLine(tailEntryRaw);
             const tailEntryHash = tailEntryMessage
@@ -2088,13 +2740,9 @@ export class LcmContextEngine implements ContextEngine {
                   }
                 }
 
-                const lastAppendedMessage =
-                  appended.messages.length > 0
-                    ? appended.messages[appended.messages.length - 1]!
-                    : tailEntryMessage;
-                await persistBootstrapState(
-                  conversationId,
-                  lastAppendedMessage ? [lastAppendedMessage] : [],
+                await persistBootstrapState(conversationId);
+                this.deps.log.info(
+                  `[lcm] bootstrap: append-only conversation=${conversationId} ${sessionLabel} existingCount=${existingCount} appendedMessages=${appended.messages.length} importedMessages=${importedMessages} duration=${formatDurationMs(Date.now() - startedAt)}`,
                 );
 
                 if (importedMessages > 0) {
@@ -2115,6 +2763,9 @@ export class LcmContextEngine implements ContextEngine {
           }
 
           const historicalMessages = await readLeafPathMessages(params.sessionFile);
+          this.deps.log.info(
+            `[lcm] bootstrap: full transcript read conversation=${conversationId} ${sessionLabel} existingCount=${existingCount} historicalMessages=${historicalMessages.length} duration=${formatDurationMs(Date.now() - startedAt)}`,
+          );
 
           // First-time import path: no LCM rows yet, so seed directly from the
           // active leaf context snapshot.
@@ -2126,7 +2777,7 @@ export class LcmContextEngine implements ContextEngine {
 
             if (bootstrapMessages.length === 0) {
               await this.conversationStore.markConversationBootstrapped(conversationId);
-              await persistBootstrapState(conversationId, historicalMessages);
+              await persistBootstrapState(conversationId);
               return {
                 bootstrapped: false,
                 importedMessages: 0,
@@ -2152,18 +2803,22 @@ export class LcmContextEngine implements ContextEngine {
               inserted.map((record) => record.messageId),
             );
             await this.conversationStore.markConversationBootstrapped(conversationId);
-            await persistBootstrapState(conversationId, historicalMessages);
 
             // Prune HEARTBEAT_OK turns from the freshly imported data
             if (this.config.pruneHeartbeatOk) {
               const pruned = await this.pruneHeartbeatOkTurns(conversationId);
               if (pruned > 0) {
-                console.error(
+                this.deps.log.info(
                   `[lcm] bootstrap: pruned ${pruned} HEARTBEAT_OK messages from conversation ${conversationId}`,
                 );
               }
             }
 
+            await persistBootstrapState(conversationId);
+            this.deps.log.info(
+              `[lcm] bootstrap: initial import conversation=${conversationId} ${sessionLabel} importedMessages=${inserted.length} sourceMessages=${historicalMessages.length} duration=${formatDurationMs(Date.now() - startedAt)}`,
+            );
+
             return {
               bootstrapped: true,
               importedMessages: inserted.length,
@@ -2178,6 +2833,9 @@ export class LcmContextEngine implements ContextEngine {
             conversationId,
             historicalMessages,
           });
+          this.deps.log.info(
+            `[lcm] bootstrap: reconcile finished conversation=${conversationId} ${sessionLabel} importedMessages=${reconcile.importedMessages} overlap=${reconcile.hasOverlap} blockedByImportCap=${reconcile.blockedByImportCap} duration=${formatDurationMs(Date.now() - startedAt)}`,
+          );
 
           if (reconcile.blockedByImportCap) {
             return {
@@ -2192,7 +2850,7 @@ export class LcmContextEngine implements ContextEngine {
           }
 
           if (reconcile.importedMessages > 0) {
-            await persistBootstrapState(conversationId, historicalMessages);
+            await persistBootstrapState(conversationId);
             return {
               bootstrapped: true,
               importedMessages: reconcile.importedMessages,
@@ -2201,7 +2859,7 @@ export class LcmContextEngine implements ContextEngine {
           }
 
           if (reconcile.hasOverlap) {
-            await persistBootstrapState(conversationId, historicalMessages);
+            await persistBootstrapState(conversationId);
           }
 
           if (conversation.bootstrappedAt) {
@@ -2220,6 +2878,7 @@ export class LcmContextEngine implements ContextEngine {
               : "conversation already has messages",
           };
         }),
+      { operationName: "bootstrap", context: sessionLabel },
     );
 
     // Post-bootstrap pruning: clean HEARTBEAT_OK turns that were already
@@ -2233,19 +2892,25 @@ export class LcmContextEngine implements ContextEngine {
         if (conversation) {
           const pruned = await this.pruneHeartbeatOkTurns(conversation.conversationId);
           if (pruned > 0) {
-            console.error(
+            await this.refreshBootstrapState({
+              conversationId: conversation.conversationId,
+              sessionFile: params.sessionFile,
+            });
+            this.deps.log.info(
               `[lcm] bootstrap: retroactively pruned ${pruned} HEARTBEAT_OK messages from conversation ${conversation.conversationId}`,
             );
           }
         }
       } catch (err) {
-        console.error(
-          `[lcm] bootstrap: heartbeat pruning failed:`,
-          err instanceof Error ? err.message : err,
+        this.deps.log.warn(
+          `[lcm] bootstrap: heartbeat pruning failed: ${describeLogError(err)}`,
         );
       }
     }
 
+    this.deps.log.info(
+      `[lcm] bootstrap: done ${sessionLabel} bootstrapped=${result.bootstrapped} importedMessages=${result.importedMessages} reason=${result.reason ?? "none"} duration=${formatDurationMs(Date.now() - startedAt)}`,
+    );
     return result;
   }
 
@@ -2385,6 +3050,12 @@ export class LcmContextEngine implements ContextEngine {
       };
     }
 
+    const rewriteTranscriptEntries = params.runtimeContext.rewriteTranscriptEntries;
+    const startedAt = Date.now();
+    const sessionLabel = [
+      `session=${params.sessionId}`,
+      ...(params.sessionKey?.trim() ? [`sessionKey=${params.sessionKey.trim()}`] : []),
+    ].join(" ");
     return this.withSessionQueue(
       this.resolveSessionQueueKey(params.sessionId, params.sessionKey),
       async () => {
@@ -2406,6 +3077,9 @@ export class LcmContextEngine implements ContextEngine {
           { limit: TRANSCRIPT_GC_BATCH_SIZE },
         );
         if (candidates.length === 0) {
+          this.deps.log.info(
+            `[lcm] maintain: no transcript GC candidates conversation=${conversation.conversationId} ${sessionLabel} duration=${formatDurationMs(Date.now() - startedAt)}`,
+          );
           return {
             changed: false,
             bytesFreed: 0,
@@ -2441,6 +3115,9 @@ export class LcmContextEngine implements ContextEngine {
         }
 
         if (replacements.length === 0) {
+          this.deps.log.info(
+            `[lcm] maintain: no matching transcript entries conversation=${conversation.conversationId} ${sessionLabel} candidates=${candidates.length} duration=${formatDurationMs(Date.now() - startedAt)}`,
+          );
           return {
             changed: false,
             bytesFreed: 0,
@@ -2449,35 +3126,29 @@ export class LcmContextEngine implements ContextEngine {
           };
         }
 
-        const result = await params.runtimeContext.rewriteTranscriptEntries({
+        const result = await rewriteTranscriptEntries({
           replacements,
         });
 
         if (result.changed) {
           try {
-            const fileStat = statSync(params.sessionFile);
-            const newSize = fileStat.size;
-            const newMtimeMs = Math.trunc(fileStat.mtimeMs);
-            const lastEntryRaw = readLastJsonlEntryBeforeOffset(params.sessionFile, newSize, true);
-            const lastEntryMsg = readBootstrapMessageFromJsonLine(lastEntryRaw);
-            const lastEntryHash = lastEntryMsg ? createBootstrapEntryHash(toStoredMessage(lastEntryMsg)) : null;
-            if (lastEntryHash) {
-              await this.summaryStore.upsertConversationBootstrapState({
-                conversationId: conversation.conversationId,
-                sessionFilePath: params.sessionFile,
-                lastSeenSize: newSize,
-                lastSeenMtimeMs: newMtimeMs,
-                lastProcessedOffset: newSize,
-                lastProcessedEntryHash: lastEntryHash,
-              });
-            }
+            await this.refreshBootstrapState({
+              conversationId: conversation.conversationId,
+              sessionFile: params.sessionFile,
+            });
           } catch (e) {
-            console.error("[lcm] Failed to update bootstrap checkpoint after maintain:", e);
+            this.deps.log.warn(
+              `[lcm] Failed to update bootstrap checkpoint after maintain: ${describeLogError(e)}`,
+            );
           }
         }
 
+        this.deps.log.info(
+          `[lcm] maintain: done conversation=${conversation.conversationId} ${sessionLabel} candidates=${candidates.length} replacements=${replacements.length} changed=${result.changed} rewrittenEntries=${result.rewrittenEntries} bytesFreed=${result.bytesFreed} duration=${formatDurationMs(Date.now() - startedAt)}`,
+        );
         return result;
       },
+      { operationName: "maintain", context: sessionLabel },
     );
   }
   private async ingestSingle(params: {
@@ -2490,6 +3161,34 @@ export class LcmContextEngine implements ContextEngine {
     if (isHeartbeat) {
       return { ingested: false };
     }
+
+    // Skip assistant messages that failed with an error and have no useful content.
+    // These occur when an API call returns a 500 or similar transient error.
+    // Ingesting them pollutes the LCM database: on retry, the error messages
+    // accumulate and get assembled into context, creating a positive feedback
+    // loop where each retry sends an increasingly large (and malformed) payload
+    // that continues to fail.
+    if (message.role === "assistant") {
+      const topLevel = message as unknown as Record<string, unknown>;
+      const stopReason =
+        typeof topLevel.stopReason === "string"
+          ? topLevel.stopReason
+          : typeof topLevel.stop_reason === "string"
+            ? topLevel.stop_reason
+            : undefined;
+      if (stopReason === "error" || stopReason === "aborted") {
+        const content = topLevel.content;
+        const isEmpty =
+          content === undefined ||
+          content === null ||
+          content === "" ||
+          (Array.isArray(content) && content.length === 0);
+        if (isEmpty) {
+          return { ingested: false };
+        }
+      }
+    }
+
     const stored = toStoredMessage(message);
 
     // Get or create conversation for this session
@@ -2570,6 +3269,13 @@ export class LcmContextEngine implements ContextEngine {
     return this.withSessionQueue(
       this.resolveSessionQueueKey(params.sessionId, params.sessionKey),
       () => this.ingestSingle(params),
+      {
+        operationName: "ingest",
+        context: [
+          `session=${params.sessionId}`,
+          ...(params.sessionKey?.trim() ? [`sessionKey=${params.sessionKey.trim()}`] : []),
+        ].join(" "),
+      },
     );
   }
 
@@ -2606,6 +3312,14 @@ export class LcmContextEngine implements ContextEngine {
         }
         return { ingestedCount };
       },
+      {
+        operationName: "ingestBatch",
+        context: [
+          `session=${params.sessionId}`,
+          ...(params.sessionKey?.trim() ? [`sessionKey=${params.sessionKey.trim()}`] : []),
+          `messages=${params.messages.length}`,
+        ].join(" "),
+      },
     );
   }
 
@@ -2630,6 +3344,11 @@ export class LcmContextEngine implements ContextEngine {
       return;
     }
     this.ensureMigrated();
+    const startedAt = Date.now();
+    const sessionLabel = [
+      `session=${params.sessionId}`,
+      ...(params.sessionKey?.trim() ? [`sessionKey=${params.sessionKey.trim()}`] : []),
+    ].join(" ");
 
     // Dedup guard: prevent duplicate ingestion when gateway restart replays
     // full history. Run on newMessages BEFORE prepending autoCompactionSummary
@@ -2651,6 +3370,9 @@ export class LcmContextEngine implements ContextEngine {
 
     ingestBatch.push(...dedupedNewMessages);
     if (ingestBatch.length === 0) {
+      this.deps.log.info(
+        `[lcm] afterTurn: nothing to ingest ${sessionLabel} newMessages=${newMessages.length} duration=${formatDurationMs(Date.now() - startedAt)}`,
+      );
       return;
     }
 
@@ -2663,9 +3385,8 @@ export class LcmContextEngine implements ContextEngine {
       });
     } catch (err) {
       // Never compact a stale or partially ingested frontier.
-      console.error(
-        `[lcm] afterTurn: ingest failed, skipping compaction:`,
-        err instanceof Error ? err.message : err,
+      this.deps.log.error(
+        `[lcm] afterTurn: ingest failed, skipping compaction: ${describeLogError(err)}`,
       );
       return;
     }
@@ -2679,16 +3400,30 @@ export class LcmContextEngine implements ContextEngine {
         if (conversation) {
           const pruned = await this.pruneHeartbeatOkTurns(conversation.conversationId);
           if (pruned > 0) {
-            console.error(
-              `[lcm] afterTurn: pruned ${pruned} heartbeat ack messages from conversation ${conversation.conversationId}`,
+            const sessionContext = this.formatSessionLogContext({
+              conversationId: conversation.conversationId,
+              sessionId: params.sessionId,
+              sessionKey: params.sessionKey,
+            });
+            try {
+              await this.refreshBootstrapState({
+                conversationId: conversation.conversationId,
+                sessionFile: params.sessionFile,
+              });
+            } catch (err) {
+              this.deps.log.warn(
+                `[lcm] afterTurn: heartbeat pruning checkpoint refresh failed for ${sessionContext}: ${describeLogError(err)}`,
+              );
+            }
+            this.deps.log.info(
+              `[lcm] afterTurn: pruned ${pruned} heartbeat ack messages for ${sessionContext}`,
             );
             return;
           }
         }
       } catch (err) {
-        console.error(
-          `[lcm] afterTurn: heartbeat pruning failed:`,
-          err instanceof Error ? err.message : err,
+        this.deps.log.warn(
+          `[lcm] afterTurn: heartbeat pruning failed: ${describeLogError(err)}`,
         );
       }
     }
@@ -2702,16 +3437,44 @@ export class LcmContextEngine implements ContextEngine {
     });
     const tokenBudget = this.applyAssemblyBudgetCap(resolvedTokenBudget ?? DEFAULT_AFTER_TURN_TOKEN_BUDGET);
     if (resolvedTokenBudget === undefined) {
-      console.warn(
+      this.deps.log.warn(
         `[lcm] afterTurn: tokenBudget not provided; using default ${DEFAULT_AFTER_TURN_TOKEN_BUDGET}`,
       );
     }
 
     const liveContextTokens = estimateSessionTokenCountForAfterTurn(params.messages);
+    const conversation = await this.conversationStore.getConversationForSession({
+      sessionId: params.sessionId,
+      sessionKey: params.sessionKey,
+    });
+    if (!conversation) {
+      this.deps.log.info(
+        `[lcm] afterTurn: conversation lookup missed ${sessionLabel} ingestBatch=${ingestBatch.length} duration=${formatDurationMs(Date.now() - startedAt)}`,
+      );
+      return;
+    }
+
+    try {
+      const rawLeafTrigger = await this.compaction.evaluateLeafTrigger(conversation.conversationId);
+      await this.updateCompactionTelemetry({
+        conversationId: conversation.conversationId,
+        runtimeContext: asRecord(params.runtimeContext),
+        tokenBudget,
+        rawTokensOutsideTail: rawLeafTrigger.rawTokensOutsideTail,
+      });
+    } catch (err) {
+      this.deps.log.warn(
+        `[lcm] afterTurn: compaction telemetry update failed: ${describeLogError(err)}`,
+      );
+    }
 
     try {
-      const leafTrigger = await this.evaluateLeafTrigger(params.sessionId, params.sessionKey);
-      if (leafTrigger.shouldCompact) {
+      const leafDecision = await this.evaluateIncrementalCompaction({
+        conversationId: conversation.conversationId,
+        tokenBudget,
+        currentTokenCount: liveContextTokens,
+      });
+      if (leafDecision.shouldCompact) {
         this.compactLeafAsync({
           sessionId: params.sessionId,
           sessionKey: params.sessionKey,
@@ -2719,6 +3482,11 @@ export class LcmContextEngine implements ContextEngine {
           tokenBudget,
           currentTokenCount: liveContextTokens,
           legacyParams,
+          maxPasses: leafDecision.maxPasses,
+          leafChunkTokens: leafDecision.leafChunkTokens,
+          fallbackLeafChunkTokens: leafDecision.fallbackLeafChunkTokens,
+          activityBand: leafDecision.activityBand,
+          allowCondensedPasses: leafDecision.allowCondensedPasses,
         }).catch(() => {
           // Leaf compaction is best-effort and should not fail the caller.
         });
@@ -2740,6 +3508,10 @@ export class LcmContextEngine implements ContextEngine {
     } catch {
       // Proactive compaction is best-effort in the post-turn lifecycle.
     }
+
+    this.deps.log.info(
+      `[lcm] afterTurn: done conversation=${conversation.conversationId} ${sessionLabel} newMessages=${newMessages.length} dedupedMessages=${dedupedNewMessages.length} ingestedMessages=${ingestBatch.length} duration=${formatDurationMs(Date.now() - startedAt)}`,
+    );
   }
 
   async assemble(params: {
@@ -2758,12 +3530,20 @@ export class LcmContextEngine implements ContextEngine {
     }
     try {
       this.ensureMigrated();
+      const startedAt = Date.now();
+      const sessionLabel = [
+        `session=${params.sessionId}`,
+        ...(params.sessionKey?.trim() ? [`sessionKey=${params.sessionKey.trim()}`] : []),
+      ].join(" ");
 
       const conversation = await this.conversationStore.getConversationForSession({
         sessionId: params.sessionId,
         sessionKey: params.sessionKey,
       });
       if (!conversation) {
+        this.deps.log.info(
+          `[lcm] assemble: conversation lookup missed ${sessionLabel} duration=${formatDurationMs(Date.now() - startedAt)}`,
+        );
         return {
           messages: params.messages,
           estimatedTokens: 0,
@@ -2772,6 +3552,9 @@ export class LcmContextEngine implements ContextEngine {
 
       const contextItems = await this.summaryStore.getContextItems(conversation.conversationId);
       if (contextItems.length === 0) {
+        this.deps.log.info(
+          `[lcm] assemble: no context items conversation=${conversation.conversationId} ${sessionLabel} duration=${formatDurationMs(Date.now() - startedAt)}`,
+        );
         return {
           messages: params.messages,
           estimatedTokens: 0,
@@ -2783,6 +3566,9 @@ export class LcmContextEngine implements ContextEngine {
       // the live path to avoid dropping prompt context.
       const hasSummaryItems = contextItems.some((item) => item.itemType === "summary");
       if (!hasSummaryItems && contextItems.length < params.messages.length) {
+        this.deps.log.info(
+          `[lcm] assemble: falling back to live context conversation=${conversation.conversationId} ${sessionLabel} contextItems=${contextItems.length} liveMessages=${params.messages.length} duration=${formatDurationMs(Date.now() - startedAt)}`,
+        );
         return {
           messages: params.messages,
           estimatedTokens: 0,
@@ -2807,12 +3593,19 @@ export class LcmContextEngine implements ContextEngine {
       // If assembly produced no messages for a non-empty live session,
       // fail safe to the live context.
       if (assembled.messages.length === 0 && params.messages.length > 0) {
+        this.deps.log.info(
+          `[lcm] assemble: empty assembled output, using live context conversation=${conversation.conversationId} ${sessionLabel} contextItems=${contextItems.length} tokenBudget=${tokenBudget} duration=${formatDurationMs(Date.now() - startedAt)}`,
+        );
         return {
           messages: params.messages,
           estimatedTokens: 0,
         };
       }
 
+      this.deps.log.info(
+        `[lcm] assemble: done conversation=${conversation.conversationId} ${sessionLabel} contextItems=${contextItems.length} hasSummaryItems=${hasSummaryItems} inputMessages=${params.messages.length} outputMessages=${assembled.messages.length} tokenBudget=${tokenBudget} estimatedTokens=${assembled.estimatedTokens} duration=${formatDurationMs(Date.now() - startedAt)}`,
+      );
+
       const result: AssembleResultWithSystemPrompt = {
         messages: assembled.messages,
         estimatedTokens: assembled.estimatedTokens,
@@ -2821,7 +3614,10 @@ export class LcmContextEngine implements ContextEngine {
           : {}),
       };
       return result;
-    } catch {
+    } catch (err) {
+      this.deps.log.info(
+        `[lcm] assemble: failed for session=${params.sessionId}${params.sessionKey?.trim() ? ` sessionKey=${params.sessionKey.trim()}` : ""} error=${describeLogError(err)}`,
+      );
       return {
         messages: params.messages,
         estimatedTokens: 0,
@@ -2856,7 +3652,7 @@ export class LcmContextEngine implements ContextEngine {
     return this.compaction.evaluateLeafTrigger(conversation.conversationId);
   }
 
-  /** Run one incremental leaf compaction pass in the per-session queue. */
+  /** Run one or more incremental leaf compaction passes in the per-session queue. */
   async compactLeafAsync(params: {
     sessionId: string;
     sessionKey?: string;
@@ -2870,6 +3666,11 @@ export class LcmContextEngine implements ContextEngine {
     legacyParams?: Record<string, unknown>;
     force?: boolean;
     previousSummaryContent?: string;
+    maxPasses?: number;
+    leafChunkTokens?: number;
+    fallbackLeafChunkTokens?: number[];
+    activityBand?: ActivityBand;
+    allowCondensedPasses?: boolean;
   }): Promise<CompactResult> {
     if (this.isStatelessSession(params.sessionKey)) {
       return {
@@ -2933,38 +3734,114 @@ export class LcmContextEngine implements ContextEngine {
           };
         }
 
-        const leafResult = await this.compaction.compactLeaf({
-          conversationId: conversation.conversationId,
-          tokenBudget,
-          summarize,
-          force: params.force,
-          previousSummaryContent: params.previousSummaryContent,
-          summaryModel,
-        });
+        const storedTokensBefore = await this.summaryStore.getContextTokenCount(
+          conversation.conversationId,
+        );
+        const maxPasses =
+          typeof params.maxPasses === "number" &&
+          Number.isFinite(params.maxPasses) &&
+          params.maxPasses > 0
+            ? Math.floor(params.maxPasses)
+            : 1;
+        const fallbackLeafChunkTokens = Array.isArray(params.fallbackLeafChunkTokens)
+          ? [...new Set(params.fallbackLeafChunkTokens
+            .filter((value): value is number => typeof value === "number" && Number.isFinite(value) && value > 0)
+            .map((value) => Math.floor(value)))]
+              .sort((a, b) => b - a)
+          : [];
+        let activeLeafChunkTokens =
+          typeof params.leafChunkTokens === "number" &&
+          Number.isFinite(params.leafChunkTokens) &&
+          params.leafChunkTokens > 0
+            ? Math.floor(params.leafChunkTokens)
+            : fallbackLeafChunkTokens[0];
+        this.deps.log.info(
+          `[lcm] compactLeafAsync start: conversation=${conversation.conversationId} session=${params.sessionId} leafChunkTokens=${activeLeafChunkTokens ?? "null"} fallbackLeafChunkTokens=${fallbackLeafChunkTokens.join(",")} maxPasses=${maxPasses} activityBand=${params.activityBand ?? "unknown"} allowCondensedPasses=${params.allowCondensedPasses !== false}`,
+        );
+
+        let rounds = 0;
+        let finalTokens = observedTokens ?? storedTokensBefore;
+        let authFailure = false;
+
+        for (let pass = 0; pass < maxPasses; pass += 1) {
+          let leafResult: Awaited<ReturnType<typeof this.compaction.compactLeaf>> | undefined;
+          while (true) {
+            try {
+              leafResult = await this.compaction.compactLeaf({
+                conversationId: conversation.conversationId,
+                tokenBudget,
+                summarize,
+                ...(activeLeafChunkTokens !== undefined ? { leafChunkTokens: activeLeafChunkTokens } : {}),
+                force: params.force,
+                previousSummaryContent: pass === 0 ? params.previousSummaryContent : undefined,
+                summaryModel,
+                allowCondensedPasses: params.allowCondensedPasses,
+              });
+              break;
+            } catch (err) {
+              const nextLeafChunkTokens = fallbackLeafChunkTokens.find(
+                (value) => activeLeafChunkTokens !== undefined && value < activeLeafChunkTokens,
+              );
+              if (!this.isRecoverableLeafChunkOverflowError(err) || nextLeafChunkTokens === undefined) {
+                throw err;
+              }
+              this.deps.log.warn(
+                `[lcm] compactLeafAsync: retrying with smaller leafChunkTokens=${nextLeafChunkTokens} after provider token-limit error: ${err instanceof Error ? err.message : String(err)}`,
+              );
+              activeLeafChunkTokens = nextLeafChunkTokens;
+            }
+          }
+          if (!leafResult) {
+            break;
+          }
+          finalTokens = leafResult.tokensAfter;
+
+          if (leafResult.authFailure) {
+            authFailure = true;
+            break;
+          }
+          if (!leafResult.actionTaken) {
+            break;
+          }
+          rounds += 1;
+          if (leafResult.tokensAfter >= leafResult.tokensBefore) {
+            break;
+          }
+        }
 
-        if (leafResult.authFailure && breakerKey) {
+        if (authFailure && breakerKey) {
           this.recordCompactionAuthFailure(breakerKey);
-        } else if (leafResult.actionTaken && breakerKey) {
+        } else if (rounds > 0 && breakerKey) {
           this.recordCompactionSuccess(breakerKey);
         }
+        if (rounds > 0) {
+          await this.markLeafCompactionTelemetrySuccess({
+            conversationId: conversation.conversationId,
+            activityBand: params.activityBand,
+          });
+        }
 
-        const tokensBefore = observedTokens ?? leafResult.tokensBefore;
+        const tokensBefore = observedTokens ?? storedTokensBefore;
+        this.deps.log.debug(
+          `[lcm] compactLeafAsync result: conversation=${conversation.conversationId} session=${params.sessionId} rounds=${rounds} compacted=${rounds > 0} authFailure=${authFailure} finalLeafChunkTokens=${activeLeafChunkTokens ?? "null"} finalTokens=${finalTokens}`,
+        );
 
         return {
           ok: true,
-          compacted: leafResult.actionTaken,
-          reason: leafResult.authFailure
+          compacted: rounds > 0,
+          reason: authFailure
             ? "provider auth failure"
-            : leafResult.actionTaken
+            : rounds > 0
               ? "compacted"
               : "below threshold",
           result: {
             tokensBefore,
-            tokensAfter: leafResult.tokensAfter,
+            tokensAfter: finalTokens,
             details: {
-              rounds: leafResult.actionTaken ? 1 : 0,
+              rounds,
               targetTokens: tokenBudget,
               mode: "leaf",
+              maxPasses,
             },
           },
         };
@@ -3093,7 +3970,7 @@ export class LcmContextEngine implements ContextEngine {
         // overflow counts can drive recovery even when persisted context is already small.
         const useSweep = manualCompactionRequested || params.compactionTarget === "threshold";
         if (useSweep) {
-          const sweepResult = await this.compaction.compactFullSweep({
+          const sweepResult = await this.compaction.compact({
             conversationId,
             tokenBudget,
             summarize,
@@ -3107,6 +3984,9 @@ export class LcmContextEngine implements ContextEngine {
           } else if (sweepResult.actionTaken && breakerKey) {
             this.recordCompactionSuccess(breakerKey);
           }
+          if (sweepResult.actionTaken) {
+            await this.markLeafCompactionTelemetrySuccess({ conversationId });
+          }
 
           return {
             ok: !sweepResult.authFailure && (sweepResult.actionTaken || !liveContextStillExceedsTarget),
@@ -3140,11 +4020,21 @@ export class LcmContextEngine implements ContextEngine {
             ? decision.threshold
             : tokenBudget;
 
+        // When forced (overflow recovery) and the caller did not supply an
+        // observed token count, assume we are at least at the token budget so
+        // compactUntilUnder does not bail with "already under target" while the
+        // live context is actually overflowing.
+        const effectiveCurrentTokens =
+          observedTokens !== undefined
+            ? observedTokens
+            : forceCompaction
+              ? tokenBudget
+              : undefined;
         const compactResult = await this.compaction.compactUntilUnder({
           conversationId,
           tokenBudget,
           targetTokens: convergenceTargetTokens,
-          ...(observedTokens !== undefined ? { currentTokens: observedTokens } : {}),
+          ...(effectiveCurrentTokens !== undefined ? { currentTokens: effectiveCurrentTokens } : {}),
           summarize,
           summaryModel,
         });
@@ -3156,6 +4046,9 @@ export class LcmContextEngine implements ContextEngine {
         }
 
         const didCompact = compactResult.rounds > 0;
+        if (didCompact) {
+          await this.markLeafCompactionTelemetrySuccess({ conversationId });
+        }
 
         return {
           ok: compactResult.success,
@@ -3344,7 +4237,7 @@ export class LcmContextEngine implements ContextEngine {
     const nextSessionKey = params.nextSessionKey?.trim() || params.sessionKey?.trim() || current?.sessionKey;
     const freshConversation = await this.conversationStore.createConversation({
       sessionId: nextSessionId,
-      sessionKey: nextSessionKey,
+      ...(nextSessionKey ? { sessionKey: nextSessionKey } : {}),
     });
     this.deps.log.info(
       `[lcm] ${params.reason} lifecycle archived prior conversation and created ${freshConversation.conversationId}`,
@@ -3455,6 +4348,10 @@ export class LcmContextEngine implements ContextEngine {
     return this.summaryStore;
   }
 
+  getCompactionTelemetryStore(): CompactionTelemetryStore {
+    return this.compactionTelemetryStore;
+  }
+
   // ── Heartbeat pruning ──────────────────────────────────────────────────
 
   /**