@remnic/core 10.0.69 → 10.0.71

package/src/buffer-drop-extraction-chunk.test.ts

@@ -1,14 +1,14 @@
-import test from "node:test";
 import assert from "node:assert/strict";
 import { mkdtemp, rm } from "node:fs/promises";
 import os from "node:os";
 import path from "node:path";
+import test from "node:test";
 import { SmartBuffer } from "./buffer.js";
 import { parseConfig } from "./config.js";
-import { StorageManager } from "./storage.js";
-import type { BufferTurn } from "./types.js";
 import { ExtractionDrain } from "./orchestration/extraction-drain.js";
 import type { ExtractionRunResult } from "./orchestration/extraction-run.js";
+import { StorageManager } from "./storage.js";
+import type { BufferTurn } from "./types.js";
 
 function makeTurn(sessionKey: string, content: string): BufferTurn {
   return {
@@ -20,7 +20,9 @@ function makeTurn(sessionKey: string, content: string): BufferTurn {
 }
 
 /** A real buffer on real storage; `restart()` is what a process restart would load from disk. */
-async function withBuffer(run: (buffer: SmartBuffer, restart: () => Promise<SmartBuffer>) => Promise<void>): Promise<void> {
+async function withBuffer(
+  run: (buffer: SmartBuffer, restart: () => Promise<SmartBuffer>) => Promise<void>
+): Promise<void> {
   const root = await mkdtemp(path.join(os.tmpdir(), "remnic-drop-chunk-"));
   try {
     const buffer = new SmartBuffer(parseConfig({}), new StorageManager(root));
@@ -43,8 +45,15 @@ test("dropExtractionChunk removes the matched prefix and the drop survives a res
     const dropped = await buffer.dropExtractionChunk("s1", turns.slice(0, 2));
 
     assert.equal(dropped, 2);
-    assert.deepEqual(buffer.getTurns("s1").map((turn) => turn.content), ["turn three"]);
-    assert.deepEqual((await restart()).getTurns("s1").map((turn) => turn.content), ["turn three"], "what a restart loads");
+    assert.deepEqual(
+      buffer.getTurns("s1").map((turn) => turn.content),
+      ["turn three"]
+    );
+    assert.deepEqual(
+      (await restart()).getTurns("s1").map((turn) => turn.content),
+      ["turn three"],
+      "what a restart loads"
+    );
     // A drop is not an extraction: the extraction bookkeeping stays untouched.
     assert.equal(buffer.getExtractionCount("s1"), 0);
   });
@@ -77,25 +86,54 @@ test("dropExtractionChunk drops the whole matched window and a restart finds the
   });
 });
 
-const FAILED: ExtractionRunResult = { status: "skipped", reason: "empty_extraction_result", retainedBuffer: true, failureClass: "parse_empty", persistedCount: 0, durableOutputCount: 0 };
+const FAILED: ExtractionRunResult = {
+  status: "skipped",
+  reason: "empty_extraction_result",
+  retainedBuffer: true,
+  failureClass: "parse_empty",
+  persistedCount: 0,
+  durableOutputCount: 0,
+};
 const DONE: ExtractionRunResult = { status: "completed", persistedCount: 1, durableOutputCount: 1 };
 
 /** w7K's probe on #3266: a pinned capture sits before a native group drained with clearMatchingTurns. */
-async function drainBehindPinnedCapture(buffer: SmartBuffer, results: ExtractionRunResult[], dropAfter: number): Promise<ExtractionRunResult> {
+async function drainBehindPinnedCapture(
+  buffer: SmartBuffer,
+  results: ExtractionRunResult[],
+  dropAfter: number,
+  overrides: {
+    getStorage?: () => Promise<{
+      saveDroppedExtractionChunk: (bufferKey: string, reason: string, turns: readonly BufferTurn[]) => Promise<string>;
+    }>;
+    runExtractionOverride?: () => Promise<ExtractionRunResult>;
+  } = {}
+): Promise<ExtractionRunResult> {
   const pinned: BufferTurn = { ...makeTurn("s1", "pinned capture"), acceptedCapture: true };
   const native = [makeTurn("s1", "a".repeat(60)), makeTurn("s1", "b".repeat(60))];
   for (const turn of [pinned, ...native]) await buffer.addTurn("s1", turn);
   await buffer.flushPendingSave();
   const drain = new ExtractionDrain({
-    config: { extractionMinUserTurns: 1, extractionMinChars: 0, extractionMaxTurnChars: 100_000, extractionChunkDropAfterFailures: dropAfter },
+    config: {
+      extractionMinUserTurns: 1,
+      extractionMinChars: 0,
+      extractionMaxTurnChars: 100_000,
+      extractionChunkDropAfterFailures: dropAfter,
+    },
     extractionWindowBudgetChars: () => 100,
     runExtraction: async (turns, options) => {
+      if (overrides.runExtractionOverride) return await overrides.runExtractionOverride();
       const result = results.shift() ?? DONE;
       // A successful run clears its chunk the way the coordinator does for a native group: by identity.
-      if (result.status === "completed") await buffer.clearAfterExtraction("s1", turns, options?.clearMatchingTurns === true ? { allowNonPrefix: true } : undefined);
+      if (result.status === "completed")
+        await buffer.clearAfterExtraction(
+          "s1",
+          turns,
+          options?.clearMatchingTurns === true ? { allowNonPrefix: true } : undefined
+        );
       return result;
     },
-    getStorage: async () => ({ saveDroppedExtractionChunk: async () => "/recovery/copy.json" }),
+    getStorage:
+      overrides.getStorage ?? (async () => ({ saveDroppedExtractionChunk: async () => "/recovery/copy.json" })),
     buffer,
   });
   return await drain.run(native, {
@@ -113,8 +151,16 @@ test("a dropped chunk behind a pinned capture is removed by identity, like a suc
     const result = await drainBehindPinnedCapture(buffer, [FAILED, DONE], 1);
 
     assert.equal(result.status, "completed");
-    assert.deepEqual(buffer.getTurns("s1").map((turn) => turn.content), ["pinned capture"], "the failed turn is gone, the pinned capture stays");
-    assert.deepEqual((await restart()).getTurns("s1").map((turn) => turn.content), ["pinned capture"], "and a restart agrees");
+    assert.deepEqual(
+      buffer.getTurns("s1").map((turn) => turn.content),
+      ["pinned capture"],
+      "the failed turn is gone, the pinned capture stays"
+    );
+    assert.deepEqual(
+      (await restart()).getTurns("s1").map((turn) => turn.content),
+      ["pinned capture"],
+      "and a restart agrees"
+    );
   });
 });
 
@@ -131,10 +177,16 @@ test("a failed chunk made only of deferred-retained turns can be dropped, and a
 
     assert.equal(dropped, 2);
     assert.deepEqual(buffer.getRetainedDeferredTurns("s1"), []);
-    assert.deepEqual(buffer.getTurns("s1").map((turn) => turn.content), ["live turn"]);
+    assert.deepEqual(
+      buffer.getTurns("s1").map((turn) => turn.content),
+      ["live turn"]
+    );
     const reloaded = await restart();
     assert.deepEqual(reloaded.getRetainedDeferredTurns("s1"), [], "retained turns stay dropped after a restart");
-    assert.deepEqual(reloaded.getTurns("s1").map((turn) => turn.content), ["live turn"]);
+    assert.deepEqual(
+      reloaded.getTurns("s1").map((turn) => turn.content),
+      ["live turn"]
+    );
   });
 });
 
@@ -149,7 +201,10 @@ test("a chunk that spans retained and live turns drops both", async () => {
     const dropped = await buffer.dropExtractionChunk("s1", [...retained, live[0]!]);
 
     assert.equal(dropped, 2);
-    assert.deepEqual((await restart()).getTurns("s1").map((turn) => turn.content), ["live two"]);
+    assert.deepEqual(
+      (await restart()).getTurns("s1").map((turn) => turn.content),
+      ["live two"]
+    );
   });
 });
 
@@ -161,7 +216,11 @@ test("deferred retention across chunks of one window stays within one retention
     for (let chunk = 0; chunk < 3; chunk++) {
       await buffer.retainDeferredTurns("s1", window.slice(chunk * 10, chunk * 10 + 10), 10, window);
     }
-    assert.equal(buffer.getRetainedDeferredTurns("s1").length, 10, "the latest chunk's retention replaces the earlier ones");
+    assert.equal(
+      buffer.getRetainedDeferredTurns("s1").length,
+      10,
+      "the latest chunk's retention replaces the earlier ones"
+    );
     assert.equal((await restart()).getRetainedDeferredTurns("s1").length, 10);
   });
 });
@@ -176,10 +235,19 @@ test("a chunk holding only some retained turns drops those and keeps the rest of
     const dropped = await buffer.dropExtractionChunk("s1", retained.slice(0, 2));
 
     assert.equal(dropped, 2);
-    assert.deepEqual(buffer.getRetainedDeferredTurns("s1").map((turn) => turn.content), ["retained three"]);
+    assert.deepEqual(
+      buffer.getRetainedDeferredTurns("s1").map((turn) => turn.content),
+      ["retained three"]
+    );
     const reloaded = await restart();
-    assert.deepEqual(reloaded.getRetainedDeferredTurns("s1").map((turn) => turn.content), ["retained three"]);
-    assert.deepEqual(reloaded.getTurns("s1").map((turn) => turn.content), ["retained three", "live turn"]);
+    assert.deepEqual(
+      reloaded.getRetainedDeferredTurns("s1").map((turn) => turn.content),
+      ["retained three"]
+    );
+    assert.deepEqual(
+      reloaded.getTurns("s1").map((turn) => turn.content),
+      ["retained three", "live turn"]
+    );
   });
 });
 
@@ -189,7 +257,154 @@ test("retention for an early chunk leaves the retained turns of a later, unproce
     await buffer.retainDeferredTurns("s1", retained);
     // Chunk 0 holds the first retained turn; nothing was deferred in it, so it clears the slot for the turns it processed.
     await buffer.retainDeferredTurns("s1", [], 0, retained.slice(0, 1));
-    assert.deepEqual(buffer.getRetainedDeferredTurns("s1").map((turn) => turn.content), ["late retained"]);
-    assert.deepEqual((await restart()).getRetainedDeferredTurns("s1").map((turn) => turn.content), ["late retained"]);
+    assert.deepEqual(
+      buffer.getRetainedDeferredTurns("s1").map((turn) => turn.content),
+      ["late retained"]
+    );
+    assert.deepEqual(
+      (await restart()).getRetainedDeferredTurns("s1").map((turn) => turn.content),
+      ["late retained"]
+    );
+  });
+});
+
+test("a no-drop drain retains a repeatedly failing chunk as durable retryable backlog", async () => {
+  await withBuffer(async (buffer, restart) => {
+    const window = [makeTurn("s1", "a".repeat(60)), makeTurn("s1", "b".repeat(60))];
+    for (const turn of window) await buffer.addTurn("s1", turn);
+    await buffer.flushPendingSave();
+
+    let sideFiles = 0;
+    const drain = new ExtractionDrain(
+      {
+        config: {
+          extractionMinUserTurns: 1,
+          extractionMinChars: 0,
+          extractionMaxTurnChars: 100_000,
+          extractionChunkDropAfterFailures: 2,
+        },
+        extractionWindowBudgetChars: () => 100,
+        runExtraction: async () => {
+          throw new Error("provider down");
+        },
+        getStorage: async () => ({
+          saveDroppedExtractionChunk: async () => {
+            sideFiles += 1;
+            return "/recovery/copy.json";
+          },
+        }),
+        buffer,
+      },
+      { dropFailedChunks: false }
+    );
+
+    const runOptions = {
+      bufferKey: "s1",
+      clearBufferAfterExtraction: true,
+      clearMatchingTurns: true,
+      skipUserTurnThreshold: false,
+      failOnExtractionFailure: false,
+      forceExtractionAttempt: false,
+    } as const;
+
+    // First window: the head chunk fails once (below the threshold) — it throws and stays.
+    await assert.rejects(() => drain.run(window, runOptions), /provider down/);
+    assert.equal(sideFiles, 0, "no recovery side-file below the threshold");
+    // Second window: the same head chunk reaches the drop threshold. The no-drop
+    // policy must RETAIN it — no side-file copy, no removal, no completion.
+    await assert.rejects(() => drain.run(window, runOptions), /provider down/);
+    assert.equal(sideFiles, 0, "a no-drop drain must never side-file a failed chunk");
+    assert.deepEqual(
+      (await restart()).getTurns("s1").map((turn) => turn.content),
+      ["a".repeat(60), "b".repeat(60)],
+      "both chunks stay in the buffer as durable retryable backlog"
+    );
+  });
+});
+
+test("a failed chunk with NO recovery copy is kept as retryable backlog, not dropped", async () => {
+  await withBuffer(async (buffer, restart) => {
+    const result = await drainBehindPinnedCapture(buffer, [FAILED, FAILED, DONE], 1, {
+      getStorage: async () => ({
+        saveDroppedExtractionChunk: async () => {
+          throw new Error("disk full");
+        },
+      }),
+    });
+
+    // dropFailedChunk returns false: the only buffered copy stays and the
+    // drain stops on that chunk (later chunks are not acked past it).
+    assert.notEqual(result.status, "completed", "no silent drop without a recovery copy");
+    assert.deepEqual(
+      (await restart()).getTurns("s1").some((turn) => turn.content === "a".repeat(60)),
+      true,
+      "the failed chunk's content stays buffered for retry"
+    );
+  });
+});
+
+test("a failing buffer removal keeps the chunk: the drop is retried next trigger", async () => {
+  await withBuffer(async (buffer, restart) => {
+    buffer.dropExtractionChunk = async () => {
+      throw new Error("buffer busy");
+    };
+    const result = await drainBehindPinnedCapture(buffer, [FAILED, FAILED, DONE], 2);
+
+    assert.notEqual(result.status, "completed");
+    assert.deepEqual(
+      (await restart()).getTurns("s1").some((turn) => turn.content === "a".repeat(60)),
+      true,
+      "the chunk stays buffered when the buffer removal itself fails"
+    );
+  });
+});
+
+test("a chunk the buffer no longer matches is kept: nothing removed is not a drop", async () => {
+  await withBuffer(async (buffer, restart) => {
+    buffer.dropExtractionChunk = async () => 0;
+    const result = await drainBehindPinnedCapture(buffer, [FAILED, FAILED, DONE], 1);
+
+    assert.notEqual(result.status, "completed");
+    assert.deepEqual(
+      (await restart()).getTurns("s1").some((turn) => turn.content === "a".repeat(60)),
+      true,
+      "nothing removed means the chunk is still buffered"
+    );
+  });
+});
+
+test("one failure below the drop threshold keeps the chunk (no drop, no side file)", async () => {
+  await withBuffer(async (buffer, restart) => {
+    const result = await drainBehindPinnedCapture(buffer, [FAILED, DONE], 3);
+
+    assert.notEqual(result.status, "completed", "a single failure below the threshold retains the window");
+    assert.deepEqual(
+      (await restart()).getTurns("s1").some((turn) => turn.content === "a".repeat(60)),
+      true,
+      "the failing chunk stays buffered for the next trigger"
+    );
+  });
+});
+
+test("a thrown chunk dropped at the threshold must not end the drain: later chunks still run", async () => {
+  await withBuffer(async (buffer, restart) => {
+    // chunk1 THROWS (model crash); at dropAfter 1 it is side-file-copied and
+    // dropped — the live drop policy then CONTINUES to the next chunk.
+    let calls = 0;
+    const result = await drainBehindPinnedCapture(buffer, [DONE], 1, {
+      getStorage: async () => ({ saveDroppedExtractionChunk: async () => "/recovery/copy.json" }),
+      runExtractionOverride: async () => {
+        calls += 1;
+        if (calls === 1) throw new Error("model connection reset");
+        return DONE;
+      },
+    });
+
+    assert.equal(result.status, "completed", "the drain finishes with the later chunk");
+    assert.deepEqual(
+      (await restart()).getTurns("s1").filter((turn) => turn.content === "a".repeat(60)),
+      [],
+      "the thrown chunk was dropped by identity"
+    );
   });
 });

package/src/buffer-turn-helpers.ts

@@ -1,4 +1,4 @@
-import type { BufferEntryState, BufferTurn } from "./types.js";
+import type { BufferEntryState, BufferState, BufferTurn } from "./types.js";
 
 export interface BufferTurnOwner {
   /**
@@ -120,6 +120,25 @@ export function subtractBufferTurns(turns: readonly BufferTurn[], drained: reado
   return kept;
 }
 
+/** Per-key pending counts over the IN-MEMORY buffer state (counts only — no
+ *  turn payloads). The claim store's answer stays authoritative for claimed
+ *  keys; callers must filter through `BufferDrainClaims.isClaimed` before
+ *  treating a count here as work. */
+export interface BufferEntryCounts {
+  turns: number;
+  retainedTurns: number;
+}
+
+export function bufferEntryCounts(state: BufferState): Map<string, BufferEntryCounts> {
+  const out = new Map<string, BufferEntryCounts>();
+  const entries = state.entries;
+  if (!entries) return out;
+  for (const [key, entry] of Object.entries(entries)) {
+    out.set(key, { turns: entry.turns.length, retainedTurns: entry.retainedTurns?.length ?? 0 });
+  }
+  return out;
+}
+
 export function bufferTurnsEqual(left: BufferTurn | undefined, right: BufferTurn): boolean {
   if (!left) return false;
   return (
@@ -200,6 +219,12 @@ export function groupTurnsByExtractionScope(turns: BufferTurn[]): ExtractionScop
       turn.writeNamespaceOverride ?? "",
       turn.principalOverride ?? "",
       scopeProfilePlanIdentity(turn.scopeProfileWritePlan),
+      // Accepted captures are deliberately accepted content: they must never
+      // share a scope group with ordinary turns, or a group-level capture
+      // policy (the explicit-mode gate, the planless-accepted null pin) is
+      // applied to ordinary conversation turns because `.some(acceptedCapture)`
+      // only looked at the merged group.
+      turn.acceptedCapture === true ? "accepted" : "",
       turn.codingContext?.projectId ?? "",
       turn.codingContext?.rootPath ?? "",
       turn.codingContext?.branch ?? "",

package/src/buffer.ts

@@ -17,6 +17,7 @@ import { scopeProfilePlanIdentity, subtractBufferTurns } from "./buffer-turn-hel
 import { ExtractionDeadlineError } from "./orchestration/extraction-run.js";
 import type { ExtractionBufferSnapshot } from "./extraction-liveness.js";
 import {
+  bufferEntryCounts,
   bufferTurnArrayIsSuffixOfSnapshot,
   bufferTurnArraysEqual,
   bufferTurnsEqual,
@@ -27,6 +28,7 @@ import {
   matchingQueuedExtractionPrefixLength,
   probeWithTimeout,
   selectRetainedDeferredTurns,
+  type BufferEntryCounts,
 } from "./buffer-turn-helpers.js";
 
 export type TriggerDecision = "extract_now" | "extract_batch" | "keep_buffering";
@@ -332,6 +334,12 @@ export class SmartBuffer {
     return this.storage.isBufferSessionClaimed?.(key) === true;
   }
 
+  /** Per-key pending counts over the IN-MEMORY state (counts only — no turn
+   *  payloads), for the daemon backlog scheduler's bounded candidate scan. */
+  entryCounts(): Map<string, BufferEntryCounts> {
+    return bufferEntryCounts(this.state);
+  }
+
   async save(): Promise<void> {
     await this.enqueueMutation(async () => this.saveUnlocked());
   }

package/src/extraction-window-drain.ts

@@ -46,22 +46,22 @@ export type DrainVerdict = "drained" | "deferred" | "failed";
 
 /**
  * Run results that deliberately RETAIN the buffer (provider breaker open,
- * per-fingerprint backoff) are deferrals, not chunk failures — counting them
- * would drop good content during a provider outage.
+ * per-fingerprint backoff, a scope policy refusal) are deferrals, not chunk
+ * failures — counting them would eventually drop good content that never got
+ * its attempt.
  */
 const DRAIN_DEFERRAL_REASONS = {
   provider_circuit_open: true,
   extraction_backoff: true,
+  scope_profile_no_writable_layer: true,
 };
 
 export function classifyDrainResult(result: ExtractionRunResult): DrainVerdict {
   if (result.status !== "skipped") return "drained";
-  // An empty result is a failure only when the run kept the window for retry;
-  // otherwise it extracted fine, found nothing durable, and cleared its turns.
-  if (result.reason === "empty_extraction_result") {
-    return result.retainedBuffer === true ? "failed" : "drained";
-  }
   if (result.reason !== undefined && result.reason in DRAIN_DEFERRAL_REASONS) return "deferred";
+  // A run that kept its window for retry did NOT consume the content: it is a
+  // chunk failure, never a drain — even when the window was retained.
+  if (result.retainedBuffer === true) return "failed";
   return "drained";
 }
 

package/src/lcm/archive.ts

@@ -188,20 +188,72 @@ export class LcmArchive {
       .all(sessionId, fromTurn, toTurn) as LcmMessage[];
   }
 
-  /** Retrieve unsummarized messages (after last leaf summary). */
-  getUnsummarizedMessages(sessionId: string): LcmMessage[] {
-    const lastLeafEnd = this.db
+  /** Highest turn index covered by a leaf summary (-1 when none exist). */
+  private lastLeafSummaryEnd(sessionId: string): number {
+    // SAFETY: the query returns exactly one aggregate column, `last_end`.
+    const row = this.db
       .prepare(
         "SELECT MAX(msg_end) as last_end FROM lcm_summary_nodes WHERE session_id = ? AND depth = 0",
       )
       .get(sessionId) as { last_end: number | null } | undefined;
+    return row?.last_end ?? -1;
+  }
 
-    const lastSummarized = lastLeafEnd?.last_end ?? -1;
-    return this.db
+  /** Retrieve unsummarized messages (after last leaf summary), optionally bounded. */
+  getUnsummarizedMessages(sessionId: string, limit?: number): LcmMessage[] {
+    const lastSummarized = this.lastLeafSummaryEnd(sessionId);
+    if (limit !== undefined && limit <= 0) return [];
+
+    const stmt = this.db.prepare(
+      limit === undefined
+        ? "SELECT * FROM lcm_messages WHERE session_id = ? AND turn_index > ? ORDER BY turn_index"
+        : "SELECT * FROM lcm_messages WHERE session_id = ? AND turn_index > ? ORDER BY turn_index LIMIT ?",
+    );
+    // SAFETY: the statement selects the lcm_messages row shape.
+    const rows = (
+      limit === undefined ? stmt.all(sessionId, lastSummarized) : stmt.all(sessionId, lastSummarized, limit)
+    ) as LcmMessage[];
+    return rows;
+  }
+
+  /** Count archived turns that no leaf summary covers yet. */
+  countUnsummarizedMessages(sessionId: string): number {
+    const lastSummarized = this.lastLeafSummaryEnd(sessionId);
+    // SAFETY: the query returns exactly one aggregate column, `n`.
+    const row = this.db
+      .prepare(
+        "SELECT COUNT(*) as n FROM lcm_messages WHERE session_id = ? AND turn_index > ?",
+      )
+      .get(sessionId, lastSummarized) as { n: number };
+    return row.n;
+  }
+
+  /**
+   * Sessions whose durable archive still holds at least `minTurns`
+   * unsummarized turns. Metadata only — used to resume outstanding summary
+   * work after a restart.
+   */
+  listSessionsWithUnsummarizedTurns(minTurns: number): string[] {
+    if (minTurns <= 0) return [];
+    // SAFETY: the query returns one `session_id` column per session.
+    const rows = this.db
       .prepare(
-        "SELECT * FROM lcm_messages WHERE session_id = ? AND turn_index > ? ORDER BY turn_index",
+        `
+        SELECT m.session_id AS session_id
+        FROM lcm_messages m
+        LEFT JOIN (
+          SELECT session_id, MAX(msg_end) AS last_end
+          FROM lcm_summary_nodes
+          WHERE depth = 0
+          GROUP BY session_id
+        ) leaves ON leaves.session_id = m.session_id
+        WHERE m.turn_index > COALESCE(leaves.last_end, -1)
+        GROUP BY m.session_id
+        HAVING COUNT(*) >= ?
+      `,
       )
-      .all(sessionId, lastSummarized) as LcmMessage[];
+      .all(minTurns) as Array<{ session_id: string }>;
+    return rows.map((row) => row.session_id);
   }
 
   /** Full-text search across all messages. */