@remnic/core 10.0.38 → 10.0.40

package/dist/write-envelope.js

@@ -12,10 +12,10 @@ import {
   isMemoryCategory,
   isSealedMemoryEnvelope,
   sealedWriteToLegacyArgs
-} from "./chunk-TDCR6PQL.js";
+} from "./chunk-TWRPWJJT.js";
 import "./chunk-BT7NVCML.js";
-import "./chunk-E5PVMB3I.js";
-import "./chunk-UBN5AOLT.js";
+import "./chunk-SDXBR5QO.js";
+import "./chunk-W5EO4USJ.js";
 import "./chunk-5FVVY2EB.js";
 import "./chunk-JDSVLVCG.js";
 import "./chunk-5GKPFL3S.js";
@@ -24,7 +24,7 @@ import "./chunk-EBRG7ZIK.js";
 import "./chunk-P7FMDTKL.js";
 import "./chunk-X4OZMANG.js";
 import "./chunk-R2MBW2S4.js";
-import "./chunk-TYGLV6ZG.js";
+import "./chunk-EYGOM6AI.js";
 import "./chunk-A6XUJE5D.js";
 import "./chunk-C7HJ3NEI.js";
 import "./chunk-VS2IYZRU.js";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remnic/core",
-  "version": "10.0.38",
+  "version": "10.0.40",
   "description": "Framework-agnostic Remnic memory engine — orchestrator, storage, extraction, search, trust zones",
   "type": "module",
   "main": "dist/index.js",
@@ -3347,7 +3347,7 @@
     "core"
   ],
   "peerDependencies": {
-    "@remnic/coding-graph": "^10.0.38"
+    "@remnic/coding-graph": "^10.0.40"
   },
   "peerDependenciesMeta": {
     "@remnic/coding-graph": {

package/src/access-entity-principal.test.ts

@@ -0,0 +1,190 @@
+import assert from "node:assert/strict";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import test, { type TestContext } from "node:test";
+import { EngramAccessHttpServer } from "./access-http.js";
+import { getOperation } from "./access-boundary.js";
+import { EngramAccessService } from "./access-service.js";
+import { createChatExecutor } from "./chat/chat-executor.js";
+import { parseConfig } from "./config.js";
+import { NamespaceStorageRouter } from "./namespaces/storage.js";
+import type { Orchestrator } from "./orchestrator.js";
+
+interface EntityHttpBody {
+  error?: string;
+  found?: boolean;
+  namespace?: string;
+  entities?: Array<{ name?: string }>;
+  entity?: { name?: string };
+}
+
+interface McpToolBody {
+  result: { isError: boolean; content: Array<{ text: string }>; structuredContent: { entity: { name: string } } };
+}
+
+async function readMcp(response: Response): Promise<McpToolBody> {
+  // SAFETY: /mcp answers tools/call with the JSON-RPC result shape named above; each test asserts the fields it reads.
+  return (await response.json()) as McpToolBody;
+}
+
+async function readBody(response: Response): Promise<EntityHttpBody> {
+  // SAFETY: the access server answers these routes with the JSON shape named above; each test asserts the fields it reads.
+  return (await response.json()) as EntityHttpBody;
+}
+
+/** Entity reads need only config and real storage routing from the orchestrator. */
+async function createFixture(t: TestContext, namespacesEnabled = true) {
+  const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-entity-acl-"));
+  t.after(() => rm(memoryDir, { recursive: true, force: true }));
+  const config = parseConfig({
+    memoryDir,
+    namespacesEnabled,
+    defaultNamespace: "default",
+    sharedNamespace: "shared",
+    namespacePolicies: [
+      { name: "default", readPrincipals: ["owner"], writePrincipals: ["owner"] },
+      { name: "restricted", readPrincipals: ["owner", "reader"], writePrincipals: ["owner"] },
+    ],
+  });
+  const router = new NamespaceStorageRouter(config);
+  for (const namespace of namespacesEnabled ? ["default", "restricted"] : ["default"]) {
+    const storage = await router.storageFor(namespace);
+    const entitiesDir = path.join(storage.dir, "entities");
+    await mkdir(entitiesDir, { recursive: true });
+    await writeFile(path.join(entitiesDir, "project-widget.md"), [
+      `# Widget-${namespace}`,
+      "",
+      "**Type:** project",
+      "",
+      "## Summary",
+      "",
+      `Synthetic entity in ${namespace}.`,
+      "",
+    ].join("\n"));
+  }
+  const dependencies: Pick<Orchestrator, "config" | "getStorage"> = {
+    config,
+    getStorage: (namespace) => router.storageFor(namespace ?? config.defaultNamespace),
+  };
+  // SAFETY: the entity read paths touch only `config` and `getStorage` (the Pick above); the rest of the Orchestrator is unused here.
+  return new EngramAccessService(dependencies as Orchestrator);
+}
+
+test("entity service enforces read ACLs and requires a principal before list/get", async (t) => {
+  const service = await createFixture(t);
+  await assert.rejects(() => service.entityList({ namespace: "restricted" }, "intruder"), /not readable/);
+  await assert.rejects(() => service.entityGet("project-widget", "restricted", "intruder"), /not readable/);
+  await assert.rejects(() => service.entityList({ namespace: "restricted" }), /authentication required/);
+  await assert.rejects(() => service.entityGet("project-widget", "restricted"), /authentication required/);
+  await assert.rejects(() => service.entityList({}, "reader"), /not readable/);
+  await assert.rejects(() => service.entityGet("project-widget", undefined, "reader"), /not readable/);
+
+  const list = await service.entityList({ namespace: " restricted ", query: "widget", limit: 1 }, "reader");
+  assert.equal(list.namespace, "restricted");
+  assert.equal(list.total, 1);
+  assert.equal(list.entities[0]?.name, "Widget-restricted");
+  const entity = await service.entityGet("project-widget", " restricted ", "reader");
+  assert.equal(entity.namespace, "restricted");
+  assert.equal(entity.entity?.name, "Widget-restricted");
+  assert.deepEqual(await service.entityGet("missing", "restricted", "reader"), {
+    found: false, namespace: "restricted",
+  });
+});
+
+test("entity service retains principal-free reads when namespaces are disabled", async (t) => {
+  const service = await createFixture(t, false);
+  assert.equal((await service.entityList()).entities[0]?.name, "Widget-default");
+  assert.equal((await service.entityGet("project-widget")).entity?.name, "Widget-default");
+});
+
+test("entity_list registry dispatch preserves the trusted principal", async (t) => {
+  const service = await createFixture(t);
+  const operation = getOperation("entity_list");
+  assert.ok(operation);
+  await assert.rejects(
+    () => operation.run({ principal: "owner" }, { service, authenticatedPrincipal: "reader" }),
+    /not readable/,
+  );
+  const output = await operation.run({}, { service, authenticatedPrincipal: "owner" });
+  const result = JSON.parse(JSON.stringify(output)).result;
+  assert.equal(result.namespace, "default");
+  assert.equal(result.entities[0]?.name, "Widget-default");
+});
+
+test("HTTP and MCP entity reads bind ACLs to the transport principal across aliases", async (t) => {
+  const service = await createFixture(t);
+  const token = "synthetic-entity-token";
+  const server = new EngramAccessHttpServer({
+    service,
+    host: "127.0.0.1",
+    port: 0,
+    principal: "reader",
+    enableAdapters: false,
+    trustPrincipalHeader: false,
+    authTokenEntriesGetter: () => [{
+      token,
+      capabilities: { version: 1, ops: ["entity_get", "entity_list"], namespaces: ["default", "restricted"] },
+    }],
+  });
+  const started = await server.start();
+  t.after(() => server.stop());
+  const base = `http://${started.host}:${started.port}`;
+  const headers = {
+    authorization: `Bearer ${token}`,
+    "content-type": "application/json",
+    "x-engram-principal": "owner",
+  };
+  for (const prefix of ["engram", "remnic"]) {
+    // The entity HTTP routes are served under /engram/v1 only; the /remnic prefix is an alias for the MCP tool names below.
+    for (const suffix of prefix === "engram" ? ["entities", "entities/project-widget"] : []) {
+      const denied = await fetch(`${base}/${prefix}/v1/${suffix}?namespace=default&principal=owner`, { headers });
+      assert.equal(denied.status, 400);
+      assert.match((await readBody(denied)).error ?? "", /not readable/);
+      const allowed = await fetch(`${base}/${prefix}/v1/${suffix}?namespace=restricted`, { headers });
+      assert.equal(allowed.status, 200);
+      const body = await readBody(allowed);
+      assert.equal(body.namespace, "restricted");
+      assert.equal(suffix === "entities" ? body.entities?.[0]?.name : body.entity?.name, "Widget-restricted");
+    }
+    if (prefix === "engram") {
+      const missing = await fetch(`${base}/${prefix}/v1/entities/missing?namespace=restricted`, { headers });
+      assert.equal(missing.status, 404);
+      assert.equal((await readBody(missing)).found, false);
+    }
+
+    for (const namespace of ["default", "restricted"]) {
+      const response = await fetch(`${base}/mcp`, {
+        method: "POST",
+        headers,
+        body: JSON.stringify({
+          jsonrpc: "2.0",
+          id: `${prefix}-${namespace}`,
+          method: "tools/call",
+          params: {
+            name: `${prefix}.entity_get`,
+            arguments: {
+              name: "project-widget", namespace, principal: "owner", authenticatedPrincipal: "owner",
+            },
+          },
+        }),
+      });
+      assert.equal(response.status, 200);
+      const body = await readMcp(response);
+      assert.equal(body.result.isError, namespace === "default");
+      if (namespace === "default") {
+        assert.match(body.result.content[0].text, /not readable/);
+      } else {
+        assert.equal(body.result.structuredContent.entity.name, "Widget-restricted");
+      }
+    }
+  }
+});
+
+test("chat entity reads preserve the bound principal instead of losing identity", async (t) => {
+  const service = await createFixture(t);
+  const allowed = createChatExecutor({ service, namespace: "restricted", principal: "reader" });
+  assert.equal(JSON.parse(await allowed.entityGet("project-widget")).entity.name, "Widget-restricted");
+  const denied = createChatExecutor({ service, namespace: "default", principal: "reader" });
+  await assert.rejects(() => denied.entityGet("project-widget"), /not readable/);
+});

package/src/access-http.ts

@@ -2168,7 +2168,7 @@ export class EngramAccessHttpServer extends ReviewDeckAccessHttpBase {
         query: parsed.searchParams.get("q") ?? undefined,
         limit,
         offset,
-      });
+      }, this.resolveRequestPrincipal(req));
       this.respondJson(res, 200, response);
       return;
     }
@@ -2178,7 +2178,7 @@ export class EngramAccessHttpServer extends ReviewDeckAccessHttpBase {
       this.enforceTokenOp("entity_get"); // boundary dispatch (issue #1525)
       const entityName = decodeURIComponent(entityMatch[1] ?? "");
       const namespace = this.resolveNamespace(req, parsed.searchParams.get("namespace") ?? undefined);
-      const response = await this.service.entityGet(entityName, namespace);
+      const response = await this.service.entityGet(entityName, namespace, this.resolveRequestPrincipal(req));
       this.respondJson(res, response.found ? 200 : 404, response);
       return;
     }

package/src/access-operations-batch.ts

@@ -279,7 +279,7 @@ defineOperation({
     };
   },
 });
-defineOperation({ name: "entity_get", description: "Get entity.", schema: strictSchema({ name: S.str, namespace: S.str }), handler: async (input, ctx) => ({ result: await ctx.service.entityGet(optStr(input.name) ?? "", optStr(input.namespace)) }) });
+defineOperation({ name: "entity_get", description: "Get entity.", schema: strictSchema({ name: S.str, namespace: S.str }), handler: async (input, ctx) => ({ result: await ctx.service.entityGet(optStr(input.name) ?? "", optStr(input.namespace), ctx.authenticatedPrincipal) }) });
 defineOperation({ name: "review_queue_list", description: "List review queue.", schema: strictSchema({ runId: S.str, namespace: S.str }), handler: async (input, ctx) => ({ result: await ctx.service.reviewQueue(optStr(input.runId) ?? "", optStr(input.namespace), ctx.authenticatedPrincipal) }) });
 // Issue #2482: observe validates through the canonical observeRequestSchema
 // (same as HTTP/MCP wire validation). Nullable wire fields (parts /
@@ -472,7 +472,7 @@ defineOperation({
     return { result: getRecallTimingStatus(ctx.service.configRef) };
   },
 });
-defineOperation({ name: "entity_list", description: "List entities.", schema: strictSchema({}), handler: async (_i, ctx) => ({ result: await ctx.service.entityList() }) });
+defineOperation({ name: "entity_list", description: "List entities.", schema: strictSchema({}), handler: async (_i, ctx) => ({ result: await ctx.service.entityList({}, ctx.authenticatedPrincipal) }) });
 defineOperation({ name: "maintenance_status", description: "Maintenance status.", schema: strictSchema({}), handler: async (_i, ctx) => ({ result: await ctx.service.maintenance() }) });
 defineOperation({ name: "quality_status", description: "Quality status.", schema: strictSchema({}), handler: async (_i, ctx) => ({ result: await ctx.service.quality() }) });
 defineOperation({ name: "trust_zones_status", description: "Trust-zone status.", schema: strictSchema({}), handler: async (_i, ctx) => ({ result: await ctx.service.trustZoneStatus() }) });

package/src/access-service.ts

@@ -3506,10 +3506,11 @@ export class EngramAccessService extends SupportPassportAccessServiceBase {
       query?: string;
       limit?: number;
       offset?: number;
-    } = {}
+    } = {},
+    principal?: string
   ): Promise<EngramAccessEntityListResponse> {
-    const storage = await this.orchestrator.getStorage(options.namespace);
-    const resolvedNamespace = options.namespace?.trim() || this.orchestrator.config.defaultNamespace;
+    const resolvedNamespace = this.resolveReadableNamespace(options.namespace, principal);
+    const storage = await this.orchestrator.getStorage(resolvedNamespace);
     const { limit, offset } = normalizePagination(options.limit, options.offset);
     const query = options.query?.trim().toLowerCase() ?? "";
 
@@ -3553,9 +3554,9 @@ export class EngramAccessService extends SupportPassportAccessServiceBase {
     };
   }
 
-  async entityGet(name: string, namespace?: string): Promise<EngramAccessEntityResponse> {
-    const storage = await this.orchestrator.getStorage(namespace);
-    const resolvedNamespace = namespace?.trim() || this.orchestrator.config.defaultNamespace;
+  async entityGet(name: string, namespace?: string, principal?: string): Promise<EngramAccessEntityResponse> {
+    const resolvedNamespace = this.resolveReadableNamespace(namespace, principal);
+    const storage = await this.orchestrator.getStorage(resolvedNamespace);
     const raw = await storage.readEntity(name);
     if (!raw) return { found: false, namespace: resolvedNamespace };
     return {

package/src/buffer-drop-extraction-chunk.test.ts

@@ -0,0 +1,196 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { mkdtemp, rm } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { SmartBuffer } from "./buffer.js";
+import { parseConfig } from "./config.js";
+import { StorageManager } from "./storage.js";
+import type { BufferTurn } from "./types.js";
+import { ExtractionDrain } from "./orchestration/extraction-drain.js";
+import type { ExtractionRunResult } from "./orchestration/extraction-run.js";
+
+function makeTurn(sessionKey: string, content: string): BufferTurn {
+  return {
+    role: "user",
+    content,
+    timestamp: "2026-10-29T12:00:00.000Z",
+    sessionKey,
+  };
+}
+
+/** A real buffer on real storage; `restart()` is what a process restart would load from disk. */
+async function withBuffer(run: (buffer: SmartBuffer, restart: () => Promise<SmartBuffer>) => Promise<void>): Promise<void> {
+  const root = await mkdtemp(path.join(os.tmpdir(), "remnic-drop-chunk-"));
+  try {
+    const buffer = new SmartBuffer(parseConfig({}), new StorageManager(root));
+    await run(buffer, async () => {
+      const reloaded = new SmartBuffer(parseConfig({}), new StorageManager(root));
+      await reloaded.load();
+      return reloaded;
+    });
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+}
+
+test("dropExtractionChunk removes the matched prefix and the drop survives a restart", async () => {
+  await withBuffer(async (buffer, restart) => {
+    const turns = [makeTurn("s1", "turn one"), makeTurn("s1", "turn two"), makeTurn("s1", "turn three")];
+    for (const turn of turns) await buffer.addTurn("s1", turn);
+    await buffer.flushPendingSave();
+
+    const dropped = await buffer.dropExtractionChunk("s1", turns.slice(0, 2));
+
+    assert.equal(dropped, 2);
+    assert.deepEqual(buffer.getTurns("s1").map((turn) => turn.content), ["turn three"]);
+    assert.deepEqual((await restart()).getTurns("s1").map((turn) => turn.content), ["turn three"], "what a restart loads");
+    // A drop is not an extraction: the extraction bookkeeping stays untouched.
+    assert.equal(buffer.getExtractionCount("s1"), 0);
+  });
+});
+
+test("dropExtractionChunk returns zero without dropping when turns no longer match", async () => {
+  await withBuffer(async (buffer, restart) => {
+    await buffer.addTurn("s1", makeTurn("s1", "kept turn"));
+    await buffer.flushPendingSave();
+
+    const dropped = await buffer.dropExtractionChunk("s1", [makeTurn("s1", "already extracted elsewhere")]);
+
+    assert.equal(dropped, 0);
+    assert.equal(buffer.getTurns("s1").length, 1);
+    assert.equal((await restart()).getTurns("s1").length, 1, "the kept turn is still there after a restart");
+  });
+});
+
+test("dropExtractionChunk drops the whole matched window and a restart finds the buffer empty", async () => {
+  await withBuffer(async (buffer, restart) => {
+    const turns = [makeTurn("s1", "only turn")];
+    await buffer.addTurn("s1", turns[0]!);
+    await buffer.flushPendingSave();
+
+    const dropped = await buffer.dropExtractionChunk("s1", turns);
+
+    assert.equal(dropped, 1);
+    assert.equal(buffer.getTurns("s1").length, 0);
+    assert.equal((await restart()).getTurns("s1").length, 0);
+  });
+});
+
+const FAILED: ExtractionRunResult = { status: "skipped", reason: "empty_extraction_result", retainedBuffer: true, failureClass: "parse_empty", persistedCount: 0, durableOutputCount: 0 };
+const DONE: ExtractionRunResult = { status: "completed", persistedCount: 1, durableOutputCount: 1 };
+
+/** w7K's probe on #3266: a pinned capture sits before a native group drained with clearMatchingTurns. */
+async function drainBehindPinnedCapture(buffer: SmartBuffer, results: ExtractionRunResult[], dropAfter: number): Promise<ExtractionRunResult> {
+  const pinned: BufferTurn = { ...makeTurn("s1", "pinned capture"), acceptedCapture: true };
+  const native = [makeTurn("s1", "a".repeat(60)), makeTurn("s1", "b".repeat(60))];
+  for (const turn of [pinned, ...native]) await buffer.addTurn("s1", turn);
+  await buffer.flushPendingSave();
+  const drain = new ExtractionDrain({
+    config: { extractionMinUserTurns: 1, extractionMinChars: 0, extractionMaxTurnChars: 100_000, extractionChunkDropAfterFailures: dropAfter },
+    extractionWindowBudgetChars: () => 100,
+    runExtraction: async (turns, options) => {
+      const result = results.shift() ?? DONE;
+      // A successful run clears its chunk the way the coordinator does for a native group: by identity.
+      if (result.status === "completed") await buffer.clearAfterExtraction("s1", turns, options?.clearMatchingTurns === true ? { allowNonPrefix: true } : undefined);
+      return result;
+    },
+    getStorage: async () => ({ saveDroppedExtractionChunk: async () => "/recovery/copy.json" }),
+    buffer,
+    extractionQueueCoordinator: { beginPreemptable: () => {} },
+  });
+  return await drain.run(native, {
+    bufferKey: "s1",
+    clearBufferAfterExtraction: true,
+    clearMatchingTurns: true,
+    skipUserTurnThreshold: false,
+    failOnExtractionFailure: false,
+    forceExtractionAttempt: false,
+  });
+}
+
+test("a dropped chunk behind a pinned capture is removed by identity, like a successful clear (w7K)", async () => {
+  await withBuffer(async (buffer, restart) => {
+    const result = await drainBehindPinnedCapture(buffer, [FAILED, DONE], 1);
+
+    assert.equal(result.status, "completed");
+    assert.deepEqual(buffer.getTurns("s1").map((turn) => turn.content), ["pinned capture"], "the failed turn is gone, the pinned capture stays");
+    assert.deepEqual((await restart()).getTurns("s1").map((turn) => turn.content), ["pinned capture"], "and a restart agrees");
+  });
+});
+
+test("a failed chunk made only of deferred-retained turns can be dropped, and a restart agrees (Codex P2)", async () => {
+  await withBuffer(async (buffer, restart) => {
+    const retained = [makeTurn("s1", "retained one"), makeTurn("s1", "retained two")];
+    const live = makeTurn("s1", "live turn");
+    await buffer.retainDeferredTurns("s1", retained);
+    await buffer.addTurn("s1", live);
+    await buffer.flushPendingSave();
+
+    // The window a trigger snapshot sees is the retained turns followed by the live ones; the first chunk is retained-only.
+    const dropped = await buffer.dropExtractionChunk("s1", retained);
+
+    assert.equal(dropped, 2);
+    assert.deepEqual(buffer.getRetainedDeferredTurns("s1"), []);
+    assert.deepEqual(buffer.getTurns("s1").map((turn) => turn.content), ["live turn"]);
+    const reloaded = await restart();
+    assert.deepEqual(reloaded.getRetainedDeferredTurns("s1"), [], "retained turns stay dropped after a restart");
+    assert.deepEqual(reloaded.getTurns("s1").map((turn) => turn.content), ["live turn"]);
+  });
+});
+
+test("a chunk that spans retained and live turns drops both", async () => {
+  await withBuffer(async (buffer, restart) => {
+    const retained = [makeTurn("s1", "retained one")];
+    const live = [makeTurn("s1", "live one"), makeTurn("s1", "live two")];
+    await buffer.retainDeferredTurns("s1", retained);
+    for (const turn of live) await buffer.addTurn("s1", turn);
+    await buffer.flushPendingSave();
+
+    const dropped = await buffer.dropExtractionChunk("s1", [...retained, live[0]!]);
+
+    assert.equal(dropped, 2);
+    assert.deepEqual((await restart()).getTurns("s1").map((turn) => turn.content), ["live two"]);
+  });
+});
+
+test("deferred retention across chunks of one window stays within one retention window (Codex P2)", async () => {
+  await withBuffer(async (buffer, restart) => {
+    const window = Array.from({ length: 30 }, (_, i) => makeTurn("s1", `turn ${i}`));
+    for (const turn of window) await buffer.addTurn("s1", turn);
+    // Three chunks of ten, each deferring all of its turns; every run names the whole window as the processed snapshot.
+    for (let chunk = 0; chunk < 3; chunk++) {
+      await buffer.retainDeferredTurns("s1", window.slice(chunk * 10, chunk * 10 + 10), 10, window);
+    }
+    assert.equal(buffer.getRetainedDeferredTurns("s1").length, 10, "the latest chunk's retention replaces the earlier ones");
+    assert.equal((await restart()).getRetainedDeferredTurns("s1").length, 10);
+  });
+});
+
+test("a chunk holding only some retained turns drops those and keeps the rest of the retention slot (Phosphene)", async () => {
+  await withBuffer(async (buffer, restart) => {
+    const retained = [makeTurn("s1", "retained one"), makeTurn("s1", "retained two"), makeTurn("s1", "retained three")];
+    await buffer.retainDeferredTurns("s1", retained);
+    await buffer.addTurn("s1", makeTurn("s1", "live turn"));
+    await buffer.flushPendingSave();
+
+    const dropped = await buffer.dropExtractionChunk("s1", retained.slice(0, 2));
+
+    assert.equal(dropped, 2);
+    assert.deepEqual(buffer.getRetainedDeferredTurns("s1").map((turn) => turn.content), ["retained three"]);
+    const reloaded = await restart();
+    assert.deepEqual(reloaded.getRetainedDeferredTurns("s1").map((turn) => turn.content), ["retained three"]);
+    assert.deepEqual(reloaded.getTurns("s1").map((turn) => turn.content), ["retained three", "live turn"]);
+  });
+});
+
+test("retention for an early chunk leaves the retained turns of a later, unprocessed chunk alone (Codex P2)", async () => {
+  await withBuffer(async (buffer, restart) => {
+    const retained = [makeTurn("s1", "early retained"), makeTurn("s1", "late retained")];
+    await buffer.retainDeferredTurns("s1", retained);
+    // Chunk 0 holds the first retained turn; nothing was deferred in it, so it clears the slot for the turns it processed.
+    await buffer.retainDeferredTurns("s1", [], 0, retained.slice(0, 1));
+    assert.deepEqual(buffer.getRetainedDeferredTurns("s1").map((turn) => turn.content), ["late retained"]);
+    assert.deepEqual((await restart()).getRetainedDeferredTurns("s1").map((turn) => turn.content), ["late retained"]);
+  });
+});

package/src/buffer-types.ts

@@ -0,0 +1,65 @@
+/** Types split out of buffer.ts (structural ratchet, issue #1995). */
+
+import type { BufferTurn } from "./types.js";
+
+/**
+ * Optional surprise probe injected into `SmartBuffer`.
+ *
+ * Computes a D-MEM-style novelty score in `[0, 1]` for an incoming turn.
+ * The buffer treats the probe as purely additive: if it is not provided, if
+ * the feature flag is off, or if the probe throws/times out, the buffer
+ * falls back to the existing signal/turn-count/time triggers unchanged.
+ *
+ * Callers are responsible for sampling recent memories and passing them
+ * through the embedding pipeline — the buffer does not want to know about
+ * storage, embeddings, or QMD.
+ *
+ * @param bufferKey Identifier for the active buffer (session/thread).
+ * @param turn      The incoming turn whose novelty is being scored.
+ * @param recentTurns Turns already buffered for this key (most recent first
+ *                    is NOT guaranteed — treat as unordered corpus).
+ * @returns A surprise score in `[0, 1]`, or `null` if no score could be
+ *          produced (e.g. empty corpus, probe declined to embed).
+ */
+export interface BufferSurpriseProbe {
+  scoreTurn(
+    bufferKey: string,
+    turn: BufferTurn,
+    recentTurns: readonly BufferTurn[],
+  ): Promise<number | null>;
+}
+
+/**
+ * Minimal data carried on the serialized telemetry write chain
+ * (issue #563 PR 3).
+ *
+ * We intentionally do NOT capture the full `BufferTurn` here: under
+ * slow filesystem latency the chain can back up, and retaining
+ * `turn.content` for every pending append causes memory pressure on
+ * large conversations. Only the fields the ledger row actually needs
+ * cross the chain boundary.
+ */
+export interface SurpriseTelemetryQueueEntry {
+  bufferKey: string;
+  turnRole: "user" | "assistant";
+  sessionKey: string | null;
+  surpriseScore: number;
+  triggered: boolean;
+  turnCountInWindow: number;
+  /**
+   * ISO timestamp captured at the moment the turn was scored, NOT when
+   * the ledger append eventually runs. Backpressure on the serialized
+   * write chain could otherwise shift event timestamps away from the
+   * real decision moment and distort the distribution report (p90
+   * inflated, current-threshold row misidentified).
+   */
+  timestamp: string;
+  /**
+   * Threshold value in force when `triggered` was computed. Must be
+   * snapshot here rather than read from `config` at emit time — a
+   * concurrent config change between queue and write would otherwise
+   * record `triggered=true` against a newer threshold the operator
+   * never set, distorting precision/recall interpretation.
+   */
+  threshold: number;
+}