@remnic/core 9.7.11 → 9.7.13

package/src/namespaces/storage.ts

@@ -414,6 +414,10 @@ export class NamespaceStorageRouter {
    * default-namespace storage (`this.storage`) that bypasses the router.
    */
   bindCatalogWriteHook(sm: StorageManager, namespace: string): void {
+    // Issue #1903: pure state-file writes only touch the catalog when explicitly
+    // opted in. Router-created storages (storageFor) and the legacy default
+    // storage both flow through here, so this is the single wiring point.
+    sm.touchStateWrites = this.config.namespacesCatalogTouchStateWrites === true;
     sm.onCatalogWrite = () => this.touchCatalogWrite(namespace, sm.dir);
   }
 
@@ -505,6 +509,11 @@ export class NamespaceStorageRouter {
    * moved should await this instead of racing a timer.
    */
   async whenWriteTouchesSettled(): Promise<void> {
+    // Flush any coalesced (buffered) touches first so their appends join the
+    // pending set below — otherwise this could resolve while a buffered touch's
+    // lastWriteAt is still unwritten on disk (#1903, Cursor). No-op when
+    // coalescing is disabled (window 0) or the catalog is inert.
+    await this.catalog?.flushPendingTouches().catch(() => undefined);
     while (this.pendingWriteTouches.size > 0) {
       await Promise.allSettled([...this.pendingWriteTouches]);
     }

package/src/orchestrator.ts

@@ -948,6 +948,9 @@ export class Orchestrator {
     }
     this.maintenanceScheduler.dispose();
     await drainRecallWrites(this);
+    // Issue #1903: flush any coalesced namespace-catalog touches before teardown
+    // so a long-lived host does not drop buffered read/write timestamps.
+    await this.namespaceCatalog.flushPendingTouches().catch(() => undefined);
     await this.namespaceSearchRouter.dispose();
     await (this.qmd as { dispose?: () => void | Promise<void> }).dispose?.();
     if (this.conversationQmd && this.conversationQmd !== this.qmd) {

package/src/relay/mission-access.test.ts

@@ -60,13 +60,33 @@ test("HTTP fixture endpoint returns one complete, namespace-authorized Relay rec
       service: fakeService(root, calls),
       port: 0,
       authToken: "relay-token",
-      principal: "relay-operator",
+      principal: "operator-build-week",
       adminConsoleEnabled: false,
     });
     const status = await server.start();
     const base = `http://127.0.0.1:${status.port}/engram/v1/relay/missions/${RELAY_DEMO_MISSION_ID}`;
     try {
       const fixture = createRelayMissionFixture();
+      const approval = fixture.find((event) => event.payload.kind === "correction_approved");
+      assert.ok(approval && approval.payload.kind === "correction_approved");
+      const mismatchedApproval = await fetch(`${base}/events`, {
+        method: "POST",
+        headers: authHeaders(),
+        body: JSON.stringify({
+          namespace: RELAY_DEMO_NAMESPACE,
+          event: {
+            ...approval,
+            idempotencyKey: "mismatched-human-approval",
+            payload: {
+              ...approval.payload,
+              approvedBy: { ...approval.payload.approvedBy, id: "different-operator" },
+            },
+          },
+        }),
+      });
+      assert.equal(mismatchedApproval.status, 400);
+      assert.equal(((await mismatchedApproval.json()) as { code: string }).code, "input_error");
+
       const blankNamespace = await fetch(`${base}/events`, {
         method: "POST",
         headers: authHeaders(),
@@ -91,7 +111,7 @@ test("HTTP fixture endpoint returns one complete, namespace-authorized Relay rec
         assert.equal(response.status, 201, responseBody);
         assert.equal(
           (JSON.parse(responseBody) as { event: { authenticatedPrincipal?: string } }).event.authenticatedPrincipal,
-          "relay-operator"
+          "operator-build-week"
         );
       }
 
@@ -108,6 +128,7 @@ test("HTTP fixture endpoint returns one complete, namespace-authorized Relay rec
       });
       assert.equal(response.status, 200);
       assert.equal(response.headers.get("cache-control"), "no-store");
+      assert.equal(response.headers.get("x-remnic-authenticated-principal"), "operator-build-week");
       const snapshot = (await response.json()) as RelayMissionSnapshot;
       assert.equal(snapshot.receipt.complete, true);
       assert.deepEqual(snapshot.receipt.activeDecisionIds, ["decision-refresh-after-expiry"]);
@@ -115,7 +136,40 @@ test("HTTP fixture endpoint returns one complete, namespace-authorized Relay rec
       assert.equal(snapshot.agents.length, 3);
       assert.equal(snapshot.tests.at(-1)?.status, "passed");
       assert.ok(calls.every((call) => call.namespace === RELAY_DEMO_NAMESPACE));
-      assert.ok(calls.every((call) => call.principal === "relay-operator"));
+      assert.ok(calls.every((call) => call.principal === "operator-build-week"));
+    } finally {
+      await server.stop();
+    }
+  });
+});
+
+test("HTTP Relay rejects human approval without a server-resolved principal before persistence", async () => {
+  await withTempRoot(async (root) => {
+    const server = new EngramAccessHttpServer({
+      service: fakeService(root, []),
+      port: 0,
+      authToken: "relay-token",
+      adminConsoleEnabled: false,
+    });
+    const status = await server.start();
+    const base = `http://127.0.0.1:${status.port}/engram/v1/relay/missions/${RELAY_DEMO_MISSION_ID}`;
+    try {
+      const approval = createRelayMissionFixture().find((event) => event.payload.kind === "correction_approved");
+      assert.ok(approval);
+      const rejected = await fetch(`${base}/events`, {
+        method: "POST",
+        headers: authHeaders(),
+        body: JSON.stringify({ namespace: RELAY_DEMO_NAMESPACE, event: approval }),
+      });
+      assert.equal(rejected.status, 400);
+      assert.equal(((await rejected.json()) as { code: string }).code, "input_error");
+
+      const snapshotResponse = await fetch(`${base}?namespace=${RELAY_DEMO_NAMESPACE}`, {
+        headers: authHeaders(),
+      });
+      assert.equal(snapshotResponse.status, 200);
+      assert.equal(snapshotResponse.headers.get("x-remnic-authenticated-principal"), null);
+      assert.equal(((await snapshotResponse.json()) as RelayMissionSnapshot).found, false);
     } finally {
       await server.stop();
     }
@@ -150,6 +204,7 @@ test("HTTP Relay surface distinguishes empty, invalid, unauthorized, and backend
 
       const empty = await fetch(emptyUrl, { headers: authHeaders("read-token") });
       assert.equal(empty.status, 200);
+      assert.equal(empty.headers.get("x-remnic-authenticated-principal"), null);
       assert.deepEqual(await empty.json(), assertEmptyMissionShape());
 
       const badLimit = await fetch(`${emptyUrl}&limit=501`, {

package/src/relay/mission-access.ts

@@ -1,7 +1,7 @@
 import { z } from "zod";
 
 import { type OperationContext, defineOperation } from "../access-boundary.js";
-import type { EngramAccessService } from "../access-service.js";
+import { EngramAccessInputError, type EngramAccessService } from "../access-service.js";
 import {
   RELAY_MISSION_MAX_EVENT_LIMIT,
   RelayIsoDateTimeSchema,
@@ -76,12 +76,26 @@ export async function appendRelayMissionEvent(
   authenticatedPrincipal?: string,
   beforeAppend?: () => void | Promise<void>
 ): Promise<RelayMissionAppendResult> {
-  const resolved = await service.getWritableStorageForNamespace(input.namespace ?? undefined, authenticatedPrincipal);
+  const event = RelayMissionEventInputSchema.parse(input.event);
+  const approvalPrincipal = authenticatedPrincipal?.trim() || undefined;
+  if (event.payload.kind === "correction_approved" && event.payload.approvedBy.kind === "human") {
+    if (!approvalPrincipal) {
+      throw new EngramAccessInputError(
+        "Relay human approval requires a server-resolved authenticated principal"
+      );
+    }
+    if (event.payload.approvedBy.id !== approvalPrincipal) {
+      throw new EngramAccessInputError(
+        "Relay human approval identity must match the server-resolved authenticated principal"
+      );
+    }
+  }
+  const resolved = await service.getWritableStorageForNamespace(input.namespace ?? undefined, approvalPrincipal);
   const store = new RelayMissionStore({
     rootDir: resolved.storage.dir,
     namespace: resolved.namespace,
   });
-  return store.append(input.missionId, input.event, { authenticatedPrincipal, beforeAppend });
+  return store.append(input.missionId, event, { authenticatedPrincipal: approvalPrincipal, beforeAppend });
 }
 
 /** Namespace-authorized, bounded Relay snapshot read shared by HTTP and UI. */

package/src/relay/mission-ui.test.ts

@@ -0,0 +1,90 @@
+import assert from "node:assert/strict";
+import { mkdtemp, rm } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import test from "node:test";
+import { fileURLToPath } from "node:url";
+
+import { EngramAccessHttpServer } from "../access-http.js";
+import type { EngramAccessService } from "../access-service.js";
+import type { StorageManager } from "../storage.js";
+
+const adminConsolePublicDir = path.resolve(
+  path.dirname(fileURLToPath(import.meta.url)),
+  "../../../../admin-console/public"
+);
+
+async function withTempRoot(run: (root: string) => Promise<void>): Promise<void> {
+  const root = await mkdtemp(path.join(os.tmpdir(), "remnic-relay-ui-"));
+  try {
+    await run(root);
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+}
+
+function fakeService(root: string): EngramAccessService {
+  const storage = { dir: root } as StorageManager;
+  return {
+    configRef: { defaultNamespace: "relay-build-week", namespacesEnabled: true },
+    getReadableStorageForNamespace: async (namespace?: string) => ({ namespace: namespace ?? "relay-build-week", storage }),
+    getWritableStorageForNamespace: async (namespace?: string) => ({ namespace: namespace ?? "relay-build-week", storage }),
+  } as unknown as EngramAccessService;
+}
+
+test("Relay Mission Control serves both aliases from one exact static allow-list", async () => {
+  await withTempRoot(async (root) => {
+    const server = new EngramAccessHttpServer({
+      service: fakeService(root),
+      host: "127.0.0.1",
+      port: 0,
+      authToken: "relay-token",
+      principal: "relay-operator",
+      adminConsoleEnabled: true,
+      adminConsolePublicDir,
+      adminConsolePrefillToken: true,
+    });
+    const status = await server.start();
+    const origin = `http://127.0.0.1:${status.port}`;
+    try {
+      for (const prefix of ["remnic", "engram"]) {
+        const redirect = await fetch(`${origin}/${prefix}/ui/relay`, { redirect: "manual" });
+        assert.equal(redirect.status, 301);
+        assert.equal(redirect.headers.get("location"), `/${prefix}/ui/relay/`);
+
+        const shell = await fetch(`${origin}/${prefix}/ui/relay/`);
+        assert.equal(shell.status, 200);
+        assert.match(shell.headers.get("content-type") ?? "", /^text\/html/);
+        assert.doesNotMatch(await shell.text(), /__REMNIC_ADMIN_CONSOLE_PREFILL_TOKEN__/);
+
+        const authorizedShell = await fetch(`${origin}/${prefix}/ui/relay/`, {
+          headers: { authorization: "Bearer relay-token" },
+        });
+        assert.equal(authorizedShell.status, 200);
+        assert.equal(authorizedShell.headers.get("cache-control"), "private, no-store");
+        assert.match(await authorizedShell.text(), /__REMNIC_ADMIN_CONSOLE_PREFILL_TOKEN__="relay-token"/);
+
+        const expectedAssets = new Map([
+          ["relay.css", /^text\/css/],
+          ["relay-model.js", /^application\/javascript/],
+          ["relay.js", /^application\/javascript/],
+          ["replay.json", /^application\/json/],
+        ]);
+        for (const [fileName, contentType] of expectedAssets) {
+          const asset = await fetch(`${origin}/${prefix}/ui/relay/${fileName}`);
+          assert.equal(asset.status, 200, `${prefix}/${fileName}`);
+          assert.match(asset.headers.get("content-type") ?? "", contentType);
+          assert.ok((await asset.text()).length > 100);
+        }
+
+        const auth = { authorization: "Bearer relay-token" };
+        const unknown = await fetch(`${origin}/${prefix}/ui/relay/not-allowed.js`, { headers: auth });
+        assert.equal(unknown.status, 404);
+        const nested = await fetch(`${origin}/${prefix}/ui/relay/relay.js/extra`, { headers: auth });
+        assert.equal(nested.status, 404);
+      }
+    } finally {
+      await server.stop();
+    }
+  });
+});

package/src/storage-catalog-touch-gating.test.ts

@@ -0,0 +1,70 @@
+/**
+ * Issue #1903 — state-file writes must not touch the namespace catalog.
+ *
+ * The post-write catalog hook (`onCatalogWrite`) fires on every successful
+ * secure-file write. Pure state files (`state/buffer.json`, ledgers, indexes)
+ * are NOT namespace memory data, so by default their writes must be excluded
+ * from the catalog touch path (they were the dominant source of catalog churn
+ * that grew `state/namespaces.jsonl` unbounded). Namespace data writes
+ * (`facts/`, ...) still touch. The `touchStateWrites` opt-in restores the
+ * pre-#1903 behavior where every write touched.
+ */
+
+import assert from "node:assert/strict";
+import { mkdtemp, rm } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import test from "node:test";
+
+import { StorageManager } from "./storage.js";
+import type { BufferState } from "./types.js";
+
+async function mkDir(): Promise<string> {
+  return await mkdtemp(path.join(os.tmpdir(), "remnic-touch-gate-"));
+}
+
+function emptyBuffer(): BufferState {
+  return { turns: [], lastExtractionAt: null, extractionCount: 0 };
+}
+
+test("#1903 state-file writes do not touch the catalog by default; data writes do", async () => {
+  const dir = await mkDir();
+  try {
+    const storage = new StorageManager(dir);
+    await storage.ensureDirectories();
+    let touches = 0;
+    storage.onCatalogWrite = () => {
+      touches += 1;
+    };
+
+    // A pure state-file write (state/buffer.json) must NOT touch the catalog.
+    await storage.saveBuffer(emptyBuffer());
+    assert.equal(touches, 0, "saving buffer.json (a state file) records no catalog touch");
+
+    // A namespace-data write (facts/) MUST touch exactly once. The lifecycle
+    // ledger append it also performs is a state write and stays excluded.
+    touches = 0;
+    await storage.writeMemory("fact", "The database is PostgreSQL.", { confidence: 0.9 });
+    assert.equal(touches, 1, "writing a fact records exactly one catalog touch");
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+});
+
+test("#1903 touchStateWrites=true restores the state-file catalog touch", async () => {
+  const dir = await mkDir();
+  try {
+    const storage = new StorageManager(dir);
+    await storage.ensureDirectories();
+    storage.touchStateWrites = true;
+    let touches = 0;
+    storage.onCatalogWrite = () => {
+      touches += 1;
+    };
+
+    await storage.saveBuffer(emptyBuffer());
+    assert.ok(touches >= 1, "with touchStateWrites enabled, saving buffer.json touches the catalog");
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+});

package/src/storage.ts

@@ -2447,6 +2447,32 @@ export class StorageManager {
     }
   }
 
+  /**
+   * Whether pure state-file writes touch the namespace catalog (issue #1903).
+   * Set by `NamespaceStorageRouter.bindCatalogWriteHook` from
+   * `namespacesCatalogTouchStateWrites` (default false). State files
+   * (`state/buffer.json`, ledgers, indexes) are not namespace memory data, so by
+   * default their writes do NOT record a catalog touch.
+   */
+  touchStateWrites = false;
+
+  /**
+   * Post-write catalog hook, gated by file path (issue #1903). Namespace data
+   * (`facts/`, `cold/`, `entities/`, `artifacts/`, `profile.md`, ...) lives
+   * OUTSIDE `stateDir`, so a data write always touches; a write under `stateDir`
+   * is a pure state write and is skipped unless `touchStateWrites` is enabled.
+   */
+  private notifyCatalogWriteForPath(filePath: string): void {
+    if (!this.touchStateWrites && this.isStateFilePath(filePath)) return;
+    this.notifyCatalogWrite();
+  }
+
+  /** Whether `filePath` resolves to `stateDir` itself or a path contained in it. */
+  private isStateFilePath(filePath: string): boolean {
+    const rel = path.relative(this.stateDir, path.resolve(filePath));
+    return rel === "" || (!rel.startsWith("..") && !path.isAbsolute(rel));
+  }
+
   // ── Tombstone store (issue #1579) ───────────────────────────────────────
   // Instance-scoped (rule 11) + namespace-scoped (rule 42) tombstone index.
   // Lazily loaded from <stateDir>/tombstones.jsonl; invalidated together with
@@ -3083,12 +3109,12 @@ export class StorageManager {
   }
   private writeStorageSecureFile(filePath: string, content: string | Buffer): Promise<void> {
     return writeMaybeEncryptedFile(filePath, content, this.resolveWriteKey(), {}, this.baseDir).then(() =>
-      this.notifyCatalogWrite()
+      this.notifyCatalogWriteForPath(filePath)
     );
   }
   private writeStorageSecureFileChunks(filePath: string, chunks: AsyncIterable<Buffer>): Promise<void> {
     return writeMaybeEncryptedFileFromChunks(filePath, chunks, this.resolveWriteKey(), {}, this.baseDir).then(() =>
-      this.notifyCatalogWrite()
+      this.notifyCatalogWriteForPath(filePath)
     );
   }
 
@@ -3340,14 +3366,14 @@ export class StorageManager {
         if (isEncryptedFile(await readFile(filePath))) {
           const existing = await this.readStorageSecureFile(filePath);
           await writeMaybeEncryptedFile(filePath, `${existing}${content}`, null, {}, this.baseDir);
-          this.notifyCatalogWrite();
+          this.notifyCatalogWriteForPath(filePath);
           return;
         }
       } catch (err) {
         if (!isErrnoCode(err, "ENOENT")) throw err;
       }
       await appendFile(filePath, content, "utf-8");
-      this.notifyCatalogWrite();
+      this.notifyCatalogWriteForPath(filePath);
       return;
     }
 
@@ -3358,7 +3384,7 @@ export class StorageManager {
       if (!isErrnoCode(err, "ENOENT")) throw err;
     }
     await writeMaybeEncryptedFile(filePath, `${existing}${content}`, writeKey, {}, this.baseDir);
-    this.notifyCatalogWrite();
+    this.notifyCatalogWriteForPath(filePath);
   }
   private get stateDir(): string {
     return path.join(this.baseDir, "state");

package/src/types.ts

@@ -1690,6 +1690,40 @@ export interface PluginConfig {
    * existing single-namespace behavior is unchanged. Default on.
    */
   namespaceCatalogEnabled: boolean;
+  /**
+   * Size (bytes) at which a namespace-catalog touch triggers an in-lock
+   * auto-compaction of `state/namespaces.jsonl` (issue #1903). The append-only
+   * log grows one line per touch; once it exceeds this many bytes the next
+   * touch folds it (last-record-wins, every namespace row preserved) and
+   * atomically rewrites it. Default `16 * 1024 * 1024` (16 MB). `0` disables
+   * auto-compaction (restores the pre-#1903 unbounded-append behavior).
+   */
+  namespacesCatalogCompactBytes: number;
+  /**
+   * Coalescing window (ms) for pure-timestamp READ touches on an
+   * already-known namespace record (issue #1903). Repeated `markRead` touches
+   * that only refresh `lastReadAt` are buffered per (namespace, kind) and
+   * flushed once per window instead of appending a line each time.
+   * Semantically-observable touches (first sight, provenance/field changes)
+   * always flush immediately. Default `60000`. `0` disables read coalescing
+   * (each touch appends immediately, the pre-#1903 behavior).
+   */
+  namespacesCatalogReadTouchCoalesceMs: number;
+  /**
+   * Coalescing window (ms) for pure-timestamp WRITE touches on an
+   * already-known namespace record (issue #1903). Mirrors
+   * `namespacesCatalogReadTouchCoalesceMs` for `markWrite`/`lastWriteAt`.
+   * Default `1000`. `0` disables write coalescing.
+   */
+  namespacesCatalogWriteTouchCoalesceMs: number;
+  /**
+   * Whether pure state-file writes (`state/buffer.json`, ledgers, indexes)
+   * touch the namespace catalog (issue #1903). State files are not namespace
+   * memory data, so by default their writes do NOT record a catalog touch.
+   * Set `true` to restore the pre-#1903 behavior where every secure-file
+   * write touched the catalog. Default `false`.
+   */
+  namespacesCatalogTouchStateWrites: boolean;
   defaultNamespace: string;
   sharedNamespace: string;
   principalFromSessionKeyMode: PrincipalFromSessionKeyMode;