@remnic/core 9.7.11 → 9.7.13

package/src/config.test.ts

@@ -2161,3 +2161,47 @@ test("parseConfig rejects non-positive extraction retry/breaker numerics and fal
   assert.equal(c.extractionBreakerAuthCooldownMs, 1_800_000, "non-positive auth cooldown -> default");
   assert.deepEqual(c.extractionRetryScheduleMs, [60_000, 300_000, 1_800_000, 7_200_000], "schedule with non-positive entry -> default");
 });
+
+test("parseConfig namespace-catalog touch-path knobs: defaults, 0-accepting numerics, and validation (#1903)", () => {
+  // Documented defaults when absent.
+  const d = parseConfig({});
+  assert.equal(d.namespacesCatalogCompactBytes, 16 * 1024 * 1024, "compactBytes default 16 MB");
+  assert.equal(d.namespacesCatalogReadTouchCoalesceMs, 60_000, "read coalesce default 60000ms");
+  assert.equal(d.namespacesCatalogWriteTouchCoalesceMs, 1_000, "write coalesce default 1000ms");
+  assert.equal(d.namespacesCatalogTouchStateWrites, false, "state-write touch default false");
+
+  // 0 is an accepted disable switch for every numeric (resolveNonNegativeIntegerConfig).
+  assert.equal(parseConfig({ namespacesCatalogCompactBytes: 0 }).namespacesCatalogCompactBytes, 0);
+  assert.equal(parseConfig({ namespacesCatalogReadTouchCoalesceMs: 0 }).namespacesCatalogReadTouchCoalesceMs, 0);
+  assert.equal(parseConfig({ namespacesCatalogWriteTouchCoalesceMs: 0 }).namespacesCatalogWriteTouchCoalesceMs, 0);
+
+  // Positive integers pass through; CLI/env numeric strings coerce.
+  assert.equal(parseConfig({ namespacesCatalogCompactBytes: 4096 }).namespacesCatalogCompactBytes, 4096);
+  assert.equal(parseConfig({ namespacesCatalogReadTouchCoalesceMs: "30000" }).namespacesCatalogReadTouchCoalesceMs, 30_000);
+
+  // Boolean-like coercion for the state-write toggle (gotcha #36).
+  assert.equal(parseConfig({ namespacesCatalogTouchStateWrites: true }).namespacesCatalogTouchStateWrites, true);
+  assert.equal(parseConfig({ namespacesCatalogTouchStateWrites: "true" }).namespacesCatalogTouchStateWrites, true);
+  assert.equal(parseConfig({ namespacesCatalogTouchStateWrites: "off" }).namespacesCatalogTouchStateWrites, false);
+
+  // Negative and non-integer numerics are rejected with the documented message.
+  for (const key of [
+    "namespacesCatalogCompactBytes",
+    "namespacesCatalogReadTouchCoalesceMs",
+    "namespacesCatalogWriteTouchCoalesceMs",
+  ]) {
+    for (const bad of [-1, 1.5, "1.5", "abc", Number.NaN, Infinity]) {
+      assert.throws(
+        () => parseConfig({ [key]: bad }),
+        new RegExp(`${key} must be a non-negative integer`),
+        `${key}=${JSON.stringify(bad)} should throw`,
+      );
+    }
+  }
+
+  // A present-but-malformed boolean toggle fails fast (gotcha #51).
+  assert.throws(
+    () => parseConfig({ namespacesCatalogTouchStateWrites: "maybe" }),
+    /namespacesCatalogTouchStateWrites must be a boolean-like value/,
+  );
+});

package/src/config.ts

@@ -435,6 +435,26 @@ function resolvePositiveIntegerConfig(
   return coerced;
 }
 
+function resolveNonNegativeIntegerConfig(
+  value: unknown,
+  defaultValue: number,
+  keyName: string,
+): number {
+  if (value === undefined || value === null) return defaultValue;
+  const coerced = coerceNumber(value);
+  if (
+    coerced === undefined ||
+    !Number.isFinite(coerced) ||
+    !Number.isInteger(coerced) ||
+    coerced < 0
+  ) {
+    throw new Error(
+      `${keyName} must be a non-negative integer; got ${JSON.stringify(value)}`,
+    );
+  }
+  return coerced;
+}
+
 export function isOpenaiApiKeyDisabled(value: unknown): boolean {
   return value === false || (typeof value === "string" && value.trim().toLowerCase() === "false");
 }
@@ -2934,6 +2954,29 @@ export function parseConfig(
     // silently defaulting to enabled (CLAUDE.md rule #51); default to enabled
     // only when the value is absent.
     namespaceCatalogEnabled: resolveNamespaceCatalogEnabled(cfg.namespaceCatalogEnabled, rawOperatorConfig, runtimeSet),
+    // Namespace catalog touch-path performance knobs (issue #1903). Numerics use
+    // resolveNonNegativeIntegerConfig so 0 is an accepted disable switch; a
+    // present-but-invalid value (negative/non-integer) is REJECTED (rule #51).
+    namespacesCatalogCompactBytes: resolveNonNegativeIntegerConfig(
+      cfg.namespacesCatalogCompactBytes,
+      16 * 1024 * 1024,
+      "namespacesCatalogCompactBytes",
+    ),
+    namespacesCatalogReadTouchCoalesceMs: resolveNonNegativeIntegerConfig(
+      cfg.namespacesCatalogReadTouchCoalesceMs,
+      60_000,
+      "namespacesCatalogReadTouchCoalesceMs",
+    ),
+    namespacesCatalogWriteTouchCoalesceMs: resolveNonNegativeIntegerConfig(
+      cfg.namespacesCatalogWriteTouchCoalesceMs,
+      1_000,
+      "namespacesCatalogWriteTouchCoalesceMs",
+    ),
+    namespacesCatalogTouchStateWrites: resolveBooleanConfig(
+      cfg.namespacesCatalogTouchStateWrites,
+      false,
+      "namespacesCatalogTouchStateWrites",
+    ),
     // NOTE: namespace identifiers are intentionally NOT sanitized here — the
     // codebase rejects unsafe namespaces at the point of use (see
     // codex-materialize-runner and NamespaceStorageRouter / resolveNamespaceDir),

package/src/namespaces/catalog.test.ts

@@ -3694,3 +3694,261 @@ test("router resolve-hook serializer recovers after a prior hook rejects (issue
     await rm(memoryDir, { recursive: true, force: true });
   }
 });
+
+// ── Issue #1903: touch-path performance (auto-compaction, coalescing, streaming) ──
+
+async function countCatalogLines(memoryDir: string): Promise<number> {
+  const raw = await readFile(path.join(memoryDir, "state", "namespaces.jsonl"), "utf8").catch(() => "");
+  return raw.split("\n").filter((line) => line.trim().length > 0).length;
+}
+
+// 1. Warm cache: with coalescing OFF every touch appends, but a warm cache means
+// none of them re-parses the JSONL after the initial warm-up parse.
+test("#1903 warm cache: repeated touches never re-parse the log (coalescing off)", async () => {
+  const memoryDir = await mkMemoryDir();
+  try {
+    const catalog = new CountingNamespaceCatalog(
+      makeConfig(memoryDir, {
+        namespacesCatalogReadTouchCoalesceMs: 0,
+        namespacesCatalogWriteTouchCoalesceMs: 0,
+      }),
+    );
+    // Warm the cache: the first write creates the file; a read then performs the
+    // one-time full parse and warms the in-process cache.
+    await catalog.markWrite("alpha", { discoveredBy: "write" });
+    await catalog.listNamespaces();
+    const warm = catalog.catalogReadCount;
+    assert.ok(warm >= 1, "the warm-up performed at least one full parse");
+    for (let i = 0; i < 25; i++) {
+      await catalog.markWrite("alpha", { discoveredBy: "write" });
+      await catalog.markRead("alpha", { discoveredBy: "read" });
+    }
+    assert.equal(
+      catalog.catalogReadCount,
+      warm,
+      "every subsequent touch is served from the warm cache without a re-parse",
+    );
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});
+
+// 2. Write-touch coalescing collapses many in-window writes into a single append,
+// and the newest buffered timestamp wins on flush.
+test("#1903 write-touch coalescing collapses 50 in-window writes to one append", async () => {
+  const memoryDir = await mkMemoryDir();
+  try {
+    const catalog = new NamespaceCatalog(
+      makeConfig(memoryDir, { namespacesCatalogWriteTouchCoalesceMs: 60_000 }),
+    );
+    // Seed the record, then read it back so the in-process cache is warm — the
+    // following writes are then pure-timestamp refreshes on a known record and
+    // are deferred (a cold cache would force the first one to flush immediately).
+    await catalog.markWrite("alpha", { discoveredBy: "write" });
+    await catalog.getNamespaceRecord("alpha");
+    const linesAfterSeed = await countCatalogLines(memoryDir);
+    let newest = new Date();
+    for (let i = 0; i < 50; i++) {
+      newest = new Date(Date.now() + (i + 1) * 1000);
+      await catalog.markWrite("alpha", { discoveredBy: "write", at: newest });
+    }
+    assert.equal(
+      await countCatalogLines(memoryDir),
+      linesAfterSeed,
+      "coalesced writes stay buffered — no per-touch append",
+    );
+    await catalog.flushPendingTouches();
+    assert.equal(
+      await countCatalogLines(memoryDir),
+      linesAfterSeed + 1,
+      "the 50 buffered writes collapse into exactly one append on flush",
+    );
+    const record = await catalog.getNamespaceRecord("alpha");
+    assert.equal(
+      record?.lastWriteAt,
+      newest.toISOString(),
+      "the newest in-window write timestamp wins",
+    );
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});
+
+// 3. Read-touch coalescing collapses in-window reads and keeps the latest freshness.
+test("#1903 read-touch coalescing keeps the latest in-window lastReadAt", async () => {
+  const memoryDir = await mkMemoryDir();
+  try {
+    const catalog = new NamespaceCatalog(
+      makeConfig(memoryDir, { namespacesCatalogReadTouchCoalesceMs: 60_000 }),
+    );
+    // Seed the read field, then warm the cache so the following reads defer.
+    await catalog.markRead("alpha", { discoveredBy: "read" });
+    await catalog.getNamespaceRecord("alpha");
+    const linesAfterSeed = await countCatalogLines(memoryDir);
+    let newest = new Date();
+    for (let i = 0; i < 30; i++) {
+      newest = new Date(Date.now() + (i + 1) * 1000);
+      await catalog.markRead("alpha", { discoveredBy: "read", at: newest });
+    }
+    assert.equal(
+      await countCatalogLines(memoryDir),
+      linesAfterSeed,
+      "coalesced reads stay buffered — no per-touch append",
+    );
+    await catalog.flushPendingTouches();
+    assert.equal(await countCatalogLines(memoryDir), linesAfterSeed + 1);
+    const record = await catalog.getNamespaceRecord("alpha");
+    assert.equal(
+      record?.lastReadAt,
+      newest.toISOString(),
+      "the newest in-window read timestamp wins",
+    );
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});
+
+// 4. Provenance upgrade under coalescing: a config pre-registration followed by a
+// write must read back "write" WITHOUT an explicit flush (immediate-flush rule).
+test("#1903 markWrite upgrades config→write under coalescing without an explicit flush", async () => {
+  const memoryDir = await mkMemoryDir();
+  try {
+    const catalog = new NamespaceCatalog(
+      makeConfig(memoryDir, {
+        namespacesCatalogReadTouchCoalesceMs: 60_000,
+        namespacesCatalogWriteTouchCoalesceMs: 60_000,
+      }),
+    );
+    const ns = "project-origin-prov";
+    const storageDir = path.join(memoryDir, "namespaces", namespaceIdentityToken(ns));
+    await catalog.registerResolved(ns, storageDir);
+    assert.equal(
+      (await catalog.getNamespaceRecord(ns))?.discoveredBy,
+      "config",
+      "pre-registration is discovered by config",
+    );
+    // A real write must upgrade the provenance immediately even with coalescing on.
+    await catalog.markWrite(ns, { discoveredBy: "write", storageDir });
+    assert.equal(
+      (await catalog.getNamespaceRecord(ns))?.discoveredBy,
+      "write",
+      "the config→write upgrade flushes immediately (not coalesced away)",
+    );
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});
+
+// 5. Zero-limit semantics: CompactBytes=0 never rewrites and CoalesceMs=0 appends
+// one line per touch (line count == touch count) — the pre-#1903 behavior.
+test("#1903 zero knobs restore append-per-touch with no compaction", async () => {
+  const memoryDir = await mkMemoryDir();
+  try {
+    const catalog = new NamespaceCatalog(
+      makeConfig(memoryDir, {
+        namespacesCatalogCompactBytes: 0,
+        namespacesCatalogReadTouchCoalesceMs: 0,
+        namespacesCatalogWriteTouchCoalesceMs: 0,
+      }),
+    );
+    const touches = 12;
+    for (let i = 0; i < touches; i++) {
+      await catalog.markWrite("alpha", { discoveredBy: "write" });
+    }
+    assert.equal(
+      await countCatalogLines(memoryDir),
+      touches,
+      "no coalescing and no compaction → exactly one append per touch",
+    );
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});
+
+// 6. Auto-compaction bounds the log to one row per namespace and preserves every
+// namespace with its newest fields. A tiny limit forces compaction on each touch.
+test("#1903 auto-compaction bounds the log to one row per namespace and preserves all", async () => {
+  const memoryDir = await mkMemoryDir();
+  try {
+    const catalog = new NamespaceCatalog(
+      makeConfig(memoryDir, {
+        namespacesCatalogCompactBytes: 100,
+        namespacesCatalogReadTouchCoalesceMs: 0,
+        namespacesCatalogWriteTouchCoalesceMs: 0,
+      }),
+    );
+    const namespaces = [
+      "project-origin-a",
+      "project-origin-b",
+      "project-origin-c",
+      "project-origin-d",
+      "project-origin-e",
+    ];
+    const newest = new Map<string, string>();
+    for (let round = 0; round < 20; round++) {
+      for (const ns of namespaces) {
+        const at = new Date(Date.now() + (round * namespaces.length + namespaces.indexOf(ns)) * 1000);
+        newest.set(ns, at.toISOString());
+        await catalog.markWrite(ns, { discoveredBy: "write", at });
+      }
+    }
+    assert.equal(
+      await countCatalogLines(memoryDir),
+      namespaces.length,
+      "the log is bounded to exactly one row per namespace after compaction",
+    );
+    const list = await catalog.listNamespaces();
+    for (const ns of namespaces) {
+      const record = list.find((r) => r.namespace === ns);
+      assert.ok(record, `namespace ${ns} survives compaction`);
+      assert.equal(
+        record?.lastWriteAt,
+        newest.get(ns),
+        `namespace ${ns} keeps its newest write timestamp through compaction`,
+      );
+    }
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});
+
+// 7. Cross-process cache-coherence: compaction on one instance must never drop
+// another instance's rows (mirrors the rebuild-race tests). A third fresh
+// instance must read back every namespace touched by either.
+test("#1903 cross-instance compaction never drops another instance's rows", async () => {
+  const memoryDir = await mkMemoryDir();
+  try {
+    const cfg = () =>
+      makeConfig(memoryDir, {
+        namespacesCatalogCompactBytes: 100,
+        namespacesCatalogReadTouchCoalesceMs: 0,
+        namespacesCatalogWriteTouchCoalesceMs: 0,
+      });
+    // Separate instances behave like separate processes (own lock owner id).
+    const a = new NamespaceCatalog(cfg());
+    const b = new NamespaceCatalog(cfg());
+    const written = new Set<string>();
+    for (let round = 0; round < 8; round++) {
+      const nsA = `project-origin-a${round}`;
+      const nsB = `project-origin-b${round}`;
+      written.add(nsA);
+      written.add(nsB);
+      // Interleave: A appends a fresh namespace while B triggers a compaction of
+      // the shared log. The cross-process lock serializes the two.
+      await Promise.all([
+        a.markWrite(nsA, { discoveredBy: "write" }),
+        b.markWrite(nsB, { discoveredBy: "write" }),
+      ]);
+    }
+    const fresh = new NamespaceCatalog(makeConfig(memoryDir));
+    const list = await fresh.listNamespaces();
+    for (const ns of written) {
+      assert.ok(
+        list.some((r) => r.namespace === ns),
+        `namespace ${ns} must survive cross-instance interleaved compaction`,
+      );
+    }
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});

package/src/namespaces/catalog.ts

@@ -12,6 +12,7 @@ import {
   stat,
   writeFile,
 } from "node:fs/promises";
+import { StringDecoder } from "node:string_decoder";
 import type { PluginConfig } from "../types.js";
 import {
   MutationSerializer,
@@ -497,12 +498,46 @@ export class NamespaceCatalog {
   // against this normalized form everywhere instead.
   private readonly defaultNamespaceIdentity: string;
 
+  // Issue #1903 — touch-path performance knobs, resolved with the documented
+  // defaults when an externally-built PluginConfig omits them (so a partial
+  // config object still yields the production behavior). `0` is a real disable
+  // switch for each and is preserved by `??` (0 is not null/undefined).
+  private readonly compactBytesLimit: number;
+  private readonly readCoalesceMs: number;
+  private readonly writeCoalesceMs: number;
+  // Size after the most recent auto-compaction (0 = none yet). Hysteresis for
+  // maybeAutoCompact: a folded catalog that is itself above the limit (many
+  // distinct namespaces) would otherwise be re-folded + re-rewritten on every
+  // touch. We only compact again once the log has grown by >= one limit-worth
+  // since the last compaction (#1903, Codex).
+  private lastCompactedSize = 0;
+
+  // Issue #1903 — per-(namespace, kind) coalescing buffer. Keyed `${kind}:${ns}`.
+  // Holds the latest buffered timestamp/metadata for a pure-timestamp touch on
+  // an already-known record; a `.unref()`'d timer flushes it once per window so
+  // the append-only log does not grow one line per repeat touch. Semantically
+  // observable touches (first sight, provenance/field change) bypass this buffer
+  // and flush immediately (see `coalesceTouch`).
+  private readonly pendingTouches = new Map<
+    string,
+    {
+      namespace: string;
+      kind: "read" | "write";
+      metadata?: NamespaceTouchMetadata;
+      at: Date;
+      timer: NodeJS.Timeout;
+    }
+  >();
+
   constructor(private readonly config: PluginConfig) {
     this.memoryDir = config.memoryDir;
     this.stateDir = path.join(this.memoryDir, STATE_DIR);
     this.catalogPath = path.join(this.stateDir, CATALOG_FILE);
     this.rebuildLockPath = path.join(this.stateDir, REBUILD_LOCK_FILE);
     this.defaultNamespaceIdentity = normalizeNamespaceIdentity(config.defaultNamespace);
+    this.compactBytesLimit = config.namespacesCatalogCompactBytes ?? 16 * 1024 * 1024;
+    this.readCoalesceMs = config.namespacesCatalogReadTouchCoalesceMs ?? 60_000;
+    this.writeCoalesceMs = config.namespacesCatalogWriteTouchCoalesceMs ?? 1_000;
   }
 
   /** Whether the catalog is active (namespaces enabled and catalog not opted out). */
@@ -663,11 +698,137 @@ export class NamespaceCatalog {
   // ── Touch API (cheap, failure-tolerant) ─────────────────────────────────
 
   async markRead(namespace: string, metadata?: NamespaceTouchMetadata): Promise<void> {
-    await this.touch(namespace, "read", metadata);
+    await this.coalesceTouch(namespace, "read", metadata);
   }
 
   async markWrite(namespace: string, metadata?: NamespaceTouchMetadata): Promise<void> {
-    await this.touch(namespace, "write", metadata);
+    await this.coalesceTouch(namespace, "write", metadata);
+  }
+
+  /**
+   * Route a read/write touch through the per-(namespace, kind) coalescing buffer
+   * (issue #1903). A touch flushes IMMEDIATELY (bypassing the buffer) whenever it
+   * is semantically observable to an in-process reader — first sight of the
+   * namespace, a `config`→`write` provenance upgrade, the first time a touch
+   * field is set (so `listNamespaces({ writtenSince })` and record presence stay
+   * correct), or a change to `kind`/`principal`/`projectId`/`branch`/
+   * `parentNamespace`. A pure timestamp refresh on an already-known record is
+   * DEFERRED and coalesced: the newest buffered `at`/metadata wins and a single
+   * `touch` runs when the window elapses (or on `flushPendingTouches`). Window
+   * `0` skips the buffer entirely (the pre-#1903 immediate-append behavior).
+   * Best-effort: coalescing decisions read only the warm `compactedCache`, never
+   * forcing a disk parse; a cold cache flushes immediately (and re-warms it).
+   */
+  private async coalesceTouch(
+    namespace: string,
+    kind: "read" | "write",
+    metadata?: NamespaceTouchMetadata,
+  ): Promise<void> {
+    if (!this.enabled) return;
+    const window = kind === "read" ? this.readCoalesceMs : this.writeCoalesceMs;
+    if (window <= 0) {
+      await this.touch(namespace, kind, metadata);
+      return;
+    }
+    // Validate up front so an unsafe namespace rejects deterministically, exactly
+    // as the immediate-append path does (touch validates too).
+    const ns = this.validateNamespace(namespace);
+    const cached = this.compactedCache?.records.get(ns);
+    if (cached === undefined || this.touchMustFlushImmediately(kind, metadata, cached)) {
+      // Semantically observable in-process → flush now (also warms the cache).
+      // Any older buffered touch for this key is now redundant (this newer touch
+      // updates the same field with a newer timestamp): drop it so its stale
+      // timer cannot regress the timestamp after this flush.
+      const stale = this.pendingTouches.get(`${kind}:${ns}`);
+      if (stale) {
+        clearTimeout(stale.timer);
+        this.pendingTouches.delete(`${kind}:${ns}`);
+      }
+      await this.touch(namespace, kind, metadata);
+      return;
+    }
+    // Pure timestamp refresh on a known record → coalesce within a FIXED window.
+    const key = `${kind}:${ns}`;
+    const at = metadata?.at ?? new Date();
+    const existing = this.pendingTouches.get(key);
+    if (existing) {
+      // Keep the ORIGINAL timer and only refresh the buffered payload (#1903,
+      // Codex). Re-arming on every touch would make this a debounce that never
+      // flushes under continuous traffic (e.g. a sustained import), leaving
+      // lastWriteAt stale and writtenSince/maintenance consumers blind to an
+      // actively-written namespace. With a fixed window, each window flushes on
+      // schedule carrying the newest buffered timestamp.
+      existing.metadata = metadata;
+      existing.at = at;
+      return;
+    }
+    // Cap at Node's maximum setTimeout delay (2^31-1 ms ≈ 24.8 days). Values
+    // above that are clamped by Node to 1ms, which would append almost every
+    // touch and recreate the churn this coalescing exists to prevent (#1903, Codex).
+    const safeWindow = Math.min(window, 2_147_483_647);
+    const timer = setTimeout(() => {
+      void this.flushPendingTouch(key);
+    }, safeWindow);
+    timer.unref();
+    this.pendingTouches.set(key, { namespace: ns, kind, metadata, at, timer });
+  }
+
+  /**
+   * Whether a read/write touch on an EXISTING cached record must flush
+   * immediately rather than coalesce. Only pure timestamp refreshes (the value
+   * returns false) are safe to defer.
+   */
+  private touchMustFlushImmediately(
+    kind: "read" | "write",
+    metadata: NamespaceTouchMetadata | undefined,
+    cached: NamespaceRecord,
+  ): boolean {
+    // A write upgrading a config pre-registration to "write" changes the
+    // discoveredBy filter result — must be observable at once (~:1261).
+    if (kind === "write" && cached.discoveredBy === "config") return true;
+    // The first time a touch field is set makes the namespace newly match
+    // presence/`writtenSince` reads, so it must not be deferred.
+    if (kind === "write" && cached.lastWriteAt === undefined) return true;
+    if (kind === "read" && cached.lastReadAt === undefined) return true;
+    if (!metadata) return false;
+    if (metadata.kind !== undefined && metadata.kind !== cached.kind) return true;
+    if (metadata.principal !== undefined && metadata.principal !== cached.principal) return true;
+    if (metadata.projectId !== undefined && metadata.projectId !== cached.projectId) return true;
+    if (metadata.branch !== undefined && metadata.branch !== cached.branch) return true;
+    if (metadata.parentNamespace !== undefined && metadata.parentNamespace !== cached.parentNamespace) {
+      return true;
+    }
+    // A changed storageDir repoints routing/containment for this namespace —
+    // must be observable at once, not held stale by the coalesce window (#1903, Cursor).
+    if (metadata.storageDir !== undefined && metadata.storageDir !== cached.storageDir) return true;
+    return false;
+  }
+
+  /** Fire a single buffered touch when its coalescing window elapses. */
+  private async flushPendingTouch(key: string): Promise<void> {
+    const pending = this.pendingTouches.get(key);
+    if (!pending) return;
+    this.pendingTouches.delete(key);
+    clearTimeout(pending.timer);
+    await this.touch(pending.namespace, pending.kind, { ...pending.metadata, at: pending.at }).catch(
+      () => undefined,
+    );
+  }
+
+  /**
+   * Flush every buffered coalesced touch now (issue #1903). Called from the
+   * router/orchestrator shutdown hook so a long-lived host does not drop
+   * buffered read/write timestamps on teardown, and by tests to make coalesced
+   * touches deterministically observable. Best-effort: a failed flush never
+   * throws (each touch is `.catch()`-guarded).
+   */
+  async flushPendingTouches(): Promise<void> {
+    const pending = [...this.pendingTouches.values()];
+    this.pendingTouches.clear();
+    for (const p of pending) {
+      clearTimeout(p.timer);
+      await this.touch(p.namespace, p.kind, { ...p.metadata, at: p.at }).catch(() => undefined);
+    }
   }
 
   async markMaintenance(namespace: string, jobName: string, at?: Date): Promise<void> {
@@ -1269,11 +1430,48 @@ export class NamespaceCatalog {
         }
 
         await this.appendUnchained(record);
+        // Size-triggered auto-compaction (issue #1903), still inside the held
+        // cross-process lock + queueCritical turn so the fold→rename is atomic
+        // against concurrent touches and cross-process rebuilds.
+        await this.maybeAutoCompact();
         return true;
       }),
     );
   }
 
+  /**
+   * Fold-and-rewrite the JSONL log when it exceeds `compactBytesLimit`
+   * (issue #1903). MUST be called only from inside `touch`'s held-lock +
+   * `queueCritical` critical section — it reuses `loadCompacted` (last-record-
+   * wins fold; every namespace row survives) and `rewriteUnchained` (atomic
+   * temp-file + rename), so the collapse is cross-process safe and never
+   * invokes the disk-scan purge in `finishRebuild`. Best-effort and fail-open
+   * (rule #40): the append already succeeded and is durable, so a failed
+   * compaction must not fail the touch — the next over-limit touch retries.
+   */
+  private async maybeAutoCompact(): Promise<void> {
+    try {
+      const limit = this.compactBytesLimit; // 0 => auto-compaction disabled
+      if (limit <= 0) return;
+      // Exact post-append size when the cache is warm; skip this touch's check
+      // when the cache is cold (a cross-process append invalidated it) — the
+      // next touch re-warms and re-checks.
+      const size = this.compactedCache?.identity.size;
+      if (size === undefined || size <= limit) return;
+      // Hysteresis (issue #1903, Codex): a folded catalog whose deduped state is
+      // itself above the limit (many distinct namespaces) would otherwise be
+      // re-folded + re-rewritten on EVERY touch — O(catalog) per touch. Only
+      // compact again once the log has grown by >= one limit-worth since the last
+      // compaction; by then the new appends are duplicates the fold can collapse.
+      if (this.lastCompactedSize > 0 && size - this.lastCompactedSize < limit) return;
+      const folded = await this.loadCompacted();
+      await this.rewriteUnchained([...folded.values()]);
+      this.lastCompactedSize = this.compactedCache?.identity.size ?? size;
+    } catch {
+      // Fail-open: compaction is best-effort maintenance on already-durable data.
+    }
+  }
+
   // ── Rebuild from disk ────────────────────────────────────────────────────
 
   async rebuildFromDisk(
@@ -1957,25 +2155,52 @@ export class NamespaceCatalog {
 
     try {
       this.onCatalogReadForTest?.();
-      const raw = await handle.readFile("utf8");
-      for (const line of raw.split("\n")) {
+      // Streaming chunked parse (issue #1903): the log can reach 100+ MB in
+      // production, so read it in bounded chunks and yield to the event loop
+      // every ~5000 lines instead of buffering the whole file and blocking on a
+      // single giant `split("\n")`. The fold is unchanged (last-record-wins with
+      // field-level touch merge across cross-process full-snapshot appends).
+      const CHUNK_BYTES = 1 << 16; // 64 KiB
+      const YIELD_EVERY_LINES = 5000;
+      const chunk = Buffer.allocUnsafe(CHUNK_BYTES);
+      // StringDecoder buffers partial multi-byte UTF-8 sequences that straddle a
+      // chunk boundary, so a namespace/principal with non-ASCII bytes never
+      // corrupts at the seam.
+      const decoder = new StringDecoder("utf8");
+      let leftover = "";
+      let linesSinceYield = 0;
+      const foldLine = (line: string): void => {
         const trimmed = line.trim();
-        if (trimmed.length === 0) continue;
+        if (trimmed.length === 0) return;
         let parsed: unknown;
         try {
           parsed = JSON.parse(trimmed);
         } catch {
           // Skip corrupt lines (CLAUDE.md rule #18 robustness).
-          continue;
+          return;
         }
         const record = coerceRecord(parsed);
-        if (!record) continue;
-        // Preserve each touch field across cross-process full-snapshot appends.
-        // Field-level touch merge preserves the newest value for every touch
-        // field when full-snapshot appends from separate processes interleave.
+        if (!record) return;
         const prior = records.get(record.namespace);
         records.set(record.namespace, prior ? mergeNewerTouchFields(record, prior) : record);
+      };
+      for (;;) {
+        const { bytesRead } = await handle.read(chunk, 0, CHUNK_BYTES, null);
+        if (bytesRead === 0) break;
+        leftover += decoder.write(chunk.subarray(0, bytesRead));
+        let newlineIndex = leftover.indexOf("\n");
+        while (newlineIndex !== -1) {
+          foldLine(leftover.slice(0, newlineIndex));
+          leftover = leftover.slice(newlineIndex + 1);
+          if (++linesSinceYield >= YIELD_EVERY_LINES) {
+            linesSinceYield = 0;
+            await new Promise<void>((resolve) => setImmediate(resolve));
+          }
+          newlineIndex = leftover.indexOf("\n");
+        }
       }
+      leftover += decoder.end();
+      foldLine(leftover);
     } catch {
       this.compactedCache = undefined;
       return new Map();