@remnic/core 10.0.56 → 10.0.58

package/src/namespaces/catalog.ts

@@ -9,6 +9,7 @@ import {
   readdir,
   realpath,
   rename,
+  rm,
   stat,
   writeFile,
 } from "node:fs/promises";
@@ -23,6 +24,12 @@ import { namespaceIdentityFromToken, namespaceIdentityLegacyToken, namespaceIden
 import { resolveDefaultNamespaceRoot, resolveNamespaceStorageRoot } from "./storage.js";
 import { ALL_CATEGORY_DIRS } from "../utils/category-dir.js";
 import { isNotFoundError } from "../utils/errno.js";
+import {
+  dropDuplicateStorageRootAliases,
+  sanitizeRecordForRead,
+  type CatalogReadBoundaryPort,
+} from "./catalog-read-boundary.js";
+import { MalformedCatalogRepairGate, malformedSidecarPath } from "./catalog-repair.js";
 
 /**
  * Rebuildable namespace catalog (issue #1499).
@@ -90,6 +97,17 @@ export interface NamespaceCatalogFilter {
 /** Out-parameter a catalog read fills in when it could not see the whole catalog. */
 interface CatalogReadHealth {
   incomplete?: string;
+  /**
+   * Set ONLY when the file was read to the end and malformed rows were skipped. An unreadable
+   * catalog also sets `incomplete` but never `malformed`: its empty result is not the catalog's
+   * content, so nothing may be rewritten from it.
+   */
+  malformed?: boolean;
+  /**
+   * The raw malformed lines the read skipped, in file order. Only filled when `malformed` is
+   * set; the repair quarantines exactly these into its sidecar before rewriting.
+   */
+  malformedLines?: string[];
 }
 
 export interface NamespaceCatalogReadOptions {
@@ -346,7 +364,7 @@ function laterIso(a: string | undefined, b: string | undefined): string | undefi
  * rebuilt record and left untouched — we only recover touch recency that a
  * concurrent (possibly cross-process) writer recorded after our initial load.
  */
-function mergeNewerTouchFields(base: NamespaceRecord, fresh: NamespaceRecord): NamespaceRecord {
+export function mergeNewerTouchFields(base: NamespaceRecord, fresh: NamespaceRecord): NamespaceRecord {
   const merged: NamespaceRecord = { ...base };
   const lr = laterIso(base.lastReadAt, fresh.lastReadAt);
   if (lr) merged.lastReadAt = lr;
@@ -496,6 +514,14 @@ export class NamespaceCatalog {
   private readonly compactBytesLimit: number;
   private readonly readCoalesceMs: number;
   private readonly writeCoalesceMs: number;
+  // #3298 review: single-flight + backoff gate for the malformed-catalog repair.
+  private readonly malformedRepairGate = new MalformedCatalogRepairGate();
+
+  /** Test-only observability: how many repair attempts actually entered the gate. */
+  get malformedRepairAttemptsForTest(): number {
+    return this.malformedRepairGate.attemptCount;
+  }
+
   // Size after the most recent auto-compaction (0 = none yet). Hysteresis for
   // maybeAutoCompact: a folded catalog that is itself above the limit (many
   // distinct namespaces) would otherwise be re-folded + re-rewritten on every
@@ -538,122 +564,31 @@ export class NamespaceCatalog {
 
   // ── Public enumeration API ──────────────────────────────────────────────
 
-  /**
-   * Sanitize a record at the enumeration boundary (round 5, cursor Medium + codex
-   * P2; round 6 — NDXHe). Reads return whatever is in `namespaces.jsonl` after
-   * schema checks only, so a tampered or pre-fix row could surface unsafe data to
-   * maintenance/QMD until a rewrite occurs. Two distinct defenses:
-   *
-   *  1. UNSAFE NAMESPACE NAME (NGZqr, codex P2): an unsafe non-default namespace
-   *     (e.g. `../evil`, a name with separators, or >64 chars) is REJECTED outright
-   *     — return `null` so the caller drops it. The disk SCAN and the hot touch
-   *     path both reject such names with the SAME default-exempt `isSafeRouteNamespace`
-   *     gate, so the read boundary MUST agree, or `listNamespaces()`/`getNamespaceRecord()`
-   *     would expose a namespace those paths reject (note `isStorageDirForNamespace`
-   *     can still build a tokenized root even for `../evil`, so storageDir sanitation
-   *     alone does not catch it). The default namespace is exempt (it may be a
-   *     non-route literal), matching every other validation site.
-   *
-   *  2. UNSAFE storageDir: for an otherwise-valid namespace, apply the SAME contract
-   *     as the write path — full containment (`isContainedStorageDir`: lexical +
-   *     symlink/realpath) AND namespace ownership (`isStorageDirForNamespace`). When
-   *     a record fails EITHER check we substitute the trusted resolved-and-safe root
-   *     for that namespace (rule 42: read and write stay symmetric).
-   */
-  private async sanitizeRecordForRead(record: NamespaceRecord): Promise<NamespaceRecord | null> {
-    // Defense 1: drop an unsafe non-default namespace name entirely. Compare
-    // against the NORMALIZED default identity — record keys are trimmed, so a raw
-    // whitespace-padded config default would never match the default row (NH-FH).
-    if (record.namespace !== this.defaultNamespaceIdentity && !isSafeRouteNamespace(record.namespace)) {
-      return null;
-    }
-    // Defense 2: keep the record but substitute a safe storageDir when needed.
-    if (
-      (await this.isContainedStorageDir(record.storageDir)) &&
-      (await this.isStorageDirForNamespace(record.namespace, record.storageDir))
-    ) {
-      return record;
-    }
-    const safe = await this.resolveSafeStorageDir(record.namespace);
-    return { ...record, storageDir: safe };
-  }
-
-  private storageRootOwnershipRank(
-    record: NamespaceRecord,
-    resolvedStorageDir: string,
-    configured: Set<string>,
-  ): number {
-    if (resolvedStorageDir === path.resolve(this.memoryDir)) {
-      return record.namespace === this.defaultNamespaceIdentity ? 0 : 3;
-    }
-
-    const leaf = path.basename(resolvedStorageDir);
-    const tokenOwnsRoot = namespaceIdentityToken(record.namespace) === leaf;
-    if (tokenOwnsRoot && configured.has(record.namespace)) {
-      return 0;
-    }
-    if (record.namespace === leaf) return 1;
-    if (tokenOwnsRoot) return 2;
-    return 3;
-  }
-
-  private configuredNamespaceIdentities(): Set<string> {
-    return new Set(
-      [
-        this.config.defaultNamespace,
-        this.config.sharedNamespace,
-        ...this.config.namespacePolicies.map((p) => p.name),
-      ]
-        .map((n) => normalizeNamespaceIdentity(n))
-        .filter((n) => n.length > 0),
-    );
-  }
-
-  private preferStorageRootOwner(
-    current: NamespaceRecord,
-    candidate: NamespaceRecord,
-    resolvedStorageDir: string,
-    configured: Set<string>,
-  ): NamespaceRecord {
-    const currentRank = this.storageRootOwnershipRank(current, resolvedStorageDir, configured);
-    const candidateRank = this.storageRootOwnershipRank(candidate, resolvedStorageDir, configured);
-    if (candidateRank < currentRank) return candidate;
-    if (candidateRank > currentRank) return current;
-
-    const byName = candidate.namespace.localeCompare(current.namespace);
-    if (byName < 0) return candidate;
-    if (byName > 0) return current;
-    return candidate.identityToken.localeCompare(current.identityToken) < 0 ? candidate : current;
+  private catalogReadBoundaryPort(): CatalogReadBoundaryPort {
+    return {
+      defaultNamespaceIdentity: this.defaultNamespaceIdentity,
+      memoryDir: this.memoryDir,
+      isContainedStorageDir: (candidate) => this.isContainedStorageDir(candidate),
+      isStorageDirForNamespace: (namespace, candidate) => this.isStorageDirForNamespace(namespace, candidate),
+      resolveSafeStorageDir: (namespace) => this.resolveSafeStorageDir(namespace),
+    };
   }
 
-  private dropDuplicateStorageRootAliases(records: NamespaceRecord[]): NamespaceRecord[] {
-    const byStorageDir = new Map<string, NamespaceRecord>();
-    const configured = this.configuredNamespaceIdentities();
-    for (const record of records) {
-      const resolvedStorageDir = path.resolve(record.storageDir);
-      const current = byStorageDir.get(resolvedStorageDir);
-      if (!current) {
-        byStorageDir.set(resolvedStorageDir, record);
-        continue;
-      }
-      const owner = this.preferStorageRootOwner(current, record, resolvedStorageDir, configured);
-      const alias = owner === current ? record : current;
-      byStorageDir.set(resolvedStorageDir, mergeNewerTouchFields(owner, alias));
-    }
-    return [...byStorageDir.values()];
-  }
   private loadSanitizedRecords(health?: CatalogReadHealth): Promise<NamespaceRecord[]> {
     return this.queueCritical(async () => {
       const records = await this.loadCompacted(false, health);
+      const port = this.catalogReadBoundaryPort();
       const sanitized = await Promise.all(
-        [...records.values()].map((r) => this.sanitizeRecordForRead(r)),
+        [...records.values()].map((r) => sanitizeRecordForRead(port, r)),
       );
       // Drop unsafe-namespace rows (sanitizer returned null) at the read boundary.
       // Then collapse duplicate root aliases so maintenance/QMD see exactly one
       // namespace owner for a physical storage root, matching rebuild ownership,
       // while preserving touch recency from every alias row.
-      return this.dropDuplicateStorageRootAliases(
+      return dropDuplicateStorageRootAliases(
+        port,
         sanitized.filter((r): r is NamespaceRecord => r !== null),
+        this.config,
       );
     });
   }
@@ -665,7 +600,16 @@ export class NamespaceCatalog {
     if (!this.enabled) return [];
     const health: CatalogReadHealth = {};
     let out = await this.loadSanitizedRecords(health);
-    if (health.incomplete !== undefined) options?.onIncomplete?.(health.incomplete);
+    if (health.incomplete !== undefined) {
+      options?.onIncomplete?.(health.incomplete);
+    }
+    if (health.malformed === true) {
+      // A permanently malformed row keeps every completion request owed until the catalog is
+      // rewritten, so repair it now: fold the readable rows and rewrite the log without the bad
+      // ones. Best-effort and fail-open like the size-triggered compaction. Only a read that
+      // reached the end of the file may trigger it (see CatalogReadHealth.malformed).
+      void this.malformedRepairGate.run(() => this.rewriteAfterIncompleteRead()).catch(() => undefined);
+    }
     if (filter?.kind) out = out.filter((r) => r.kind === filter.kind);
     if (filter?.discoveredBy) out = out.filter((r) => r.discoveredBy === filter.discoveredBy);
     if (filter?.writtenSince) {
@@ -1424,6 +1368,47 @@ export class NamespaceCatalog {
     );
   }
 
+  /**
+   * Rewrite the catalog after a read skipped malformed rows (#3297). Same fold-and-rewrite as the
+   * size-triggered compaction — the readable rows survive, the malformed ones do not — so a stuck
+   * bad row stops keeping every completion request owed. Best-effort and fail-open (rule #40): a
+   * failed rewrite leaves the catalog unchanged and the next read reports and retries.
+   *
+   * Locking: the fold runs inside `queueCritical` (in-process serialization, like every other
+   * read), and the rewrite's load→rename window runs under the held catalog file lock, so it can
+   * never interleave with another process's touch append or rebuild. If another process already
+   * holds the lock, the rewrite is dropped (not queued): that process's own rewrite or rebuild is
+   * the likeliest repair, and the next incomplete read retries anyway.
+   *
+   * Data safety: the locked re-read must itself reach the end of the file and still find
+   * malformed rows, and must leave at least one readable row. A failed re-read folds to an empty
+   * map, and a catalog with nothing readable is left for `rebuildFromDisk`: rewriting either
+   * would replace a healthy or recoverable catalog with an empty file.
+   */
+  private async rewriteAfterIncompleteRead(): Promise<"repaired" | "dropped" | "failed"> {
+    return this.queueCritical(async () =>
+      this.withHeldCatalogLock(async (acquired) => {
+        if (!acquired) return "dropped";
+        const health: CatalogReadHealth = { malformedLines: [] };
+        const folded = await this.loadCompacted(true, health);
+        if (health.malformed !== true || folded.size === 0) return "failed";
+        // Quarantine the rows the rewrite drops before the rename, so nothing is lost
+        // silently. Best-effort like the repair itself: a failed sidecar write must not
+        // leave a stuck malformed catalog unrepaired.
+        const dropped = health.malformedLines ?? [];
+        if (dropped.length > 0) {
+          // Hash-named (#3298 review): repeated attempts over the same bad rows
+          // reuse one sidecar instead of piling one per failed attempt.
+          const sidecar = malformedSidecarPath(this.catalogPath, dropped);
+          await writeFile(sidecar, dropped.join("\n") + "\n", "utf8").catch(() => undefined);
+        }
+        await this.rewriteUnchained([...folded.values()]);
+        return "repaired";
+      }),
+    );
+  }
+
+
   /**
    * Fold-and-rewrite the JSONL log when it exceeds `compactBytesLimit`
    * (issue #1903). MUST be called only from inside `touch`'s held-lock +
@@ -2132,7 +2117,12 @@ export class NamespaceCatalog {
         this.compactedCache.identity.ino === identity.ino
       ) {
         await handle.close().catch(() => undefined);
-        if (health && this.compactedCache.incomplete) health.incomplete = this.compactedCache.incomplete;
+        if (health && this.compactedCache.incomplete) {
+          health.incomplete = this.compactedCache.incomplete;
+          // Invariant: cache.incomplete is set ONLY by the malformed branch below, so
+          // deriving `malformed` here cannot misfire for an unreadable catalog.
+          health.malformed = true;
+        }
         return this.cloneCompactedRecords(this.compactedCache.records);
       }
     } catch {
@@ -2167,11 +2157,13 @@ export class NamespaceCatalog {
           // Skip corrupt lines (CLAUDE.md rule #18 robustness), but count them so a caller that
           // needs the whole catalog can tell the read was incomplete (issue #3293).
           malformed += 1;
+          health?.malformedLines?.push(trimmed);
           return;
         }
         const record = coerceRecord(parsed);
         if (!record) {
           malformed += 1;
+          health?.malformedLines?.push(trimmed);
           return;
         }
         const prior = records.get(record.namespace);
@@ -2203,7 +2195,10 @@ export class NamespaceCatalog {
     }
 
     const incomplete = malformed > 0 ? `${malformed} malformed catalog line(s) skipped` : undefined;
-    if (health && incomplete) health.incomplete = incomplete;
+    if (health && incomplete) {
+      health.incomplete = incomplete;
+      health.malformed = true;
+    }
     this.compactedCache = identity ? { identity, records, incomplete } : undefined;
     return this.cloneCompactedRecords(records);
   }
@@ -2269,8 +2264,15 @@ export class NamespaceCatalog {
     const body = records.map((r) => serializeRecord(r)).join("\n") + (records.length > 0 ? "\n" : "");
     await mkdir(this.stateDir, { recursive: true });
     const tmp = `${this.catalogPath}.${process.pid}.${Date.now()}.tmp`;
-    await writeFile(tmp, body, "utf8");
-    await rename(tmp, this.catalogPath);
+    try {
+      await writeFile(tmp, body, "utf8");
+      await rename(tmp, this.catalogPath);
+    } catch (error) {
+      // A failed write or rename (for example the catalog path is a directory) must not leave
+      // the temp file behind; the original error still surfaces to the caller.
+      await rm(tmp, { force: true }).catch(() => undefined);
+      throw error;
+    }
     await this.refreshCacheIdentity(new Map(records.map((record) => [record.namespace, record])));
   }
 

package/src/testing/subjects/catalog-read-repair.test.ts

@@ -0,0 +1,124 @@
+/**
+ * Lifecycle matrix subject for the namespace catalog's repair after an incomplete read (#3297).
+ * A catalog with a malformed row is read through the public `listNamespaces` API: the read
+ * reports the incomplete catalog and schedules a rewrite that drops the bad row, so a fresh
+ * instance over the persisted file and a repeated read both see a whole catalog.
+ */
+import assert from "node:assert/strict";
+import { appendFile, readFile, rm } from "node:fs/promises";
+import path from "node:path";
+
+import { NamespaceCatalog } from "../../namespaces/catalog.js";
+import type { PluginConfig } from "../../types.js";
+import { type LifecycleSubject, type MatrixRow, runLifecycleMatrix } from "../lifecycle-matrix.js";
+import { mkTempMemoryDir } from "../orchestrator-lite.js";
+
+const BAD_ROW = "{this is not json";
+
+interface CatalogReadRepairState {
+  dir: string;
+  catalogPath: string;
+  catalog: NamespaceCatalog;
+  firstReasons: string[];
+  firstNamespaces: string[];
+  secondReasons?: string[];
+  afterRepair?: string;
+  afterSecondRead?: string;
+}
+
+/** Fixture config: the catalog reads only memoryDir, the namespace names and the enable flags. */
+function catalogConfig(memoryDir: string): PluginConfig {
+  return Object.assign(Object.create(null), {
+    memoryDir,
+    namespacesEnabled: true,
+    namespaceCatalogEnabled: true,
+    defaultNamespace: "default",
+    sharedNamespace: "shared",
+    namespacePolicies: [],
+  });
+}
+
+/** Wait until the scheduled rewrite has removed the bad row (the read does not await it). */
+async function waitForRepair(catalogPath: string): Promise<string> {
+  const deadline = Date.now() + 2_000;
+  for (;;) {
+    const text = await readFile(catalogPath, "utf8");
+    if (!text.includes(BAD_ROW) || Date.now() >= deadline) {
+      // Let the rewrite finish its rename and release the catalog lock before anyone reads again.
+      await new Promise((resolve) => setTimeout(resolve, 50));
+      return readFile(catalogPath, "utf8");
+    }
+    await new Promise((resolve) => setTimeout(resolve, 20));
+  }
+}
+
+const subject: LifecycleSubject<CatalogReadRepairState> = {
+  appliesTo(row: MatrixRow): boolean | string {
+    if (row.id === "restart-reload-recovery") return true;
+    if (row.id === "dedupe-replay") return true;
+    return "catalog repair applies to persisted-state reload and repeated reads only";
+  },
+
+  async setup(row: MatrixRow): Promise<CatalogReadRepairState> {
+    const dir = await mkTempMemoryDir(`catalog-read-repair-${row.id}`);
+    try {
+      const catalog = new NamespaceCatalog(catalogConfig(dir));
+      await catalog.markWrite("project-origin-abc123", { discoveredBy: "write" });
+      const catalogPath = path.join(dir, "state", "namespaces.jsonl");
+      await appendFile(catalogPath, `${BAD_ROW}\n`, "utf8");
+      return { dir, catalogPath, catalog, firstReasons: [], firstNamespaces: [] };
+    } catch (error) {
+      await rm(dir, { recursive: true, force: true });
+      throw error;
+    }
+  },
+
+  async exercise(state: CatalogReadRepairState, row: MatrixRow): Promise<void> {
+    const first = await state.catalog.listNamespaces(undefined, {
+      onIncomplete: (reason) => state.firstReasons.push(reason),
+    });
+    state.firstNamespaces = first.map((record) => record.namespace);
+    state.afterRepair = await waitForRepair(state.catalogPath);
+
+    if (row.dimensions.restart) {
+      // A fresh instance over the persisted (repaired) file.
+      const restarted = new NamespaceCatalog(catalogConfig(state.dir));
+      state.secondReasons = [];
+      const reasons = state.secondReasons;
+      await restarted.listNamespaces(undefined, { onIncomplete: (reason) => reasons.push(reason) });
+    }
+
+    if (row.dimensions.dedupeOrReplay) {
+      // The same instance reads again: nothing to report and no second rewrite.
+      state.secondReasons = [];
+      const reasons = state.secondReasons;
+      await state.catalog.listNamespaces(undefined, { onIncomplete: (reason) => reasons.push(reason) });
+      await new Promise((resolve) => setTimeout(resolve, 100));
+      state.afterSecondRead = await readFile(state.catalogPath, "utf8");
+    }
+  },
+
+  async invariants(state: CatalogReadRepairState, row: MatrixRow): Promise<void> {
+    assert.equal(state.firstReasons.length, 1, `${row.id}: the first read reports the malformed row`);
+    assert.deepEqual(
+      state.firstNamespaces,
+      ["project-origin-abc123"],
+      `${row.id}: the readable record is still returned`,
+    );
+    assert.ok(
+      !(state.afterRepair ?? "").includes(BAD_ROW),
+      `${row.id}: the scheduled rewrite dropped the malformed row`,
+    );
+    assert.match(state.afterRepair ?? "", /project-origin-abc123/, `${row.id}: the readable row survived the rewrite`);
+    assert.deepEqual(state.secondReasons, [], `${row.id}: after the repair the catalog reads whole`);
+    if (row.dimensions.dedupeOrReplay) {
+      assert.equal(state.afterSecondRead, state.afterRepair, `${row.id}: a clean read does not rewrite again`);
+    }
+  },
+
+  async teardown(state: CatalogReadRepairState): Promise<void> {
+    await rm(state.dir, { recursive: true, force: true });
+  },
+};
+
+runLifecycleMatrix("catalog-read-repair", subject);