@remnic/core 9.23.2 → 9.23.4

package/src/operator-doctor-corpus.test.ts

@@ -0,0 +1,171 @@
+import assert from "node:assert/strict";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+/**
+ * Corpus-watermark doctor check (issue #2149).
+ *
+ * Verifies `summarizeCorpusWatermark` reports per-namespace counts + a short
+ * digest prefix and always resolves `ok` (this PR has no peer to compare
+ * against, so it never warns), and that `runOperatorDoctor` wires it in.
+ */
+import test from "node:test";
+import { parseConfig } from "./config.js";
+import type { CorpusWatermark } from "./corpus-watermark.js";
+import { summarizeCorpusWatermark } from "./operator-doctor-corpus.js";
+import { type OperatorToolkitOrchestrator, runOperatorDoctor } from "./operator-toolkit.js";
+import type { OperatorDoctorCheck } from "./operator-toolkit.js";
+import { StorageManager } from "./storage.js";
+import type { PluginConfig } from "./types.js";
+
+const storageFactory = (dir: string) => new StorageManager(dir);
+
+// The doctor stores the watermark array under `details.corpus`; `details` is
+// typed `unknown`, so narrow it once here rather than at each read.
+function corpusOf(check: OperatorDoctorCheck | undefined): CorpusWatermark[] {
+  const details = check?.details;
+  if (!details || typeof details !== "object" || !("corpus" in details)) return [];
+  // `details.corpus` is `unknown` after the `in` guard; the doctor always
+  // stores the watermark array here.
+  const corpus = details.corpus as CorpusWatermark[];
+  return corpus;
+}
+
+async function writeMemory(memoryDir: string, rel: string): Promise<void> {
+  const full = path.join(memoryDir, rel);
+  await mkdir(path.dirname(full), { recursive: true });
+  await writeFile(full, `---\nid: ${path.basename(rel, ".md")}\n---\n\nbody\n`, "utf-8");
+}
+
+async function makeFixture(): Promise<{
+  root: string;
+  memoryDir: string;
+  config: PluginConfig;
+  configPath: string;
+  orchestrator: OperatorToolkitOrchestrator;
+}> {
+  const root = await mkdtemp(path.join(os.tmpdir(), "engram-corpus-doctor-"));
+  const memoryDir = path.join(root, "memory");
+  const workspaceDir = path.join(root, "workspace");
+  await mkdir(memoryDir, { recursive: true });
+  await mkdir(workspaceDir, { recursive: true });
+  const rawConfig = {
+    openaiApiKey: "sk-test",
+    memoryDir,
+    workspaceDir,
+    qmdEnabled: false,
+    transcriptEnabled: false,
+    hourlySummariesEnabled: false,
+    identityEnabled: false,
+    identityContinuityEnabled: false,
+    sharedContextEnabled: false,
+    captureMode: "implicit",
+    namespacesEnabled: false,
+    defaultNamespace: "global",
+  };
+  const config = parseConfig(rawConfig);
+  const configPath = path.join(root, "openclaw.json");
+  await writeFile(
+    configPath,
+    JSON.stringify({ plugins: { entries: { "openclaw-remnic": { config: rawConfig } } } }, null, 2),
+    "utf-8"
+  );
+  const orchestrator: OperatorToolkitOrchestrator = {
+    config,
+    storage: new StorageManager(memoryDir),
+    qmd: {
+      async probe() {
+        return false;
+      },
+      isAvailable() {
+        return false;
+      },
+      async ensureCollection() {
+        return "skipped";
+      },
+      debugStatus() {
+        return "disabled";
+      },
+    },
+    conversationIndexCoordinator: {
+      async getHealth() {
+        return {
+          enabled: false,
+          backend: "qmd" as const,
+          status: "disabled" as const,
+          chunkDocCount: 0,
+          lastUpdateAt: null,
+        };
+      },
+      async rebuild() {
+        return { chunks: 0, skipped: true, reason: "disabled", embedded: false, rebuilt: false };
+      },
+    },
+  };
+  return { root, memoryDir, config, configPath, orchestrator };
+}
+
+test("summarizeCorpusWatermark: reports an ok check with per-namespace count and a short digest prefix", async () => {
+  const { root, memoryDir, config } = await makeFixture();
+  try {
+    await writeMemory(memoryDir, "facts/2026-03-08/a.md");
+    await writeMemory(memoryDir, "facts/2026-03-08/b.md");
+    await writeMemory(memoryDir, "procedures/2026-03-09/c.md");
+
+    const check = await summarizeCorpusWatermark(config, storageFactory);
+    assert.equal(check.key, "corpus_watermark");
+    assert.equal(check.status, "ok");
+    const corpus = corpusOf(check);
+    assert.equal(corpus.length, 1);
+    assert.equal(corpus[0]?.namespace, "global");
+    assert.equal(corpus[0]?.memoryFileCount, 3);
+    assert.match(check.summary, /global: 3 files/);
+    assert.match(check.summary, /digest=[0-9a-f]{12}/);
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+});
+
+test("summarizeCorpusWatermark: an empty corpus still resolves ok with a zero count", async () => {
+  const { root, config } = await makeFixture();
+  try {
+    const check = await summarizeCorpusWatermark(config, storageFactory);
+    assert.equal(check.status, "ok");
+    const corpus = corpusOf(check);
+    assert.equal(corpus.length, 1);
+    assert.equal(corpus[0]?.memoryFileCount, 0);
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+});
+
+test("runOperatorDoctor: includes the corpus_watermark check", async () => {
+  const { root, memoryDir, configPath, orchestrator } = await makeFixture();
+  try {
+    await writeMemory(memoryDir, "facts/2026-03-08/a.md");
+    const report = await runOperatorDoctor({ configPath, orchestrator });
+    const check = report.checks.find((c) => c.key === "corpus_watermark");
+    assert.ok(check, "expected a corpus_watermark check");
+    assert.equal(check?.status, "ok");
+    assert.equal(corpusOf(check)[0]?.memoryFileCount, 1);
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+});
+
+test("summarizeCorpusWatermark: warns (not ok) when a namespace scan fails", async () => {
+  const { root, config } = await makeFixture();
+  try {
+    // A storage factory that fails every scan (e.g. an EACCES permission
+    // regression). The namespace is omitted, so the check must WARN — not
+    // certify ok, and not conflate the failure with an empty deployment.
+    const throwingFactory = (_dir: string): never => {
+      throw new Error("simulated backend read failure");
+    };
+    const check = await summarizeCorpusWatermark(config, throwingFactory);
+    assert.equal(check.status, "warn", "a scan failure must not be reported as ok");
+    assert.match(check.summary, /unavailable|unreadable|churning/i);
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+});

package/src/operator-doctor-corpus.ts

@@ -0,0 +1,95 @@
+/**
+ * Corpus-watermark doctor check (issue #2149).
+ *
+ * Lives beside operator-toolkit.ts (which is at its structural-ratchet ceiling)
+ * so the god file only gains the wiring. Reports each namespace's memory-file
+ * count, newest write, and a short census-digest prefix, and WARNS when a
+ * namespace could not be scanned (unreadable/churning) or enumeration failed —
+ * a scan failure omits a namespace, so an unconditional "ok" would certify a
+ * partial fleet or conflate an unreadable corpus with an empty deployment
+ * (issue #2156 round-7). Peer comparison (count/age divergence thresholds) is a
+ * follow-up.
+ *
+ * Namespaces are resolved through the shared resolveCorpusNamespaceRoots helper
+ * (issue #2156 finding C) so the doctor and the /health surface enumerate the
+ * SAME tenant set and cannot drift. The caller supplies `storageFactory` (a
+ * `dir -> CorpusStorage`) so this module never imports the storage god-module
+ * directly (issue #1533 ratchet).
+ */
+
+import {
+  type CorpusStorage,
+  type CorpusWatermark,
+  computeCorpusWatermarks,
+  resolveCorpusNamespaceRoots,
+} from "./corpus-watermark.js";
+import type { OperatorDoctorCheck } from "./operator-toolkit.js";
+import type { PluginConfig } from "./types.js";
+
+const DIGEST_PREFIX_LENGTH = 12;
+
+export async function summarizeCorpusWatermark(
+  config: PluginConfig,
+  storageFactory: (dir: string) => CorpusStorage,
+  now?: Date
+): Promise<OperatorDoctorCheck> {
+  let resolved: string[] = [];
+  let watermarks: CorpusWatermark[] = [];
+  let enumerationFailed = false;
+  try {
+    // The doctor orchestrator has no live namespace catalog, so config-driven
+    // enumeration (deduped by root inside the shared helper) is the whole set.
+    const rootByNamespace = new Map<string, string>();
+    for (const root of await resolveCorpusNamespaceRoots({ config })) {
+      rootByNamespace.set(root.namespace, root.rootDir);
+    }
+    resolved = [...rootByNamespace.keys()];
+    watermarks = await computeCorpusWatermarks(
+      resolved,
+      (namespace) => storageFactory(rootByNamespace.get(namespace) ?? config.memoryDir),
+      now
+    );
+  } catch {
+    // Namespace ENUMERATION itself failed (distinct from a per-namespace scan).
+    enumerationFailed = true;
+  }
+
+  const computed = new Set(watermarks.map((w) => w.namespace));
+  const failed = resolved.filter((namespace) => !computed.has(namespace));
+  const lines = watermarks.map(
+    (w) =>
+      `${w.namespace}: ${w.memoryFileCount} files, newest=${w.newestWriteAt ?? "never"}, ` +
+      `digest=${w.digest.slice(0, DIGEST_PREFIX_LENGTH)}`
+  );
+
+  if (enumerationFailed) {
+    return {
+      key: "corpus_watermark",
+      status: "warn",
+      summary: "Could not resolve corpus namespaces (enumeration failed).",
+      remediation: "Ensure the memory directory and namespace catalog are readable, then rerun `remnic doctor`.",
+      details: { corpus: watermarks, resolved: resolved.length, failed: failed.length },
+    };
+  }
+  if (failed.length > 0) {
+    return {
+      key: "corpus_watermark",
+      status: "warn",
+      summary:
+        `Corpus watermark unavailable for ${failed.length} of ${resolved.length} namespace(s) ` +
+        `(unreadable or churning): ${failed.join(", ")}.` +
+        (lines.length > 0 ? ` Reporting ${lines.length}: ${lines.join("; ")}` : ""),
+      remediation: "Inspect the omitted namespace roots for permission/backend errors, then rerun `remnic doctor`.",
+      details: { corpus: watermarks, resolved: resolved.length, failed },
+    };
+  }
+  return {
+    key: "corpus_watermark",
+    status: "ok",
+    summary:
+      resolved.length === 0
+        ? "No corpus namespaces resolved yet."
+        : `Corpus watermark across ${watermarks.length} namespace(s): ${lines.join("; ")}`,
+    details: { corpus: watermarks },
+  };
+}

package/src/operator-toolkit.ts

@@ -28,7 +28,7 @@ import { analyzeGraphHealth, type GraphHealthReport } from "./graph.js";
 import { resolveAccessSetupCapabilities,
   resolveCapabilities,
   resolveGraphConstructionCapabilities,
-  resolveQmdCapabilities,
+  resolveQmdCapabilities, resolveIndexingCapabilities,
   resolveEvalCapabilities, resolvePresentationCapabilities, resolveConsolidationCapabilities } from "./capabilities.js";
 import {
   analyzeSessionIntegrity,
@@ -55,8 +55,7 @@ import type {
 } from "./types.js";
 import { reportBufferSurpriseDistribution } from "./buffer-surprise-report.js";
 import { readJudgeVerdictStats } from "./extraction-judge-telemetry.js";
-import { resolveIndexingCapabilities } from "./capabilities.js";
-
+import { summarizeCorpusWatermark } from "./operator-doctor-corpus.js";
 
 const OPENCLAW_REMNIC_PLUGIN_IDS = ["openclaw-remnic", "openclaw-engram"] as const;
 
@@ -1501,6 +1500,7 @@ export async function runOperatorDoctor(options: OperatorDoctorOptions): Promise
   // Surfaces per-phase: enabled status, cadence, threshold values, and the
   // best-available last-run timestamp for each of the three pipeline phases.
   checks.push(await summarizeDreamsPhases(config, storage));
+  checks.push(await summarizeCorpusWatermark(config, (dir) => new StorageManager(dir))); // corpus watermark #2149
 
   // Security mitigation status (issue #565).
   // Reports whether the cross-namespace budget and anomaly detection

package/src/recall/archive-scoring.ts

@@ -250,6 +250,17 @@ class ArchiveScoringWorkerPool {
   private idle: Worker[] = [];
   private waiters: Array<{ resolve: (worker: Worker) => void; reject: (err: Error) => void }> = [];
   private terminated = false;
+  private busy = 0;
+
+  /**
+   * Workers currently checked out and scoring — NOT callers waiting in line.
+   * A caller parked in `acquire()` is queued, not running, so this is the only
+   * honest measure of real task overlap: with a size-1 pool it never exceeds 1
+   * no matter how many callers are queued behind it.
+   */
+  get busyWorkers(): number {
+    return this.busy;
+  }
 
   constructor(size: number = defaultPoolSize()) {
     this.targetSize = Math.max(1, size);
@@ -265,11 +276,13 @@ class ArchiveScoringWorkerPool {
       return [];
     }
     let abandoned = false;
+    this.busy += 1;
     try {
       return await this.dispatch(worker, task, abortSignal, () => {
         abandoned = true;
       });
     } finally {
+      this.busy -= 1;
       if (abandoned) this.retireWorker(worker);
       else this.release(worker);
     }
@@ -424,6 +437,11 @@ export class OffThreadArchiveScoring implements ArchiveScoringStrategy {
     }
   }
 
+  /** Workers currently scoring; 0 when no pool has been created yet. */
+  get busyWorkers(): number {
+    return this.pool?.busyWorkers ?? 0;
+  }
+
   async score(
     items: ReadonlyArray<ArchiveScoreItem>,
     tokens: ReadonlyArray<string>,

package/src/storage/memory-read-store.ts

@@ -9,7 +9,7 @@
  * listing.
  */
 
-import type { Dirent } from "node:fs";
+import type { Dirent, Stats } from "node:fs";
 import { lstat, readdir, realpath, stat } from "node:fs/promises";
 import path from "node:path";
 import { toMemoryPathRel } from "../memory-lifecycle-ledger-utils.js";
@@ -79,6 +79,20 @@ export interface MemoryReadStoreDeps {
   readonly wearablesDir: string;
 }
 
+/**
+ * A backend READ failure worth propagating from a strict (census) scan: an
+ * errno-coded fs error that is NOT `ENOENT`. `ENOENT` (an absent dir) and
+ * code-less rejections (symlink-escape containment failures from
+ * assertPathInsideRoot) are expected and skipped; `EACCES`/`EIO`/… mean the
+ * corpus is unreadable and a divergence census must fail loud rather than
+ * silently publish a partial count (issue #2156 round-5). Non-strict (recall)
+ * callers ignore this and degrade to a best-effort partial scan.
+ */
+function isPropagatableReadError(err: unknown): boolean {
+  const code = (err as NodeJS.ErrnoException).code;
+  return typeof code === "string" && code !== "ENOENT";
+}
+
 export class MemoryReadStore {
   constructor(
     private readonly deps: MemoryReadStoreDeps,
@@ -184,33 +198,66 @@ export class MemoryReadStore {
     return latestArtifacts;
   }
 
-  async collectActiveMemoryPaths(): Promise<string[]> {
+  /**
+   * Recursively collect `*.md` paths under `startDirs`, hardened against symlink
+   * escape: a category dir symlinked outside memoryDir (e.g. decisions/ -> an
+   * external dir) must NOT pull out-of-store files into a scan (info leak). Same
+   * walker-hardening pattern as document-scanner.ts / cli.ts /
+   * consolidation-provenance-check.ts; reuses the shared containment helper.
+   * Paths only — no frontmatter parse.
+   */
+  private async collectContainedMarkdownPaths(
+    startDirs: readonly string[],
+    propagateReadErrors = false,
+  ): Promise<string[]> {
     const filePaths: string[] = [];
 
-    // Resolve the memory root once for containment checks below. A category dir
-    // symlinked outside memoryDir (e.g. decisions/ -> an external dir) must NOT
-    // pull out-of-store files into the QMD-unavailable recall fallback (info
-    // leak). Same walker-hardening pattern as document-scanner.ts / cli.ts /
-    // consolidation-provenance-check.ts; reuses the shared containment helper.
+    // Resolve the memory root once for the containment checks below.
     let memoryRootReal: string;
     try {
       memoryRootReal = await realpath(this.deps.baseDir);
-    } catch {
+    } catch (err) {
+      // ENOENT: a not-yet-created root ⇒ empty. Any other errno is a backend
+      // read failure — propagate for strict census callers (never publish an
+      // unreadable corpus as empty); non-strict (recall) callers degrade.
+      if (propagateReadErrors && isPropagatableReadError(err)) throw err;
       return filePaths;
     }
 
-    const collectPaths = async (dir: string) => {
+    const collectPaths = async (dir: string): Promise<void> => {
       // Directory-level guard, isolated from per-entry handling: skip symlinked
       // or non-directory category dirs and assert the resolved dir stays inside
       // the memory root before reading. A failure here means the whole subtree
       // does not exist or escaped the store — fail closed by skipping it.
+      let dirStat: Stats;
+      try {
+        dirStat = await lstat(dir);
+      } catch (err) {
+        // ENOENT (absent category dir) is expected; a backend read error
+        // (EACCES/EIO/…) propagates in strict census mode.
+        if (propagateReadErrors && isPropagatableReadError(err)) throw err;
+        return;
+      }
+      if (dirStat.isSymbolicLink()) return; // never follow symlinks out of the store (both modes)
+      if (!dirStat.isDirectory()) {
+        // A category root that EXISTS but is not a directory (e.g. `facts/`
+        // replaced by a regular file) is a layout/backend failure: a strict
+        // census must not silently publish the remaining categories as healthy.
+        // Non-strict (recall) stays best-effort and skips it (issue #2156 round-10).
+        if (propagateReadErrors) {
+          throw new Error(`corpus scan: expected a directory but found a non-directory at ${dir}`);
+        }
+        return;
+      }
       let entries: Dirent[];
       try {
-        const dirStat = await lstat(dir);
-        if (dirStat.isSymbolicLink() || !dirStat.isDirectory()) return;
         assertPathInsideRoot(memoryRootReal, await realpath(dir), dir);
         entries = await readdir(dir, { withFileTypes: true });
-      } catch {
+      } catch (err) {
+        // A containment failure carries no errno (skip); a backend read error
+        // (EACCES …) propagates in strict census mode so a partial subtree is
+        // not silently counted.
+        if (propagateReadErrors && isPropagatableReadError(err)) throw err;
         return;
       }
 
@@ -231,8 +278,11 @@ export class MemoryReadStore {
           try {
             assertPathInsideRoot(memoryRootReal, await realpath(fullPath), fullPath);
             filePaths.push(fullPath);
-          } catch {
-            // Skip just this entry (symlink/containment/realpath failure).
+          } catch (err) {
+            // ENOENT (vanished mid-scan) or a code-less containment/symlink
+            // rejection ⇒ skip just this file. A backend read error propagates
+            // in strict census mode.
+            if (propagateReadErrors && isPropagatableReadError(err)) throw err;
           }
         }
       }
@@ -243,6 +293,13 @@ export class MemoryReadStore {
       }
     };
 
+    for (const dir of startDirs) {
+      await collectPaths(dir);
+    }
+    return filePaths;
+  }
+
+  async collectActiveMemoryPaths(options?: { propagateReadErrors?: boolean }): Promise<string[]> {
     // Scan EVERY supported memory category directory, not just the legacy four
     // (facts/procedures/reasoning-traces/corrections). Issue #1497: the QMD
     // filesystem-fallback recall path (orchestrator `recent_scan` ->
@@ -269,14 +326,26 @@ export class MemoryReadStore {
     // those files (they have a `---` frontmatter block) and leak queue items
     // into recall. None of these excluded dirs are in RECALL_FALLBACK_DIRS; the
     // exclusion is asserted by tests in storage-fallback-category-dirs.test.ts.
-    //
-    // collectPaths() already ignores missing dirs (try/catch) and the parser
-    // returns null for non-memory markdown, so unrelated files never crash the
-    // scan.
-    for (const dir of RECALL_FALLBACK_DIRS) {
-      await collectPaths(path.join(this.deps.baseDir, dir));
-    }
-    return filePaths;
+    return this.collectContainedMarkdownPaths(
+      RECALL_FALLBACK_DIRS.map((dir) => path.join(this.deps.baseDir, dir)),
+      options?.propagateReadErrors ?? false,
+    );
+  }
+
+  /**
+   * Cold-tier memory paths for the corpus census (issue #2156 finding D).
+   * Demoted memories move to `<baseDir>/cold/...` but stay active and reachable
+   * via cold recall, so the divergence census MUST count them — otherwise two
+   * replicas whose cold tiers differ show a false "converged" digest. Paths
+   * only, same symlink/containment hardening as the hot scan. This is NOT part
+   * of the recall fallback corpus (collectActiveMemoryPaths), whose scan is
+   * intentionally unchanged.
+   */
+  async collectColdMemoryPaths(options?: { propagateReadErrors?: boolean }): Promise<string[]> {
+    return this.collectContainedMarkdownPaths(
+      [this.deps.resolveTierRootDir("cold")],
+      options?.propagateReadErrors ?? false,
+    );
   }
 
   async readMemoriesWindow(options: {

package/src/storage.ts

@@ -4643,8 +4643,33 @@ export class StorageManager {
     return Math.max(1, Math.floor(batchSize));
   }
 
-  private async collectActiveMemoryPaths(): Promise<string[]> {
-    return this.memoryReadStore.collectActiveMemoryPaths();
+  /**
+   * Public cheap active-memory path scan (issue #2149 corpus watermark). Lists
+   * active memory file paths without parsing frontmatter — safe on a 100k+
+   * corpus, unlike readAllMemories().
+   */
+  async collectActiveMemoryPaths(options?: { propagateReadErrors?: boolean }): Promise<string[]> {
+    return this.memoryReadStore.collectActiveMemoryPaths(options);
+  }
+
+  /**
+   * Public cheap cold-tier path scan (issue #2156 finding D corpus census).
+   * Lists demoted-but-reachable memory paths under `cold/` without parsing
+   * frontmatter, so the divergence watermark counts the cold tier too.
+   */
+  async collectColdMemoryPaths(options?: { propagateReadErrors?: boolean }): Promise<string[]> {
+    return this.memoryReadStore.collectColdMemoryPaths(options);
+  }
+
+  /**
+   * Combined hot+cold corpus-mutation sentinel (issue #2156). Changes whenever
+   * EITHER tier is written, so a divergence census can bracket its hot/cold scan
+   * and detect a tier migration (write-cold-then-unlink-hot) racing the walkers,
+   * then retry for a consistent snapshot instead of caching a transient
+   * double-count or miss.
+   */
+  getCorpusScanVersion(): string {
+    return `${this.getMemoryCorpusVersion()}:${this.readColdWriteVersion()}`;
   }
 
   private async readParsedMemoriesFromPaths(filePaths: string[], batchSize?: number): Promise<MemoryFile[]> {