@remnic/core 9.27.0 → 9.29.0

package/src/operator-doctor-replica.test.ts

@@ -0,0 +1,261 @@
+import assert from "node:assert/strict";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import { createServer, type RequestListener } from "node:http";
+import type { AddressInfo, Socket } from "node:net";
+import os from "node:os";
+import path from "node:path";
+import test from "node:test";
+/**
+ * Replica-divergence doctor check (issue #2149).
+ *
+ * Verifies `summarizeReplicaDivergence` is ok when disabled / peerless /
+ * converged, warns (never ok) when a peer diverges or cannot be polled, surfaces
+ * concrete deltas, never leaks a token, and is wired into `runOperatorDoctor`.
+ * The check takes a pre-computed local watermark set (kept DTS-light); a real
+ * ephemeral 127.0.0.1 server stands in for the peer daemon.
+ */
+import { parseConfig } from "./config.js";
+import type { CorpusWatermark } from "./corpus-watermark.js";
+import { summarizeReplicaDivergence } from "./operator-doctor-replica.js";
+import { type OperatorToolkitOrchestrator, runOperatorDoctor } from "./operator-toolkit.js";
+import type { ReplicaDivergenceStatus } from "./replica-divergence.js";
+import { type ReplicaPeersConfig, parseReplicaPeersConfig } from "./replica-peers-config.js";
+import { StorageManager } from "./storage.js";
+import type { PluginConfig } from "./types.js";
+
+function watermark(namespace: string, overrides: Partial<CorpusWatermark> = {}): CorpusWatermark {
+  return {
+    namespace,
+    memoryFileCount: 10,
+    newestPartition: "2026-03-08",
+    newestWriteAt: "2026-03-08T00:00:00.000Z",
+    digest: `digest-${namespace}`,
+    computedAt: new Date().toISOString(),
+    ...overrides,
+  };
+}
+
+function replicaConfig(block: Record<string, unknown>): ReplicaPeersConfig {
+  return parseReplicaPeersConfig({ replicaPeers: block });
+}
+
+interface Peer {
+  url: string;
+  requests: { authorization: string | undefined }[];
+  close: () => Promise<void>;
+}
+
+async function startPeer(handler: RequestListener): Promise<Peer> {
+  const requests: Peer["requests"] = [];
+  const sockets = new Set<Socket>();
+  const server = createServer((req, res) => {
+    requests.push({ authorization: req.headers.authorization });
+    handler(req, res);
+  });
+  server.on("connection", (socket) => {
+    sockets.add(socket);
+    socket.on("close", () => sockets.delete(socket));
+  });
+  const listening = Promise.withResolvers<void>();
+  server.once("error", listening.reject);
+  server.listen(0, "127.0.0.1", listening.resolve);
+  await listening.promise;
+  const address = server.address() as AddressInfo;
+  return {
+    url: `http://127.0.0.1:${address.port}`,
+    requests,
+    close: async () => {
+      for (const socket of sockets) socket.destroy();
+      await new Promise<void>((resolve) => server.close(() => resolve()));
+    },
+  };
+}
+
+function healthHandler(corpus: CorpusWatermark[]): RequestListener {
+  return (req, res) => {
+    if (new URL(req.url ?? "/", "http://127.0.0.1").pathname === "/engram/v1/health") {
+      res.writeHead(200, { "content-type": "application/json" });
+      res.end(JSON.stringify({ ok: true, corpus }));
+      return;
+    }
+    res.writeHead(404, { "content-type": "application/json" });
+    res.end(JSON.stringify({ error: "not_found" }));
+  };
+}
+
+test("summarizeReplicaDivergence: disabled resolves ok and performs no network I/O", async () => {
+  const peer = await startPeer(healthHandler([]));
+  try {
+    const check = await summarizeReplicaDivergence(
+      replicaConfig({ enabled: false, peers: [{ url: peer.url }] }),
+      [watermark("global")],
+    );
+    assert.equal(check.key, "replica_divergence");
+    assert.equal(check.status, "ok");
+    assert.match(check.summary, /disabled/i);
+    assert.equal(peer.requests.length, 0, "a disabled check must not poll");
+  } finally {
+    await peer.close();
+  }
+});
+
+test("summarizeReplicaDivergence: enabled with no peers resolves ok", async () => {
+  const check = await summarizeReplicaDivergence(replicaConfig({ enabled: true, peers: [] }), [watermark("global")]);
+  assert.equal(check.status, "ok");
+  assert.match(check.summary, /no peers/i);
+});
+
+test("summarizeReplicaDivergence: a converged peer resolves ok", async () => {
+  const local = [watermark("global")];
+  const peer = await startPeer(healthHandler(local));
+  try {
+    const check = await summarizeReplicaDivergence(replicaConfig({ enabled: true, peers: [{ url: peer.url }] }), local);
+    assert.equal(check.status, "ok");
+    assert.match(check.summary, /converged/i);
+  } finally {
+    await peer.close();
+  }
+});
+
+test("summarizeReplicaDivergence: an incomplete local census downgrades a converged peer AND warns (round 6)", async () => {
+  // /health downgrades a would-be-converged peer to unknown/local_census_incomplete
+  // when the local census is partial; the doctor must derive the SAME peer state
+  // through the shared gate, not render `peer: converged` with converged details.
+  const local = [watermark("global")];
+  const peer = await startPeer(healthHandler(local));
+  try {
+    const check = await summarizeReplicaDivergence(
+      replicaConfig({ enabled: true, peers: [{ url: peer.url }] }),
+      local,
+      { localCensusComplete: false },
+    );
+    assert.equal(check.status, "warn", "an incomplete local census cannot certify convergence");
+    assert.match(check.summary, /local census incomplete/i);
+    const report = check.details as ReplicaDivergenceStatus;
+    assert.equal(report.censusComplete, false);
+    assert.equal(report.peers[0]?.state, "unknown", "the peer detail is downgraded, not left converged");
+    assert.equal(report.peers[0]?.reason, "local_census_incomplete");
+    assert.notEqual(report.peers[0]?.state, "converged");
+  } finally {
+    await peer.close();
+  }
+});
+
+test("summarizeReplicaDivergence: a diverged peer warns with the concrete delta", async () => {
+  const peer = await startPeer(healthHandler([watermark("global", { memoryFileCount: 340_000, digest: "peer" })]));
+  try {
+    const check = await summarizeReplicaDivergence(
+      replicaConfig({ enabled: true, peers: [{ url: peer.url }] }),
+      [watermark("global", { memoryFileCount: 190_000, digest: "local" })],
+    );
+    assert.equal(check.status, "warn", "a divergence must warn, never ok");
+    assert.match(check.summary, /diverged/);
+    assert.match(check.summary, /file_count_delta=150000/, "the operator sees the concrete delta");
+  } finally {
+    await peer.close();
+  }
+});
+
+test("summarizeReplicaDivergence: an unreachable peer warns and is not conflated with ok (§22)", async () => {
+  const peer = await startPeer((_req, res) => {
+    res.writeHead(500);
+    res.end("boom");
+  });
+  try {
+    const check = await summarizeReplicaDivergence(
+      replicaConfig({ enabled: true, peers: [{ url: peer.url }] }),
+      [watermark("global")],
+    );
+    assert.equal(check.status, "warn");
+    assert.match(check.summary, /unreachable/);
+    assert.match(check.summary, /http_500/);
+  } finally {
+    await peer.close();
+  }
+});
+
+test("summarizeReplicaDivergence: the peer token never appears in the doctor check output", async () => {
+  const secret = "DOCTOR-SECRET-TOKEN";
+  const peer = await startPeer(healthHandler([watermark("global", { memoryFileCount: 999_999, digest: "x" })]));
+  try {
+    const check = await summarizeReplicaDivergence(
+      replicaConfig({ enabled: true, peers: [{ url: peer.url, token: secret }] }),
+      [watermark("global")],
+    );
+    assert.ok(peer.requests.some((request) => request.authorization === `Bearer ${secret}`), "token IS used (non-vacuous)");
+    assert.ok(!JSON.stringify(check).includes(secret), "token must never appear in the doctor output");
+  } finally {
+    await peer.close();
+  }
+});
+
+async function writeMemory(memoryDir: string, rel: string): Promise<void> {
+  const full = path.join(memoryDir, rel);
+  await mkdir(path.dirname(full), { recursive: true });
+  await writeFile(full, `---\nid: ${path.basename(rel, ".md")}\n---\n\nbody\n`, "utf-8");
+}
+
+async function makeFixture(): Promise<{
+  root: string;
+  memoryDir: string;
+  configPath: string;
+  orchestrator: OperatorToolkitOrchestrator;
+}> {
+  const root = await mkdtemp(path.join(os.tmpdir(), "remnic-replica-doctor-"));
+  const memoryDir = path.join(root, "memory");
+  await mkdir(memoryDir, { recursive: true });
+  const rawConfig = {
+    openaiApiKey: "sk-test",
+    memoryDir,
+    qmdEnabled: false,
+    namespacesEnabled: false,
+    defaultNamespace: "global",
+  };
+  const config: PluginConfig = parseConfig(rawConfig);
+  const configPath = path.join(root, "openclaw.json");
+  await writeFile(
+    configPath,
+    JSON.stringify({ plugins: { entries: { "openclaw-remnic": { config: rawConfig } } } }, null, 2),
+    "utf-8",
+  );
+  const orchestrator: OperatorToolkitOrchestrator = {
+    config,
+    storage: new StorageManager(memoryDir),
+    qmd: {
+      async probe() {
+        return false;
+      },
+      isAvailable() {
+        return false;
+      },
+      async ensureCollection() {
+        return "skipped";
+      },
+      debugStatus() {
+        return "disabled";
+      },
+    },
+    conversationIndexCoordinator: {
+      async getHealth() {
+        return { enabled: false, backend: "qmd" as const, status: "disabled" as const, chunkDocCount: 0, lastUpdateAt: null };
+      },
+      async rebuild() {
+        return { chunks: 0, skipped: true, reason: "disabled", embedded: false, rebuilt: false };
+      },
+    },
+  };
+  return { root, memoryDir, configPath, orchestrator };
+}
+
+test("runOperatorDoctor: includes the replica_divergence check (disabled -> ok)", async () => {
+  const { root, memoryDir, configPath, orchestrator } = await makeFixture();
+  try {
+    await writeMemory(memoryDir, "facts/2026-03-08/a.md");
+    const report = await runOperatorDoctor({ configPath, orchestrator });
+    const check = report.checks.find((entry) => entry.key === "replica_divergence");
+    assert.ok(check, "expected a replica_divergence check");
+    assert.equal(check?.status, "ok");
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+});

package/src/operator-doctor-replica.ts

@@ -0,0 +1,129 @@
+/**
+ * Replica-divergence doctor check (issue #2149).
+ *
+ * Kept deliberately LIGHT: it takes the ALREADY-computed local corpus watermark
+ * set (the same array the `corpus_watermark` check produces) instead of importing
+ * the corpus census helpers or `PluginConfig`. That keeps this new tsup DTS entry
+ * from pulling the heavy `PluginConfig` type graph into its `.d.ts` — two sibling
+ * PRs OOMed the default-heap DTS worker exactly that way (the issue #1562 heap
+ * cliff). `runOperatorDoctor` computes the local watermarks once for the corpus
+ * check and hands them here via `corpusWatermarksFromCheck`, so there is also no
+ * second corpus scan.
+ *
+ * Status: `ok` when disabled, peerless, or every peer converged; `warn` when any
+ * peer diverged beyond threshold, could not be polled (unreachable/unknown), or
+ * the local corpus census was incomplete — a scan that cannot ask a peer, or
+ * cannot fully read its OWN corpus, must never read as agreement (AGENTS.md §22).
+ * The summary carries concrete deltas so an operator sees numbers, not a verdict.
+ * Peer tokens are never resolved into, logged from, or printed by this check; a
+ * peer is identified only by its redacted host:port. The CLI forwards the host
+ * SecretRef resolver (when one is configured) so a SecretRef peer token
+ * authenticates here too; without a resolver such a token degrades to a per-peer
+ * `token_error` (never a throw). The always-on `/health` surface remains the
+ * primary reporting path. Reconciliation is issue #2150.
+ */
+
+import type { CorpusWatermark } from "./corpus-watermark.js";
+import type { OperatorDoctorCheck } from "./operator-doctor-types.js";
+import { type FetchLike, type ReplicaPeerReport, gateReportByCensus, pollReplicaPeers } from "./replica-divergence.js";
+import { resolveReplicaPeersConfig, type ReplicaPeersConfig } from "./replica-peers-config.js";
+import type { ResolveSecretRefFn } from "./resolve-auth-token.js";
+
+export interface SummarizeReplicaDivergenceOptions {
+  now?: Date;
+  resolveSecretRef?: ResolveSecretRefFn | null;
+  /** Injectable for tests; production polls with global fetch. */
+  fetchImpl?: FetchLike;
+  /**
+   * Whether the LOCAL corpus census that produced `localWatermarks` was complete
+   * (the `corpus_watermark` check was `ok`). When false, a would-be-converged
+   * result is downgraded to `warn`: a namespace missing from an incomplete local
+   * scan must never let the replica check certify convergence (cursor finding).
+   */
+  localCensusComplete?: boolean;
+}
+
+function formatPeerLine(peer: ReplicaPeerReport): string {
+  if (peer.state === "unreachable" || peer.state === "unknown") {
+    return `${peer.peer}: ${peer.state} (${peer.reason ?? "unknown"})`;
+  }
+  if (peer.state === "converged") return `${peer.peer}: converged`;
+  const divergedNamespaces = peer.namespaces
+    .filter((delta) => delta.diverged)
+    .map((delta) => `${delta.namespace}: ${delta.reasons.join("+")}`);
+  return `${peer.peer}: diverged [${divergedNamespaces.join(", ")}]`;
+}
+
+export async function summarizeReplicaDivergence(
+  rawConfig: ReplicaPeersConfig | undefined,
+  localWatermarks: readonly CorpusWatermark[],
+  options: SummarizeReplicaDivergenceOptions = {},
+): Promise<OperatorDoctorCheck> {
+  // A host adapter or hand-built PluginConfig can omit the block entirely;
+  // normalize through the same read-boundary resolver /health uses so doctor
+  // degrades to disabled instead of aborting the whole run (round 5, codex P2).
+  const replicaConfig = resolveReplicaPeersConfig(rawConfig);
+  if (!replicaConfig.enabled) {
+    return {
+      key: "replica_divergence",
+      status: "ok",
+      summary: "Replica divergence detection is disabled.",
+      details: { enabled: false, pending: false, polledAt: null, peers: [] },
+    };
+  }
+  if (replicaConfig.peers.length === 0) {
+    return {
+      key: "replica_divergence",
+      status: "ok",
+      summary: "Replica divergence detection is enabled, but no peers are configured.",
+      details: { enabled: true, pending: false, polledAt: null, peers: [] },
+    };
+  }
+
+  const report = await pollReplicaPeers({
+    config: replicaConfig,
+    localWatermarks,
+    now: options.now,
+    resolveSecretRef: options.resolveSecretRef,
+    fetchImpl: options.fetchImpl,
+  });
+  // Apply the SAME census gate /health uses (round 6): otherwise the doctor
+  // renders `peer: converged` lines and converged `details` while /health
+  // downgrades those same peers to `unknown`/`local_census_incomplete`, so a
+  // machine reading doctor JSON sees agreement /health denies. Gating here
+  // turns every would-be-converged peer into `unknown` when the local census
+  // was incomplete, so both surfaces derive peer state from one function.
+  const gated = gateReportByCensus(report, options.localCensusComplete !== false);
+  const diverged = gated.peers.filter((peer) => peer.state === "diverged");
+  const unreachable = gated.peers.filter((peer) => peer.state === "unreachable" || peer.state === "unknown");
+  const lines = gated.peers.map(formatPeerLine);
+  const censusIncomplete = gated.censusComplete === false;
+
+  if (diverged.length > 0 || unreachable.length > 0 || censusIncomplete) {
+    const counts = [`${diverged.length} diverged`, `${unreachable.length} unreachable/unknown`];
+    if (censusIncomplete) counts.push("local census incomplete");
+    return {
+      key: "replica_divergence",
+      status: "warn",
+      summary:
+        `Replica divergence across ${gated.peers.length} peer(s): ` +
+        `${counts.join(", ")}. ${lines.join("; ")}`,
+      remediation:
+        (censusIncomplete
+          ? "The local corpus census was incomplete (see the corpus_watermark check), so convergence cannot be " +
+            "certified — resolve that first. "
+          : "") +
+        "Investigate the flagged peer(s): a diverged peer's corpus differs beyond the configured threshold " +
+        "(a possible split-brain — the pair's recall will differ across failover); an unreachable/unknown peer " +
+        "could not be polled or compared. Detection only; reconciliation is tracked in issue #2150.",
+      details: gated,
+    };
+  }
+
+  return {
+    key: "replica_divergence",
+    status: "ok",
+    summary: `All ${gated.peers.length} replica peer(s) converged. ${lines.join("; ")}`,
+    details: gated,
+  };
+}

package/src/operator-toolkit.ts

@@ -57,7 +57,8 @@ import type {
 } from "./types.js";
 import { reportBufferSurpriseDistribution } from "./buffer-surprise-report.js";
 import { readJudgeVerdictStats } from "./extraction-judge-telemetry.js";
-import { summarizeCorpusWatermark } from "./operator-doctor-corpus.js";
+import { summarizeCorpusAndReplica } from "./operator-doctor-corpus.js";
+import type { ResolveSecretRefFn } from "./resolve-auth-token.js";
 
 const OPENCLAW_REMNIC_PLUGIN_IDS = ["openclaw-remnic", "openclaw-engram"] as const;
 
@@ -384,6 +385,8 @@ export interface OperatorDoctorOptions {
   orchestrator: OperatorToolkitOrchestrator;
   configPath?: string;
   now?: Date;
+  /** Host SecretRef resolver so a SecretRef peer token authenticates during doctor replica polling (#2149). */
+  resolveSecretRef?: ResolveSecretRefFn | null;
 }
 
 export interface OperatorConfigReviewOptions {
@@ -1496,11 +1499,9 @@ export async function runOperatorDoctor(options: OperatorDoctorOptions): Promise
   // Shows hot/cold counts, per-status breakdown, and forgotten-memory count.
   // Informational only — never errors, never blocks doctor from returning ok.
   checks.push(await summarizeTierDistribution(options.orchestrator.storage));
-
   checks.push(await summarizeDreamsPhases(config, storage));
   checks.push(await summarizeExtractionLiveness(config, storage, options.orchestrator.buffer));
-  checks.push(await summarizeCorpusWatermark(config, (dir) => new StorageManager(dir))); // corpus watermark #2149
-
+  checks.push(...(await summarizeCorpusAndReplica(config, (d) => new StorageManager(d), options.resolveSecretRef)));
   // Security mitigation status (issue #565).
   // Reports whether the cross-namespace budget and anomaly detection
   // mitigations are enabled and surfaces config values for operator review.

package/src/orchestration/orchestrator-init.ts

@@ -116,8 +116,8 @@ export class OrchestratorInitCoordinator {
         quiet: true,
         logger: (message) => log.info(message),
       });
-      await this.deps.storage.ensureDirectories();
       await this.deps.storage.loadAliases();
+      await this.deps.storage.ensureDirectories();
       if (resolveNamespaceCapabilities(this.deps.config).namespaces) {
         const namespaces = new Set<string>([
           this.deps.config.defaultNamespace,
@@ -126,8 +126,8 @@ export class OrchestratorInitCoordinator {
         ]);
         for (const ns of namespaces) {
           const sm = await this.deps.storageRouter.storageFor(ns);
-          await sm.ensureDirectories();
           await sm.loadAliases().catch(() => undefined);
+          await sm.ensureDirectories();
         }
         // Explicitly seed the catalog with all configured namespaces at startup
         // (round 6, cursor Medium — NBLlR). The storageFor loop above fires the