@remnic/core 10.0.52 → 10.0.53

package/dist/who-knows.js

@@ -9,19 +9,19 @@ import {
   loadWhoKnowsEntities,
   tokenizeTopic,
   validateWhoKnowsInput
-} from "./chunk-2ONKXEC7.js";
-import "./chunk-JIJAWTRK.js";
+} from "./chunk-5LBBXI2A.js";
+import "./chunk-OCXDZJL4.js";
 import "./chunk-UHVXE24Y.js";
 import "./chunk-DM5QZ3YU.js";
 import "./chunk-G6A3SGCE.js";
 import "./chunk-G6OFEGUR.js";
-import "./chunk-LXXIV7Y2.js";
-import "./chunk-JS44PBCM.js";
-import "./chunk-5CWH2LKL.js";
-import "./chunk-XKDQTDXN.js";
+import "./chunk-JEX2JL3U.js";
+import "./chunk-U2A56LLF.js";
+import "./chunk-DOJBV6WR.js";
+import "./chunk-PFUXPTBT.js";
 import "./chunk-CYEPCZN5.js";
-import "./chunk-XRFLR3XF.js";
-import "./chunk-7VO6E5UD.js";
+import "./chunk-DLZGSZVC.js";
+import "./chunk-OVZIYLBZ.js";
 import "./chunk-WNEVEMJ3.js";
 import "./chunk-EXYT52NR.js";
 import "./chunk-Y5NLBXPV.js";
@@ -57,8 +57,8 @@ import "./chunk-7CT3AI52.js";
 import "./chunk-G7D6GZ5J.js";
 import "./chunk-7VLBZKNB.js";
 import "./chunk-UUZ7KZAO.js";
-import "./chunk-AJJPXUE6.js";
-import "./chunk-G26YNEOC.js";
+import "./chunk-WBN66HPI.js";
+import "./chunk-NWA6IXR3.js";
 import "./chunk-VILPVT4S.js";
 import "./chunk-AF2XMJKD.js";
 import "./chunk-OR42ZI54.js";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remnic/core",
-  "version": "10.0.52",
+  "version": "10.0.53",
   "description": "Framework-agnostic Remnic memory engine — orchestrator, storage, extraction, search, trust zones",
   "type": "module",
   "main": "dist/index.js",
@@ -3346,7 +3346,7 @@
     "core"
   ],
   "peerDependencies": {
-    "@remnic/coding-graph": "^10.0.52"
+    "@remnic/coding-graph": "^10.0.53"
   },
   "peerDependenciesMeta": {
     "@remnic/coding-graph": {

package/src/namespaces/search.test.ts

@@ -1,6 +1,11 @@
 import assert from "node:assert/strict";
 import test from "node:test";
+import os from "node:os";
+import path from "node:path";
+import { mkdtemp, rm } from "node:fs/promises";
 import { NamespaceSearchRouter, normalizeQmdResultPath } from "./search.js";
+import { MaintenanceScheduler } from "../orchestration/maintenance.js";
+import type { NamespaceCatalog } from "./catalog.js";
 import type {
   SearchBackend,
   SearchBackendStatus,
@@ -977,3 +982,119 @@ test("normalizeQmdResultPath strips a real collection prefix but not a category
   // A non-category collection prefix on a plain hit is still stripped.
   assert.equal(normalizeQmdResultPath("openclaw-engram/facts/a.md", "openclaw-engram"), "facts/a.md");
 });
+
+/**
+ * Orama, LanceDB, Meilisearch and remote backends have no updateStrict or
+ * updateCollectionStrict; their plain update() is fail-open. They report whether
+ * an update ran through updateCompleted() (issue #3286).
+ */
+class NonQmdBackend extends FakeBackend {
+  completedCalls = 0;
+
+  constructor(completed: boolean) {
+    super(false);
+    // Instance properties shadow the QMD-only strict methods FakeBackend defines.
+    Object.defineProperty(this, "updateStrict", { value: undefined });
+    Object.defineProperty(this, "updateCollectionStrict", { value: undefined });
+    Object.defineProperty(this, "updateCompleted", {
+      value: async (): Promise<boolean> => {
+        this.completedCalls += 1;
+        return completed;
+      },
+    });
+  }
+}
+
+function nonQmdRouter(completed: boolean, created: NonQmdBackend[], cfg: PluginConfig = config()): NamespaceSearchRouter {
+  return new NamespaceSearchRouter(
+    cfg,
+    { storageFor: async (namespace: string) => ({ dir: `/tmp/remnic/${namespace}` }) },
+    () => {
+      const backend = new NonQmdBackend(completed);
+      created.push(backend);
+      return backend;
+    },
+  );
+}
+
+test("strict updateNamespaces fails for a non-QMD backend whose refresh did not complete", async () => {
+  const created: NonQmdBackend[] = [];
+  const router = nonQmdRouter(false, created);
+
+  await assert.rejects(
+    router.updateNamespaces(["main"], undefined, { strict: true }),
+    /did not complete/,
+  );
+  assert.equal(created.reduce((sum, backend) => sum + backend.updates, 0), 0, "the fail-open update() is not what strict runs");
+});
+
+test("strict updateNamespaces succeeds for a non-QMD backend whose refresh completed", async () => {
+  const created: NonQmdBackend[] = [];
+  const router = nonQmdRouter(true, created);
+
+  assert.equal(await router.updateNamespaces(["main", "shared"], undefined, { strict: true }), 2);
+  assert.equal(created.reduce((sum, backend) => sum + backend.completedCalls, 0), 2);
+  assert.equal(created.reduce((sum, backend) => sum + backend.updates, 0), 0);
+});
+
+test("non-strict updateNamespaces keeps the fail-open update() for non-QMD backends", async () => {
+  const created: NonQmdBackend[] = [];
+  const router = nonQmdRouter(false, created);
+
+  assert.equal(await router.updateNamespaces(["main"]), 1);
+  assert.equal(created.reduce((sum, backend) => sum + backend.updates, 0), 1);
+  assert.equal(created.reduce((sum, backend) => sum + backend.completedCalls, 0), 0);
+});
+
+test("strict updateNamespaces falls back to update() for a backend with no strict or completed variant", async () => {
+  const created: FakeBackend[] = [];
+  const router = new NamespaceSearchRouter(
+    config(),
+    { storageFor: async (namespace: string) => ({ dir: `/tmp/remnic/${namespace}` }) },
+    () => {
+      const backend = new FakeBackend(false);
+      Object.defineProperty(backend, "updateCollectionStrict", { value: undefined });
+      created.push(backend);
+      return backend;
+    },
+  );
+
+  assert.equal(await router.updateNamespaces(["main"], undefined, { strict: true }), 1);
+  assert.equal(created.reduce((sum, backend) => sum + backend.updates, 0), 1);
+});
+
+test("namespace-mode maintenance is served only when a non-QMD backend's refresh completed", async () => {
+  for (const completed of [true, false]) {
+    const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-3286-ns-completion-"));
+    const cfg: PluginConfig = {
+      ...config(),
+      memoryDir,
+      sharedNamespace: "shared",
+      namespacePolicies: [],
+      maintenanceNamespaceLockStaleMs: 100,
+      qmdMaintenanceEnabled: true,
+      qmdMaintenanceDebounceMs: 60_000,
+      qmdAutoEmbedEnabled: false,
+    };
+    const created: NonQmdBackend[] = [];
+    const router = nonQmdRouter(completed, created, cfg);
+    const catalog: NamespaceCatalog = Object.assign(Object.create(null), {
+      enabled: true,
+      async listNamespaces() {
+        return [{ namespace: "main" }];
+      },
+      async markMaintenance() {},
+    });
+    const qmd: SearchBackend = Object.assign(Object.create(null), { isAvailable: () => true });
+    const scheduler = new MaintenanceScheduler({ config: cfg, getQmd: () => qmd, namespaceSearchRouter: router, namespaceCatalog: catalog });
+    try {
+      const served = scheduler.requestQmdMaintenanceCompletion();
+      await scheduler.runQmdMaintenance();
+      assert.equal(await served, completed, completed ? "a completed refresh serves the request" : "an incomplete refresh must keep the marker owed");
+      assert.ok(created.some((backend) => backend.completedCalls > 0), "maintenance asked the backend whether it completed");
+    } finally {
+      await scheduler.dispose();
+      await rm(memoryDir, { recursive: true, force: true });
+    }
+  }
+});

package/src/namespaces/search.ts

@@ -596,6 +596,14 @@ async function updateBackendRecord(
       await record.backend.updateCollectionStrict(record.collection, execution);
       return;
     }
+    // Backends with no strict variant (Orama, LanceDB, Meilisearch, remote) have a fail-open
+    // update(): it resolves when it returned early or failed. Their updateCompleted() says
+    // whether the refresh ran, so strict callers get a failure instead of a fake success.
+    const completed = await record.backend.updateCompleted?.(execution);
+    if (completed !== undefined) {
+      if (!completed) throw new Error("search backend update did not complete");
+      return;
+    }
   }
   await record.backend.update(execution);
 }

package/src/orchestration/lifecycle-pending-fallback-drain.ts

@@ -0,0 +1,97 @@
+/**
+ * Fallback lifecycle pending-append drain for targets with no secure
+ * StorageManager. Moved verbatim out of maintenance.ts (issue #2033) so that
+ * file stays under the structural size ceiling; behavior is unchanged.
+ */
+
+import { appendFile, mkdir, readFile, writeFile } from "node:fs/promises";
+import path from "node:path";
+
+import { log } from "../logger.js";
+import {
+  drainPendingLifecycleLedgerIfAny,
+  pendingLifecycleLedgerDir,
+  type LifecyclePendingIo,
+} from "../storage/memory-lifecycle-ledger-access.js";
+import { probeEncryptedRegularFileHeader } from "../secure-store/secure-fs.js";
+import { listContainedSpillFiles } from "../utils/path-containment.js";
+
+/**
+ * Drain a filesystem-fallback target's pending lifecycle-append spill when no
+ * secure StorageManager is available (catalog-disabled namespace; issue
+ * #2033). Uses the safe path-scoped PLAINTEXT context this fallback already
+ * operates in: a keyless drain can only fold plaintext spills into a plaintext
+ * ledger, so it refuses when the ledger or ANY spill is encrypted at rest,
+ * leaving those pending for a keyed (catalog-path) drain — exactly mirroring
+ * the fallback compaction's encrypted-ledger deferral. Non-fatal; a fast
+ * no-op when nothing is pending. The shared spill lister enforces the symlink/
+ * containment guard on every spill file it reads or deletes.
+ */
+export async function drainPendingLifecycleAppendsForPath(memoryDir: string): Promise<void> {
+  const ledgerPath = path.join(memoryDir, "state", "memory-lifecycle-ledger.jsonl");
+  const spillDir = pendingLifecycleLedgerDir(ledgerPath);
+  // Enumerate spills through the SAME guarded lister the actual drain uses
+  // (listContainedSpillFiles): it rejects a symlinked/non-directory spill dir
+  // and skips any symlink/FIFO/device/escaping entry, returning only regular
+  // files contained in the dir. Never raw-readdir + open() an untrusted name:
+  // opening a FIFO would block until a writer appears and a symlink would
+  // redirect the probe outside the dir (#2033). Enumerate BOTH live `*.jsonl`
+  // spills and crash-orphaned `*.jsonl.claimed` files: a drain that died
+  // between claiming a spill (rename to `.claimed`) and committing it leaves
+  // the ONLY copy of those rows as a claimed orphan. Gating solely on live
+  // spills would early-return here whenever the pending dir holds nothing but
+  // orphans, stranding those rows forever; proceeding lets
+  // drainPendingLifecycleLedgerIfAny -> recoverOrphanedClaims restore and
+  // re-commit them under the ledger lock (#2033).
+  let spillFiles: string[];
+  let claimedOrphans: string[];
+  try {
+    spillFiles = await listContainedSpillFiles(spillDir);
+    claimedOrphans = await listContainedSpillFiles(spillDir, ".jsonl.claimed");
+  } catch (err) {
+    log.debug(`fallback lifecycle pending scan failed (non-fatal) for ${memoryDir}: ${err}`);
+    return;
+  }
+  if (spillFiles.length === 0 && claimedOrphans.length === 0) return;
+  // The pending encryption state is authoritative only UNDER the ledger lock:
+  // drainPendingLifecycleLedgerIfAny waits for the lock and re-enumerates the
+  // pending dir, so a secure-store writer that spills an encrypted file — or a
+  // keyed compaction that rewrites the ledger encrypted — while this fallback
+  // waits would defeat any pre-lock probe (#2033). Probe each file at the
+  // moment of use inside the held lock instead: readSecure refuses an
+  // encrypted spill (the fold reads each spill lazily and propagates the throw
+  // BEFORE claiming that spill, leaving it intact) and the append refuses a
+  // ledger that became encrypted (the fold rolls the claims back). Either way
+  // the plaintext IO
+  // never reads or writes ciphertext; the keyed catalog-path drain handles the
+  // deferred rows.
+  const io: LifecyclePendingIo = {
+    writeSecure: async (filePath, payload) => {
+      await mkdir(path.dirname(filePath), { recursive: true });
+      await writeFile(filePath, payload, "utf-8");
+    },
+    readSecure: async (filePath) => {
+      if (await probeEncryptedRegularFileHeader(filePath)) {
+        throw new Error(`fallback lifecycle drain deferred: encrypted spill at ${filePath}`);
+      }
+      return readFile(filePath, "utf-8");
+    },
+  };
+  try {
+    await drainPendingLifecycleLedgerIfAny(
+      ledgerPath,
+      io,
+      async (payload) => {
+        if (await probeEncryptedRegularFileHeader(ledgerPath)) {
+          throw new Error(`fallback lifecycle drain deferred: ledger became encrypted at ${ledgerPath}`);
+        }
+        await appendFile(ledgerPath, payload, "utf-8");
+      },
+      async () => {
+        await mkdir(path.dirname(ledgerPath), { recursive: true });
+      },
+    );
+  } catch (err) {
+    log.debug(`fallback lifecycle pending drain failed (non-fatal) for ${memoryDir}: ${err}`);
+  }
+}