@remnic/core 9.7.15 → 9.8.0

package/src/namespaces/search.test.ts

@@ -1,8 +1,9 @@
 import assert from "node:assert/strict";
 import test from "node:test";
-import { NamespaceSearchRouter } from "./search.js";
+import { NamespaceSearchRouter, normalizeQmdResultPath } from "./search.js";
 import type {
   SearchBackend,
+  SearchBackendStatus,
   SearchExecutionOptions,
   SearchQueryOptions,
 } from "../search/port.js";
@@ -79,6 +80,13 @@ class FakeBackend implements SearchBackend {
     return this.diagnostics.versionStatus ?? null;
   }
 
+  statusResult: SearchBackendStatus | null = null;
+
+  async status(): Promise<SearchBackendStatus> {
+    if (!this.statusResult) throw new Error("no status available");
+    return this.statusResult;
+  }
+
   async dispose(): Promise<void> {
     this.disposed += 1;
   }
@@ -205,6 +213,7 @@ function config(): PluginConfig {
     qmdCollection: "openclaw-engram",
     defaultNamespace: "main",
     qmdMaxResults: 10,
+    qmdEmbeddingBacklogThreshold: 5000,
   } as PluginConfig;
 }
 
@@ -404,10 +413,14 @@ test("searchAcrossNamespaces preserves same path results from distinct namespace
     maxResults: 10,
   });
 
-  assert.deepEqual(
-    results.map((result) => result.snippet),
-    ["main", "shared"],
-  );
+  assert.deepEqual(results.map((result) => result.snippet), ["main", "shared"]);
+  assert.deepEqual(results.map((result) => result.namespace), ["main", "shared"]);
+  // Same-relative-path hits from distinct namespaces resolve to distinct
+  // absolute paths (globally-unique identity); display surfaces relativize.
+  assert.deepEqual(results.map((result) => result.path), [
+    "/tmp/remnic/main/facts/a.md",
+    "/tmp/remnic/shared/facts/a.md",
+  ]);
 });
 
 test("searchAcrossNamespaces passes scoped collection to backend search methods", async () => {
@@ -657,6 +670,53 @@ test("healthForNamespace reports daemon mode from live cached namespace backend"
   assert.deepEqual(created[1]?.checkCollections, []);
 });
 
+test("healthForNamespace reports embedding backlog from backend status", async () => {
+  const created: FakeBackend[] = [];
+  const router = new NamespaceSearchRouter(
+    config(),
+    { storageFor: async (namespace: string) => ({ dir: `/tmp/remnic/${namespace}` }) },
+    () => {
+      const backend = new FakeBackend(false, [], { ensure: "present" }, false, {
+        debugStatus: "backlog-test",
+      });
+      backend.statusResult = {
+        pendingEmbeddings: 42,
+        oldestPendingAgeMs: 120_000,
+        totalFiles: 100,
+        embeddedFiles: 58,
+      };
+      created.push(backend);
+      return backend;
+    },
+  );
+
+  const health = await router.healthForNamespace("shared");
+
+  assert.equal(health.pendingEmbeddings, 42);
+  assert.equal(health.oldestPendingAgeMs, 120_000);
+  assert.equal(health.embeddingBacklogThreshold, 5000);
+  assert.equal(created.length, 1);
+  assert.equal(created[0]?.disposed, 1);
+});
+
+test("healthForNamespace returns null backlog when backend status fails", async () => {
+  const router = new NamespaceSearchRouter(
+    config(),
+    { storageFor: async (namespace: string) => ({ dir: `/tmp/remnic/${namespace}` }) },
+    () => {
+      const backend = new FakeBackend(false, [], { ensure: "present" });
+      backend.statusResult = null;
+      return backend;
+    },
+  );
+
+  const health = await router.healthForNamespace("shared");
+
+  assert.equal(health.pendingEmbeddings, null);
+  assert.equal(health.oldestPendingAgeMs, null);
+  assert.equal(health.embeddingBacklogThreshold, 5000);
+});
+
 test("healthForNamespace preserves cached missing collection state", async () => {
   const created: FakeBackend[] = [];
   const router = new NamespaceSearchRouter(
@@ -853,3 +913,17 @@ test("legacy default namespace root overfetches before filtering nested namespac
     ["main-a", "main-b"],
   );
 });
+
+test("normalizeQmdResultPath strips a real collection prefix but not a category dir (#2020)", () => {
+  // Genuine QMD collection prefix (qmd:// URI) is stripped.
+  assert.equal(
+    normalizeQmdResultPath("qmd://openclaw-engram/facts/a.md", "openclaw-engram"),
+    "facts/a.md",
+  );
+  // Plain relative hit is left untouched when its leading segment is a memory
+  // category dir that happens to equal the collection name (qmdCollection ==
+  // "facts"): stripping would corrupt "facts/a.md" -> "a.md" and break reads.
+  assert.equal(normalizeQmdResultPath("facts/a.md", "facts"), "facts/a.md");
+  // A non-category collection prefix on a plain hit is still stripped.
+  assert.equal(normalizeQmdResultPath("openclaw-engram/facts/a.md", "openclaw-engram"), "facts/a.md");
+});

package/src/namespaces/search.ts

@@ -8,6 +8,7 @@ import type {
 } from "../search/port.js";
 import { createSearchBackend } from "../search/factory.js";
 import { namespaceIdentityToken, normalizeNamespaceIdentity } from "./identity.js";
+import { ALL_CATEGORY_DIRS } from "../utils/category-dir.js";
 
 const NESTED_NAMESPACE_FILTER_OVERFETCH_FACTOR = 4;
 const NESTED_NAMESPACE_FILTER_OVERFETCH_MIN = 50;
@@ -64,6 +65,9 @@ export interface NamespaceSearchHealth {
   upgradeAvailable: boolean | null;
   doctorAvailable: boolean | null;
   daemonMode: boolean | null;
+  pendingEmbeddings: number | null;
+  oldestPendingAgeMs: number | null;
+  embeddingBacklogThreshold: number;
 }
 
 type NamespaceScopedSearchConfig = PluginConfig & {
@@ -89,6 +93,18 @@ export class NamespaceSearchRouter {
     return (await this.backendRecordFor(namespace)).collection;
   }
 
+  /**
+   * Resolve the search backend for a namespace (#2019). Returns `unknown` so
+   * callers (e.g. the prioritized-embed path) duck-type for capability like
+   * `embedFiles` rather than depending on a concrete QMD shape. Reuses the
+   * cached backend record, so this never re-creates a backend. Returns `null`
+   * when the namespace backend is unavailable.
+   */
+  async backendForNamespace(namespace: string): Promise<unknown> {
+    const record = await this.backendRecordFor(namespace);
+    return record.available ? record.backend : null;
+  }
+
   async searchAcrossNamespaces(options: {
     query: string;
     namespaces: string[];
@@ -165,7 +181,15 @@ export class NamespaceSearchRouter {
             );
             break;
         }
-        results = filterNamespaceSubtreeResults(record, results);
+        results = filterNamespaceSubtreeResults(record, results).map((result) => ({
+          ...result,
+          namespace,
+          // Resolve to an absolute path so the (namespace, path) identity is
+          // globally unique — same-relative-path hits from different namespaces
+          // stay distinct across every downstream consumer with no special
+          // handling. Display/citation surfaces relativize for portability (#2020).
+          path: resolveNamespaceResultPath(record.memoryDir, record.collection, result.path),
+        }));
         return { namespace, results };
       }),
     );
@@ -263,6 +287,14 @@ export class NamespaceSearchRouter {
           ? diagnosticBackend.getVersionStatus()
           : null;
       const daemonMode = daemonModeForBackend(diagnosticBackend);
+      const backendStatus =
+        "status" in diagnosticBackend &&
+        typeof diagnosticBackend.status === "function"
+          ? await Promise.race([
+              diagnosticBackend.status().catch(() => null),
+              new Promise<null>((resolve) => setTimeout(() => resolve(null), 5_000).unref?.()),
+            ])
+          : null;
       const collectionState =
         liveRecord?.collectionState === "missing"
           ? "missing"
@@ -280,6 +312,9 @@ export class NamespaceSearchRouter {
         upgradeAvailable: versionStatus?.upgradeAvailable ?? null,
         doctorAvailable: versionStatus?.capabilities?.doctor ?? null,
         daemonMode,
+        pendingEmbeddings: backendStatus?.pendingEmbeddings ?? null,
+        oldestPendingAgeMs: backendStatus?.oldestPendingAgeMs ?? null,
+        embeddingBacklogThreshold: this.config.qmdEmbeddingBacklogThreshold,
       };
     } finally {
       const dispose = (record.backend as { dispose?: () => void | Promise<void> }).dispose;
@@ -550,8 +585,7 @@ function pathIsInsideNamespaceSubtree(
   const relative = path.relative(namespacesRoot, candidate);
   return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative));
 }
-
-function normalizeQmdResultPath(resultPath: string, collection: string): string {
+export function normalizeQmdResultPath(resultPath: string, collection: string): string {
   let value = resultPath.trim();
   if (value.startsWith("qmd://")) {
     try {
@@ -568,13 +602,31 @@ function normalizeQmdResultPath(resultPath: string, collection: string): string
     }
   }
 
+  // Only strip a genuine QMD collection prefix — never when the collection name
+  // collides with a memory category dir (e.g. qmdCollection == "facts"), or a
+  // bare "facts/<id>.md" hit would be corrupted to "<id>.md" and fail namespace
+  // reads/access tracking (#2020).
   const collectionPrefix = `${collection}/`;
-  if (value.startsWith(collectionPrefix)) {
+  if (value.startsWith(collectionPrefix) && !ALL_CATEGORY_DIRS.includes(collection)) {
     value = value.slice(collectionPrefix.length);
   }
   return value;
 }
-
+function resolveNamespaceResultPath(
+  memoryDir: string,
+  collection: string,
+  resultPath: string,
+): string {
+  const normalized = normalizeQmdResultPath(resultPath, collection);
+  if (path.isAbsolute(normalized)) return normalized;
+  const root = path.resolve(memoryDir);
+  const resolved = path.resolve(root, normalized);
+  const relative = path.relative(root, resolved);
+  if (relative === "" || relative.startsWith("..") || path.isAbsolute(relative)) {
+    return resultPath;
+  }
+  return resolved;
+}
 function mergeNamespaceSearchResults(
   lists: Array<{ namespace: string; results: QmdSearchResult[] }>,
   maxResults: number,
@@ -586,12 +638,13 @@ function mergeNamespaceSearchResults(
       const key = `${namespace}\0${result.path || result.docid}`;
       const existing = merged.get(key);
       if (!existing) {
-        merged.set(key, result);
+        merged.set(key, { ...result, namespace });
         continue;
       }
       if (result.score > existing.score) {
         merged.set(key, {
           ...result,
+          namespace,
           snippet: existing.snippet || result.snippet || "",
         });
       }

package/src/namespaces/storage.ts

@@ -1,8 +1,9 @@
-import { resolveNamespaceCapabilities } from "../capabilities.js";
+import { resolveNamespaceCapabilities, resolveRecallAuxiliaryCapabilities } from "../capabilities.js";
 import path from "node:path";
 import { access, lstat, readdir } from "node:fs/promises";
 import { isSafeRouteNamespace } from "../routing/engine.js";
 import { StorageManager } from "../storage.js";
+import { keyring, secureStoreDir } from "../secure-store/index.js";
 import type { PluginConfig } from "../types.js";
 import { ALL_CATEGORY_DIRS } from "../utils/category-dir.js";
 import { namespaceIdentityToken, normalizeNamespaceIdentity } from "./identity.js";
@@ -242,6 +243,9 @@ export class NamespaceStorageRouter {
   // default name routes the default namespace to a tokenized non-default root.
   private readonly defaultNamespaceIdentity: string;
 
+  /** #2019: hook fired when a new namespace storage is created via storageFor. */
+  onStorageCreated?: (storage: StorageManager, namespace: string) => void;
+
   constructor(
     private readonly config: PluginConfig,
     private readonly hooks: NamespaceStorageRouterHooks = {},
@@ -324,7 +328,18 @@ export class NamespaceStorageRouter {
     // #1579: apply the tombstone non-resurrection config so every namespace
     // storage enforces the invariant at its own writeMemory chokepoint.
     this.applyTombstonesConfig(sm, ns);
+    // #2033: install the secure-store context on router-created stores. The
+    // orchestrator keys ONLY its primary default-namespace storage; without
+    // this, a store resolved through the router reads isSecureStoreUnlocked()
+    // === false even after the operator unlocked the keyring, so secure-store
+    // recovery paths (e.g. the CLI lifecycle-ledger rebuild) refuse an encrypted
+    // ledger they could actually rewrite, and namespace writes could silently
+    // fall back to plaintext. Mirror the orchestrator's setup exactly.
+    this.applySecureStoreConfig(sm);
     this.cache.set(ns, sm);
+    // #2019: let the orchestrator wire write-triggered prioritized embedding on
+    // every router-created namespace storage, not just the primary default store.
+    this.onStorageCreated?.(sm, ns);
     this.notifyResolved(ns, root);
     return sm;
   }
@@ -453,6 +468,25 @@ export class NamespaceStorageRouter {
     sm.setTombstonesConfig({ ...this.tombstonesGlobalConfig, namespace });
   }
 
+  /**
+   * Install the at-rest secure-store context on a router-created StorageManager
+   * (#2033). Mirrors the orchestrator's primary-storage setup: mark the store
+   * required so a locked store throws instead of silently writing plaintext, and
+   * install the master key when the process-local keyring already holds it (the
+   * operator unlocked). The secure store is one key per memory ROOT (single
+   * header + per-file path-bound AAD), so every namespace store keys off the
+   * root keyring id — exactly the id the orchestrator and `secure-store unlock`
+   * register under. No-op when secure-store is disabled.
+   */
+  private applySecureStoreConfig(sm: StorageManager): void {
+    if (!resolveRecallAuxiliaryCapabilities(this.config).secureStore) return;
+    sm.setSecureStoreRequired(true);
+    const existingKey = keyring.getKey(secureStoreDir(this.config.memoryDir));
+    if (existingKey) {
+      sm.setSecureStoreKey(existingKey, this.config.secureStoreEncryptOnWrite);
+    }
+  }
+
   /**
    * Post-write catalog touch (issue #1522 chokepoint). Called by every
    * StorageManager's post-write hook AFTER a successful write. Best-effort
@@ -537,4 +571,12 @@ export class NamespaceStorageRouter {
       await Promise.allSettled([...this.pendingResolveHooks]);
     }
   }
+
+  /**
+   * #2019: iterate the currently cached router storages with their namespace so
+   * callers can wire hooks on stores resolved before the hook was installed.
+   */
+  forEachCachedStorage(fn: (storage: StorageManager, namespace: string) => void): void {
+    for (const [namespace, storage] of this.cache) fn(storage, namespace);
+  }
 }

package/src/offline-sync-exclude-globs.ts

@@ -0,0 +1,58 @@
+// Issue #1786: node-local runtime state that each node rebuilds from synced
+// records. Pushing these wastes bandwidth, corrupts the remote's live state
+// dir, and trips the large-file push retry loop on a live SQLite database.
+// Extracted from offline-sync.ts (issue #1995) so the god-file line-count
+// ratchet does not grow when the list gains entries; behavior is unchanged.
+/**
+ * Directory-name prefix for the transient plaintext staging dir the CLI offline
+ * decrypt path (`offline-storage-io.ts`) creates under the memory root when it
+ * must materialize a decrypted secure-store file before hashing/pushing (#2033
+ * P1). Shared here so the hard-exclude glob below and the CLI's `mkdtemp` prefix
+ * can never drift apart: a crash-orphaned staging dir must NEVER be enumerated
+ * into an offline snapshot, or decrypted secure-store plaintext would leak into
+ * the remote push.
+ */
+export const OFFLINE_DECRYPT_STAGING_DIR_PREFIX = ".remnic-offline-decrypt-";
+
+export const DEFAULT_OFFLINE_SYNC_EXCLUDE_GLOBS: readonly string[] = [
+  // Leading `**/` matches zero or more segments, so each pattern covers both
+  // the root `state/` dir AND per-namespace `namespaces/<ns>/state/` dirs
+  // (Cursor review on PR #1793: multi-namespace deployments previously kept
+  // pushing their namespaced live sqlite files).
+  "**/state/*.sqlite",
+  "**/state/*.sqlite-*",
+  "**/state/index_tags.json",
+  "**/state/entity-mention-index.json",
+  "**/state/memory-governance/runs/**",
+  // Rotated recall-impression archives (issue #1910). The active
+  // recall_impressions.jsonl stays remote-authoritative; only the .1..N
+  // archives and the .lock are node-local and never pushed/hashed.
+  "**/state/recall_impressions.jsonl.*",
+  // Durable recall-impression pending spill directory (issue #2033). Node-local:
+  // an impression spills a per-event file here when its rotation lock cannot be
+  // acquired, and the next lock holder folds them back into the synced active
+  // recall_impressions.jsonl. The `.*` glob above matches the directory name but
+  // not its children, so exclude the contents explicitly.
+  "**/state/recall_impressions.jsonl.pending.d/**",
+  // Durable lifecycle-append pending spill directory (issue #2033). Node-local:
+  // an append spills a per-event file here when it cannot get the ledger lock,
+  // and the next lock holder folds them back into the synced
+  // memory-lifecycle-ledger.jsonl. Pushing them would duplicate rows remotely.
+  // The offline-sync snapshot entrypoints drain this queue first (access-service
+  // `drainPendingLifecycleForSync` -> `StorageManager.drainPendingMemoryLifecycleEventsForSync`),
+  // aborting the snapshot when durable rows cannot be folded, so excluding the
+  // dir never silently drops append-only lifecycle history (promotions/imports).
+  "**/state/memory-lifecycle-ledger.jsonl.pending.d/**",
+  // The active lifecycle ledger lock is node-local and must never be
+  // transferred to another node during an offline snapshot.
+  "**/state/memory-lifecycle-ledger.jsonl.lock",
+  // Transient plaintext decrypt staging (issue #2033 P1). The CLI offline
+  // decrypt path materializes a decrypted secure-store file under an owner-only
+  // `<OFFLINE_DECRYPT_STAGING_DIR_PREFIX>*` dir before hashing/pushing, removing
+  // it in a `finally`. A hard crash between create and cleanup would otherwise
+  // let a later snapshot walk `<prefix>*/content` and push decrypted secure-store
+  // plaintext to the remote. Exclude the dir and its contents unconditionally so
+  // a crash-orphan can never be enumerated into a snapshot; the leading `**/`
+  // matches the root-level dir (zero segments) and any nested placement.
+  "**/.remnic-offline-decrypt-*/**",
+];