@remnic/core 9.3.647 → 9.3.648

package/src/namespaces/search.test.ts

@@ -1,20 +1,42 @@
 import assert from "node:assert/strict";
 import test from "node:test";
 import { NamespaceSearchRouter } from "./search.js";
-import type { SearchBackend } from "../search/port.js";
+import type {
+  SearchBackend,
+  SearchExecutionOptions,
+  SearchQueryOptions,
+} from "../search/port.js";
 import type { PluginConfig, QmdSearchResult } from "../types.js";
 
+type CollectionState = "present" | "missing" | "unknown" | "skipped";
+
 class FakeBackend implements SearchBackend {
   updates = 0;
-  calls: Array<{ method: string; collection: string | undefined }> = [];
+  calls: Array<{
+    method: string;
+    collection: string | undefined;
+    maxResults: number | undefined;
+  }> = [];
   ensureSignals: Array<AbortSignal | undefined> = [];
   ensureCollections: Array<string | undefined> = [];
+  checkSignals: Array<AbortSignal | undefined> = [];
+  checkCollections: Array<string | undefined> = [];
 
   constructor(
     private readonly globalUpdate: boolean,
     private readonly results: QmdSearchResult[] = [],
+    private readonly collectionStates: {
+      check?: CollectionState;
+      ensure?: CollectionState;
+    } = {},
   ) {}
 
+  private limitedResults(maxResults: number | undefined): QmdSearchResult[] {
+    return typeof maxResults === "number"
+      ? this.results.slice(0, maxResults)
+      : this.results;
+  }
+
   async probe(): Promise<boolean> {
     return true;
   }
@@ -27,28 +49,53 @@ class FakeBackend implements SearchBackend {
     return "fake";
   }
 
-  async search(_query?: string, collection?: string): Promise<QmdSearchResult[]> {
-    this.calls.push({ method: "search", collection });
-    return this.results;
+  async search(
+    _query: string,
+    collection?: string,
+    maxResults?: number,
+    _options?: SearchQueryOptions,
+    _execution?: SearchExecutionOptions,
+  ): Promise<QmdSearchResult[]> {
+    this.calls.push({ method: "search", collection, maxResults });
+    return this.limitedResults(maxResults);
   }
 
-  async searchGlobal(): Promise<QmdSearchResult[]> {
-    return [];
+  async searchGlobal(
+    _query: string,
+    maxResults?: number,
+    _execution?: SearchExecutionOptions,
+  ): Promise<QmdSearchResult[]> {
+    return this.limitedResults(maxResults);
   }
 
-  async bm25Search(_query?: string, collection?: string): Promise<QmdSearchResult[]> {
-    this.calls.push({ method: "bm25", collection });
-    return [];
+  async bm25Search(
+    _query: string,
+    collection?: string,
+    maxResults?: number,
+    _execution?: SearchExecutionOptions,
+  ): Promise<QmdSearchResult[]> {
+    this.calls.push({ method: "bm25", collection, maxResults });
+    return this.limitedResults(maxResults);
   }
 
-  async vectorSearch(_query?: string, collection?: string): Promise<QmdSearchResult[]> {
-    this.calls.push({ method: "vector", collection });
-    return [];
+  async vectorSearch(
+    _query: string,
+    collection?: string,
+    maxResults?: number,
+    _execution?: SearchExecutionOptions,
+  ): Promise<QmdSearchResult[]> {
+    this.calls.push({ method: "vector", collection, maxResults });
+    return this.limitedResults(maxResults);
   }
 
-  async hybridSearch(_query?: string, collection?: string): Promise<QmdSearchResult[]> {
-    this.calls.push({ method: "hybrid", collection });
-    return [];
+  async hybridSearch(
+    _query: string,
+    collection?: string,
+    maxResults?: number,
+    _execution?: SearchExecutionOptions,
+  ): Promise<QmdSearchResult[]> {
+    this.calls.push({ method: "hybrid", collection, maxResults });
+    return this.limitedResults(maxResults);
   }
 
   async update(): Promise<void> {
@@ -69,7 +116,7 @@ class FakeBackend implements SearchBackend {
     _memoryDir?: string,
     collectionOrExecution?: string | { signal?: AbortSignal },
     execution?: { signal?: AbortSignal },
-  ): Promise<"present"> {
+  ): Promise<CollectionState> {
     const collection = typeof collectionOrExecution === "string"
       ? collectionOrExecution
       : undefined;
@@ -78,12 +125,29 @@ class FakeBackend implements SearchBackend {
       : collectionOrExecution ?? execution;
     this.ensureCollections.push(collection);
     this.ensureSignals.push(effectiveExecution?.signal);
-    return "present";
+    return this.collectionStates.ensure ?? "present";
+  }
+
+  async checkCollection(
+    collectionOrExecution?: string | { signal?: AbortSignal },
+    execution?: { signal?: AbortSignal },
+  ): Promise<CollectionState> {
+    const collection = typeof collectionOrExecution === "string"
+      ? collectionOrExecution
+      : undefined;
+    const effectiveExecution = typeof collectionOrExecution === "string"
+      ? execution
+      : collectionOrExecution ?? execution;
+    this.checkCollections.push(collection);
+    this.checkSignals.push(effectiveExecution?.signal);
+    return this.collectionStates.check ?? "present";
   }
 }
 
 function config(): PluginConfig {
   return {
+    memoryDir: "/tmp/remnic",
+    namespacesEnabled: true,
     qmdCollection: "openclaw-engram",
     defaultNamespace: "main",
     qmdMaxResults: 10,
@@ -205,3 +269,139 @@ test("ensureNamespaceCollection forwards abort signals to backend collection che
   assert.deepEqual(backend.ensureSignals, [controller.signal]);
   assert.deepEqual(backend.ensureCollections, ["openclaw-engram--ns-6d61696e"]);
 });
+
+test("legacy default namespace root checks collection without auto-creating broad root", async () => {
+  const backend = new FakeBackend(false);
+  const router = new NamespaceSearchRouter(
+    config(),
+    { storageFor: async () => ({ dir: "/tmp/remnic" }) },
+    () => backend,
+  );
+  const controller = new AbortController();
+
+  const state = await router.ensureNamespaceCollection("main", {
+    signal: controller.signal,
+  });
+
+  assert.equal(state, "present");
+  assert.deepEqual(backend.checkSignals, [controller.signal]);
+  assert.deepEqual(backend.checkCollections, ["openclaw-engram"]);
+  assert.deepEqual(backend.ensureCollections, []);
+});
+
+test("legacy default namespace root fail-opens missing guarded collections", async () => {
+  const backend = new FakeBackend(false, [], { check: "missing" });
+  const router = new NamespaceSearchRouter(
+    config(),
+    { storageFor: async () => ({ dir: "/tmp/remnic" }) },
+    () => backend,
+  );
+
+  const state = await router.ensureNamespaceCollection("main");
+
+  assert.equal(state, "unknown");
+  assert.deepEqual(backend.checkCollections, ["openclaw-engram"]);
+  assert.deepEqual(backend.ensureCollections, []);
+});
+
+test("legacy default namespace root filters nested namespace search results", async () => {
+  const router = new NamespaceSearchRouter(
+    config(),
+    { storageFor: async () => ({ dir: "/tmp/remnic" }) },
+    () => new FakeBackend(false, [
+      {
+        path: "/tmp/remnic/facts/main.md",
+        docid: "main",
+        score: 0.9,
+        snippet: "main",
+      },
+      {
+        path: "/tmp/remnic/namespaces/shared/facts/shared.md",
+        docid: "shared",
+        score: 0.95,
+        snippet: "shared",
+      },
+      {
+        path: "namespaces/project/facts/project.md",
+        docid: "project",
+        score: 0.8,
+        snippet: "project",
+      },
+      {
+        path: "qmd://openclaw-engram/facts/qmd-main.md",
+        docid: "qmd-main",
+        score: 0.85,
+        snippet: "qmd-main",
+      },
+      {
+        path: "qmd://openclaw-engram/namespaces/uri/facts/uri.md",
+        docid: "uri",
+        score: 0.99,
+        snippet: "uri",
+      },
+      {
+        path: "openclaw-engram/namespaces/prefix/facts/prefix.md",
+        docid: "prefix",
+        score: 0.98,
+        snippet: "prefix",
+      },
+    ]),
+  );
+
+  const results = await router.searchAcrossNamespaces({
+    query: "a",
+    namespaces: ["main"],
+    maxResults: 10,
+  });
+
+  assert.deepEqual(
+    results.map((result) => result.docid),
+    ["main", "qmd-main"],
+  );
+});
+
+test("legacy default namespace root overfetches before filtering nested namespace results", async () => {
+  const backend = new FakeBackend(false, [
+    {
+      path: "/tmp/remnic/namespaces/shared/facts/shared.md",
+      docid: "shared",
+      score: 0.99,
+      snippet: "shared",
+    },
+    {
+      path: "qmd://openclaw-engram/namespaces/project/facts/project.md",
+      docid: "project",
+      score: 0.98,
+      snippet: "project",
+    },
+    {
+      path: "/tmp/remnic/facts/main-a.md",
+      docid: "main-a",
+      score: 0.9,
+      snippet: "main-a",
+    },
+    {
+      path: "/tmp/remnic/facts/main-b.md",
+      docid: "main-b",
+      score: 0.8,
+      snippet: "main-b",
+    },
+  ]);
+  const router = new NamespaceSearchRouter(
+    config(),
+    { storageFor: async () => ({ dir: "/tmp/remnic" }) },
+    () => backend,
+  );
+
+  const results = await router.searchAcrossNamespaces({
+    query: "a",
+    namespaces: ["main"],
+    maxResults: 2,
+  });
+
+  assert.equal(backend.calls[0]?.maxResults, 50);
+  assert.deepEqual(
+    results.map((result) => result.docid),
+    ["main-a", "main-b"],
+  );
+});

package/src/namespaces/search.ts

@@ -1,8 +1,16 @@
+import path from "node:path";
 import type { PluginConfig, QmdSearchResult } from "../types.js";
-import type { SearchBackend, SearchExecutionOptions, SearchQueryOptions } from "../search/port.js";
+import type {
+  SearchBackend,
+  SearchExecutionOptions,
+  SearchQueryOptions,
+} from "../search/port.js";
 import { createSearchBackend } from "../search/factory.js";
 import { namespaceIdentityToken, normalizeNamespaceIdentity } from "./identity.js";
 
+const NESTED_NAMESPACE_FILTER_OVERFETCH_FACTOR = 4;
+const NESTED_NAMESPACE_FILTER_OVERFETCH_MIN = 50;
+
 export function namespaceCollectionName(
   baseCollection: string,
   namespace: string,
@@ -32,9 +40,12 @@ type NamespaceBackendRecord = {
   collection: string;
   memoryDir: string;
   available: boolean;
-  collectionState: "present" | "missing" | "unknown" | "skipped";
+  collectionState: CollectionState;
+  filtersNestedNamespaces: boolean;
 };
 
+type CollectionState = "present" | "missing" | "unknown" | "skipped";
+
 type NamespaceScopedSearchConfig = PluginConfig & {
   hostEmbeddingProviderScope?: string;
 };
@@ -75,27 +86,44 @@ export class NamespaceSearchRouter {
         if (!record.available || record.collectionState === "missing") {
           return { namespace, results: [] as QmdSearchResult[] };
         }
+        const backendLimit = backendSearchLimit(record, maxResults);
         let results: QmdSearchResult[];
         switch (method) {
           case "hybrid":
-            results = await record.backend.hybridSearch(query, record.collection, maxResults, options.execution);
+            results = await record.backend.hybridSearch(
+              query,
+              record.collection,
+              backendLimit,
+              options.execution,
+            );
             break;
           case "bm25":
-            results = await record.backend.bm25Search(query, record.collection, maxResults, options.execution);
+            results = await record.backend.bm25Search(
+              query,
+              record.collection,
+              backendLimit,
+              options.execution,
+            );
             break;
           case "vector":
-            results = await record.backend.vectorSearch(query, record.collection, maxResults, options.execution);
+            results = await record.backend.vectorSearch(
+              query,
+              record.collection,
+              backendLimit,
+              options.execution,
+            );
             break;
           default:
             results = await record.backend.search(
               query,
               record.collection,
-              maxResults,
+              backendLimit,
               options.searchOptions,
               options.execution,
             );
             break;
         }
+        results = filterNamespaceSubtreeResults(record, results);
         return { namespace, results };
       }),
     );
@@ -187,6 +215,8 @@ export class NamespaceSearchRouter {
       const storage = await this.storageRouter.storageFor(key);
       const useLegacyDefaultCollection =
         key === this.config.defaultNamespace && storage.dir === this.config.memoryDir;
+      const filtersNestedNamespaces =
+        this.config.namespacesEnabled === true && useLegacyDefaultCollection;
       const rootHostEmbeddingScope =
         (this.config as NamespaceScopedSearchConfig).hostEmbeddingProviderScope ??
         this.config.memoryDir;
@@ -203,7 +233,10 @@ export class NamespaceSearchRouter {
       const backend = this.createBackend(scopedConfig);
       const available = await backend.probe().catch(() => false);
       const collectionState = available
-        ? await backend.ensureCollection(storage.dir, scopedConfig.qmdCollection, execution).catch(() => "unknown" as const)
+        ? await this.collectionStateForBackend(backend, storage.dir, scopedConfig.qmdCollection, {
+          skipAutoCreate: filtersNestedNamespaces,
+          execution,
+        })
         : "unknown";
       return {
         backend,
@@ -211,12 +244,94 @@ export class NamespaceSearchRouter {
         memoryDir: storage.dir,
         available,
         collectionState,
+        filtersNestedNamespaces,
       };
     })();
 
     this.cache.set(key, pending);
     return await pending;
   }
+
+  private async collectionStateForBackend(
+    backend: SearchBackend,
+    memoryDir: string,
+    collection: string,
+    options: {
+      skipAutoCreate: boolean;
+      execution?: SearchExecutionOptions;
+    },
+  ): Promise<CollectionState> {
+    if (options.skipAutoCreate) {
+      if (!backend.checkCollection) return "unknown";
+      const collectionState = await backend
+        .checkCollection(collection, options.execution)
+        .catch(() => "unknown" as const);
+      return collectionState === "missing" ? "unknown" : collectionState;
+    }
+    return await backend.ensureCollection(memoryDir, collection, options.execution).catch(() => "unknown" as const);
+  }
+}
+
+function filterNamespaceSubtreeResults(
+  record: NamespaceBackendRecord,
+  results: QmdSearchResult[],
+): QmdSearchResult[] {
+  if (!record.filtersNestedNamespaces) return results;
+  return results.filter((result) =>
+    !pathIsInsideNamespaceSubtree(record.memoryDir, record.collection, result.path)
+  );
+}
+
+function backendSearchLimit(
+  record: NamespaceBackendRecord,
+  maxResults: number,
+): number {
+  if (!record.filtersNestedNamespaces) return maxResults;
+  return Math.max(
+    maxResults,
+    maxResults * NESTED_NAMESPACE_FILTER_OVERFETCH_FACTOR,
+    NESTED_NAMESPACE_FILTER_OVERFETCH_MIN,
+  );
+}
+
+function pathIsInsideNamespaceSubtree(
+  memoryDir: string,
+  collection: string,
+  resultPath: string | undefined,
+): boolean {
+  if (!resultPath) return false;
+  const normalizedResultPath = normalizeQmdResultPath(resultPath, collection);
+  const memoryRoot = path.resolve(memoryDir);
+  const namespacesRoot = path.join(memoryRoot, "namespaces");
+  const candidate = path.isAbsolute(normalizedResultPath)
+    ? path.normalize(normalizedResultPath)
+    : path.resolve(memoryRoot, normalizedResultPath);
+  const relative = path.relative(namespacesRoot, candidate);
+  return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative));
+}
+
+function normalizeQmdResultPath(resultPath: string, collection: string): string {
+  let value = resultPath.trim();
+  if (value.startsWith("qmd://")) {
+    try {
+      const parsed = new URL(value);
+      if (parsed.protocol === "qmd:" && parsed.hostname === collection) {
+        value = decodeURIComponent(parsed.pathname.replace(/^\/+/, ""));
+      }
+    } catch {
+      const remainder = value.slice("qmd://".length);
+      const slashIndex = remainder.indexOf("/");
+      if (slashIndex !== -1) {
+        value = remainder.slice(slashIndex + 1);
+      }
+    }
+  }
+
+  const collectionPrefix = `${collection}/`;
+  if (value.startsWith(collectionPrefix)) {
+    value = value.slice(collectionPrefix.length);
+  }
+  return value;
 }
 
 function mergeNamespaceSearchResults(

package/src/qmd-client.test.ts

@@ -46,7 +46,11 @@ test("QmdClient rechecks daemon availability before returning unavailable", asyn
 
 type SubprocessInternals = {
   available: boolean;
-  runQmdCommand: (args: string[]) => Promise<{ stdout: string; stderr: string }>;
+  runQmdCommand: (
+    args: string[],
+    timeoutMs?: number,
+    signal?: AbortSignal,
+  ) => Promise<{ stdout: string; stderr: string }>;
   searchViaSubprocess: (
     query: string,
     collection: string,
@@ -66,6 +70,75 @@ function captureSubprocessArgs(client: QmdClient): string[][] {
   return calls;
 }
 
+test("ensureCollection treats cancelled auto-create as unknown", async () => {
+  const client = new QmdClient("memories", 3, {});
+  const internals = client as unknown as SubprocessInternals & {
+    daemonAvailable: boolean;
+  };
+  const controller = new AbortController();
+  const calls: string[][] = [];
+
+  internals.available = true;
+  internals.daemonAvailable = false;
+  internals.runQmdCommand = async (args, _timeoutMs, signal) => {
+    calls.push(args);
+    if (args[0] === "collection" && args[1] === "list") {
+      return { stdout: "", stderr: "" };
+    }
+    if (args[0] === "collection" && args[1] === "add") {
+      assert.equal(signal, controller.signal);
+      controller.abort();
+      throw new Error("startup timeout aborted collection add");
+    }
+    throw new Error(`unexpected qmd command: ${args.join(" ")}`);
+  };
+
+  const result = await client.ensureCollection("/tmp/remnic-memory", "memories", {
+    signal: controller.signal,
+  });
+
+  assert.equal(result, "unknown");
+  assert.deepEqual(calls, [
+    ["collection", "list"],
+    ["collection", "add", "/tmp/remnic-memory", "--name", "memories"],
+  ]);
+});
+
+test("ensureCollection rechecks collection state after auto-create failure", async () => {
+  const client = new QmdClient("memories", 3, {});
+  const internals = client as unknown as SubprocessInternals & {
+    daemonAvailable: boolean;
+  };
+  const calls: string[][] = [];
+  let listCount = 0;
+
+  internals.available = true;
+  internals.daemonAvailable = false;
+  internals.runQmdCommand = async (args) => {
+    calls.push(args);
+    if (args[0] === "collection" && args[1] === "list") {
+      listCount += 1;
+      return {
+        stdout: listCount === 1 ? "" : "memories (qmd://memories/)",
+        stderr: "",
+      };
+    }
+    if (args[0] === "collection" && args[1] === "add") {
+      throw new Error("qmd collection add timed out after indexing started");
+    }
+    throw new Error(`unexpected qmd command: ${args.join(" ")}`);
+  };
+
+  const result = await client.ensureCollection("/tmp/remnic-memory", "memories");
+
+  assert.equal(result, "present");
+  assert.deepEqual(calls, [
+    ["collection", "list"],
+    ["collection", "add", "/tmp/remnic-memory", "--name", "memories"],
+    ["collection", "list"],
+  ]);
+});
+
 test("subprocess fallback defaults to `qmd query` for scoped and global recall", async () => {
   const client = new QmdClient("memories", 3, {});
   const calls = captureSubprocessArgs(client);

package/src/qmd.ts

@@ -2700,8 +2700,7 @@ export class QmdClient implements SearchBackend {
     }
   }
 
-  async ensureCollection(
-    memoryDir: string,
+  async checkCollection(
     collectionOrExecution?: string | SearchExecutionOptions,
     execution?: SearchExecutionOptions,
   ): Promise<"present" | "missing" | "unknown" | "skipped"> {
@@ -2723,6 +2722,7 @@ export class QmdClient implements SearchBackend {
       if (collectionRegex.test(stdout)) {
         return "present";
       }
+      return "missing";
     } catch (err) {
       // Treat command/probe failures as unknown so callers do not disable features
       // permanently after a transient CLI or daemon hiccup.
@@ -2731,12 +2731,58 @@ export class QmdClient implements SearchBackend {
       );
       return "unknown";
     }
+  }
 
-    log.info(
-      `QMD collection "${targetCollection}" not found. ` +
-        `Add it to ~/.config/qmd/index.yml pointing at ${memoryDir}`,
+  async ensureCollection(
+    memoryDir: string,
+    collectionOrExecution?: string | SearchExecutionOptions,
+    execution?: SearchExecutionOptions,
+  ): Promise<"present" | "missing" | "unknown" | "skipped"> {
+    const { collection, execution: effectiveExecution } = resolveEnsureCollectionArgs(
+      collectionOrExecution,
+      execution,
     );
-    return "missing";
+    const targetCollection = collection ?? this.collection;
+    const collectionState = await this.checkCollection(targetCollection, effectiveExecution);
+    if (collectionState !== "missing") return collectionState;
+
+    try {
+      await this.runQmdCommand(
+        ["collection", "add", memoryDir, "--name", targetCollection],
+        QMD_TIMEOUT_MS,
+        effectiveExecution?.signal,
+      );
+      log.info(
+        `QMD collection "${targetCollection}" auto-created at ${memoryDir}`,
+      );
+      return "present";
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err);
+      if (isCallerCancellation(err, effectiveExecution?.signal)) {
+        log.debug(
+          `QMD collection auto-create for "${targetCollection}" was cancelled; keeping collection state unknown`,
+        );
+        return "unknown";
+      }
+      const postCreateState = await this.checkCollection(targetCollection, effectiveExecution);
+      if (postCreateState === "present") {
+        log.info(
+          `QMD collection "${targetCollection}" is present after auto-create failure; continuing`,
+        );
+        return "present";
+      }
+      if (/\balready exists\b|\bexists already\b/i.test(msg)) {
+        log.info(
+          `QMD collection "${targetCollection}" already exists after concurrent auto-create; continuing`,
+        );
+        return "present";
+      }
+      if (postCreateState !== "missing") return postCreateState;
+      log.warn(
+        `QMD collection "${targetCollection}" not found and auto-create failed: ${msg}`,
+      );
+      return "missing";
+    }
   }
 }
 

package/src/search/port.ts

@@ -93,6 +93,15 @@ export interface SearchBackend {
   embedCollection(collection: string): Promise<void>;
 
   // ── Collection management ──
+  /**
+   * Optional non-mutating collection probe. Backends that can distinguish a
+   * missing collection from a transient probe failure should implement this so
+   * callers can avoid auto-creating collections in unsafe layouts.
+   */
+  checkCollection?(
+    collectionOrExecution?: string | SearchExecutionOptions,
+    execution?: SearchExecutionOptions,
+  ): Promise<"present" | "missing" | "unknown" | "skipped">;
   ensureCollection(
     memoryDir: string,
     execution?: SearchExecutionOptions,