@remnic/core 10.0.84 → 10.0.86

package/src/namespaces/search.test.ts

@@ -3,12 +3,14 @@ import test from "node:test";
 import os from "node:os";
 import path from "node:path";
 import { mkdtemp, rm } from "node:fs/promises";
-import { NamespaceSearchRouter, normalizeQmdResultPath } from "./search.js";
+import { setImmediate } from "node:timers/promises";
+import { NamespaceSearchRouter, NAMESPACE_SEARCH_LEG_TIMEOUT_MS, normalizeQmdResultPath } from "./search.js";
 import { MaintenanceScheduler } from "../orchestration/maintenance.js";
 import type { NamespaceCatalog } from "./catalog.js";
 import type {
   SearchBackend,
   SearchBackendStatus,
+  SearchDegradation,
   SearchExecutionOptions,
   SearchQueryOptions,
 } from "../search/port.js";
@@ -432,6 +434,144 @@ test("searchAcrossNamespaces preserves same path results from distinct namespace
   ]);
 });
 
+class StalledBackend extends FakeBackend {
+  override async search(): Promise<QmdSearchResult[]> {
+    return new Promise<QmdSearchResult[]>(() => {});
+  }
+}
+
+test("searchAcrossNamespaces degrades only the namespace whose leg never settles", async (t) => {
+  const degradations: SearchDegradation[] = [];
+  const router = new NamespaceSearchRouter(
+    config(),
+    { storageFor: async (namespace: string) => ({ dir: `/tmp/remnic/${namespace}` }) },
+    (scopedConfig) =>
+      scopedConfig.memoryDir.endsWith("/stuck")
+        ? new StalledBackend(false)
+        : new FakeBackend(false, [
+            { path: "facts/a.md", docid: "a", score: 0.9, snippet: "main" },
+          ]),
+  );
+  t.mock.timers.enable({ apis: ["setTimeout"] });
+  const call = router.searchAcrossNamespaces({
+    query: "a",
+    namespaces: ["main", "stuck"],
+    maxResults: 10,
+    execution: { onDegradation: (degradation) => degradations.push(degradation) },
+  });
+  let outcome: "settled" | "rejected" | "stalled" | undefined;
+  const guard = new Promise<"stalled">((resolve) => {
+    setTimeout(() => resolve("stalled"), 60_000);
+  });
+  void Promise.race([
+    call.then(() => "settled" as const, () => "rejected" as const),
+    guard,
+  ]).then((value) => {
+    outcome = value;
+  });
+  await setImmediate();
+  t.mock.timers.tick(NAMESPACE_SEARCH_LEG_TIMEOUT_MS);
+  await setImmediate();
+  if (outcome === undefined) {
+    t.mock.timers.tick(60_000);
+    await setImmediate();
+  }
+  assert.equal(outcome, "settled", "a stalled namespace leg must not hold the whole search");
+  const results = await call;
+  assert.deepEqual(results.map((result) => result.snippet), ["main"]);
+  assert.equal(degradations.length, 1);
+  assert.equal(degradations[0].backend, "qmd");
+  assert.equal(degradations[0].code, "deadline_exceeded");
+  assert.match(degradations[0].detail ?? "", /stuck/);
+});
+
+test("searchAcrossNamespaces still rejects when the caller aborts", async () => {
+  const degradations: SearchDegradation[] = [];
+  const router = new NamespaceSearchRouter(
+    config(),
+    { storageFor: async (namespace: string) => ({ dir: `/tmp/remnic/${namespace}` }) },
+    () => new StalledBackend(false),
+  );
+  const caller = new AbortController();
+  const call = router.searchAcrossNamespaces({
+    query: "a",
+    namespaces: ["main"],
+    maxResults: 10,
+    execution: {
+      signal: caller.signal,
+      onDegradation: (degradation) => degradations.push(degradation),
+    },
+  });
+  caller.abort();
+  await assert.rejects(call, { message: "operation aborted" });
+  assert.equal(degradations.length, 0, "a caller abort is not a backend degradation");
+});
+
+test("searchAcrossNamespaces propagates backend errors instead of masking them", async () => {
+  const degradations: SearchDegradation[] = [];
+  class ThrowingBackend extends FakeBackend {
+    override async search(): Promise<QmdSearchResult[]> {
+      throw new Error("boom");
+    }
+  }
+  const router = new NamespaceSearchRouter(
+    config(),
+    { storageFor: async (namespace: string) => ({ dir: `/tmp/remnic/${namespace}` }) },
+    () => new ThrowingBackend(false),
+  );
+  const call = router.searchAcrossNamespaces({
+    query: "a",
+    namespaces: ["main"],
+    maxResults: 10,
+    execution: { onDegradation: (degradation) => degradations.push(degradation) },
+  });
+  await assert.rejects(call, { message: "boom" });
+  assert.equal(degradations.length, 0, "a backend error is not a deadline");
+});
+
+test("searchAcrossNamespaces hands each leg its deadline signal", async (t) => {
+  const signals = new Map<string, AbortSignal | undefined>();
+  const degradations: SearchDegradation[] = [];
+  class RecordingBackend extends FakeBackend {
+    constructor(private readonly name: string) {
+      super(false);
+    }
+    override async search(
+      _query: string,
+      _collection?: string,
+      _maxResults?: number,
+      _options?: SearchQueryOptions,
+      execution?: SearchExecutionOptions,
+    ): Promise<QmdSearchResult[]> {
+      signals.set(this.name, execution?.signal);
+      if (this.name === "stuck") return new Promise<QmdSearchResult[]>(() => {});
+      return [{ path: "facts/a.md", docid: "a", score: 0.9, snippet: this.name }];
+    }
+  }
+  const router = new NamespaceSearchRouter(
+    config(),
+    { storageFor: async (namespace: string) => ({ dir: `/tmp/remnic/${namespace}` }) },
+    (scopedConfig) => new RecordingBackend(scopedConfig.memoryDir.endsWith("/stuck") ? "stuck" : "healthy"),
+  );
+  t.mock.timers.enable({ apis: ["setTimeout"] });
+  const call = router.searchAcrossNamespaces({
+    query: "a",
+    namespaces: ["healthy", "stuck"],
+    maxResults: 10,
+    execution: { onDegradation: (degradation) => degradations.push(degradation) },
+  });
+  await setImmediate();
+  t.mock.timers.tick(NAMESPACE_SEARCH_LEG_TIMEOUT_MS);
+  await setImmediate();
+  const results = await call;
+  assert.deepEqual(results.map((result) => result.snippet), ["healthy"]);
+  assert.equal(signals.get("stuck")?.aborted, true, "the stuck leg's backend must see its deadline abort");
+  assert.equal(signals.get("healthy")?.aborted, false, "a finished leg's timer must be cleared");
+  const deadlineCodes = degradations.filter((degradation) => degradation.code === "deadline_exceeded");
+  assert.equal(deadlineCodes.length, 1);
+  assert.match(deadlineCodes[0].detail ?? "", /stuck/);
+});
+
 test("searchAcrossNamespaces passes scoped collection to backend search methods", async () => {
   const created: FakeBackend[] = [];
   const router = new NamespaceSearchRouter(
@@ -831,6 +971,345 @@ test("healthForNamespace stops waiting when namespace availability probe aborts"
   assert.equal(backend.disposed, 1);
 });
 
+test("healthForNamespace reuses a completed result within the TTL without re-probing", async (t) => {
+  t.mock.timers.enable({ apis: ["Date"], now: 1_700_000_000_000 });
+  try {
+    const created: FakeBackend[] = [];
+    const router = new NamespaceSearchRouter(
+      config(),
+      { storageFor: async (namespace: string) => ({ dir: `/tmp/remnic/${namespace}` }) },
+      () => {
+        const backend = new FakeBackend(false, [], { check: "present" }, false, {
+          debugStatus: "cached-health",
+        });
+        backend.statusResult = {
+          pendingEmbeddings: 3,
+          oldestPendingAgeMs: 1_000,
+          totalFiles: 10,
+          embeddedFiles: 7,
+        };
+        created.push(backend);
+        return backend;
+      },
+    );
+
+    const first = await router.healthForNamespace("shared");
+    const second = await router.healthForNamespace("shared");
+
+    assert.equal(created.length, 1, "second call must not create another backend probe");
+    assert.equal(created[0]?.availabilitySignals.length, 1);
+    assert.deepEqual(created[0]?.checkCollections, ["openclaw-engram--ns-736861726564"]);
+    assert.deepEqual(first, second);
+    assert.equal(first.debugStatus, "cached-health");
+    assert.equal(first.pendingEmbeddings, 3);
+
+    first.available = false;
+    first.debugStatus = "mutated";
+    const third = await router.healthForNamespace("shared");
+    assert.equal(third.available, true, "returned health objects must be copies");
+    assert.equal(third.debugStatus, "cached-health");
+    assert.equal(created.length, 1);
+  } finally {
+    t.mock.timers.reset();
+  }
+});
+
+test("healthForNamespace re-probes after the health result TTL elapses", async (t) => {
+  t.mock.timers.enable({ apis: ["Date"], now: 1_700_000_000_000 });
+  try {
+    const created: FakeBackend[] = [];
+    const router = new NamespaceSearchRouter(
+      config(),
+      { storageFor: async (namespace: string) => ({ dir: `/tmp/remnic/${namespace}` }) },
+      () => {
+        const backend = new FakeBackend(false, [], { check: "present" });
+        backend.statusResult = {
+          pendingEmbeddings: created.length,
+          oldestPendingAgeMs: 0,
+          totalFiles: 1,
+          embeddedFiles: 1,
+        };
+        created.push(backend);
+        return backend;
+      },
+    );
+
+    const first = await router.healthForNamespace("shared");
+    assert.equal(created.length, 1);
+    assert.equal(first.pendingEmbeddings, 0);
+
+    t.mock.timers.tick(10_000);
+    const second = await router.healthForNamespace("shared");
+
+    assert.equal(created.length, 2, "TTL expiry must force a fresh backend probe");
+    assert.equal(second.pendingEmbeddings, 1);
+    assert.equal(created[0]?.disposed, 1);
+    assert.equal(created[1]?.disposed, 1);
+  } finally {
+    t.mock.timers.reset();
+  }
+});
+
+test("healthForNamespace clearCache forces a re-probe before the TTL", async () => {
+  const created: FakeBackend[] = [];
+  const router = new NamespaceSearchRouter(
+    config(),
+    { storageFor: async (namespace: string) => ({ dir: `/tmp/remnic/${namespace}` }) },
+    () => {
+      const backend = new FakeBackend(false, [], { check: "present" });
+      backend.statusResult = {
+        pendingEmbeddings: created.length + 1,
+        oldestPendingAgeMs: 0,
+        totalFiles: 1,
+        embeddedFiles: 1,
+      };
+      created.push(backend);
+      return backend;
+    },
+  );
+
+  const first = await router.healthForNamespace("shared");
+  assert.equal(created.length, 1);
+  assert.equal(first.pendingEmbeddings, 1);
+
+  router.clearCache();
+  const second = await router.healthForNamespace("shared");
+
+  assert.equal(created.length, 2, "clearCache must drop the health result cache");
+  assert.equal(second.pendingEmbeddings, 2);
+});
+
+test("healthForNamespace does not cache a thrown probe", async () => {
+  let attempts = 0;
+  const created: FakeBackend[] = [];
+  const router = new NamespaceSearchRouter(
+    config(),
+    {
+      storageFor: async (namespace: string) => {
+        attempts += 1;
+        if (attempts === 1) {
+          throw new Error("storage unavailable");
+        }
+        return { dir: `/tmp/remnic/${namespace}` };
+      },
+    },
+    () => {
+      const backend = new FakeBackend(false, [], { check: "present" });
+      backend.statusResult = {
+        pendingEmbeddings: 9,
+        oldestPendingAgeMs: 0,
+        totalFiles: 1,
+        embeddedFiles: 1,
+      };
+      created.push(backend);
+      return backend;
+    },
+  );
+
+  await assert.rejects(
+    () => router.healthForNamespace("shared"),
+    /storage unavailable/,
+  );
+  assert.equal(created.length, 0, "failed call must not create a backend");
+
+  const health = await router.healthForNamespace("shared");
+  assert.equal(health.pendingEmbeddings, 9);
+  assert.equal(created.length, 1);
+
+  const cached = await router.healthForNamespace("shared");
+  assert.equal(created.length, 1, "only the successful result is cached");
+  assert.deepEqual(cached, health);
+});
+
+test("healthForNamespace does not cache an unavailable answer", async () => {
+  const created: FakeBackend[] = [];
+  const router = new NamespaceSearchRouter(
+    config(),
+    { storageFor: async (namespace: string) => ({ dir: `/tmp/remnic/${namespace}` }) },
+    () => {
+      const backend = new FakeBackend(false, [], { check: "present" });
+      if (created.length === 0) {
+        backend.available = false;
+      }
+      backend.statusResult = {
+        pendingEmbeddings: created.length,
+        oldestPendingAgeMs: 0,
+        totalFiles: 1,
+        embeddedFiles: 1,
+      };
+      created.push(backend);
+      return backend;
+    },
+  );
+
+  const first = await router.healthForNamespace("shared");
+  assert.equal(first.available, false);
+  assert.equal(created.length, 1);
+
+  const second = await router.healthForNamespace("shared");
+  assert.equal(second.available, true, "recovery must be visible on the next probe");
+  assert.equal(created.length, 2, "unavailable answers must not be cached");
+});
+
+test("healthForNamespace does not cache a missing-collection answer", async () => {
+  const created: FakeBackend[] = [];
+  const router = new NamespaceSearchRouter(
+    config(),
+    { storageFor: async (namespace: string) => ({ dir: `/tmp/remnic/${namespace}` }) },
+    () => {
+      const backend = new FakeBackend(
+        false,
+        [],
+        created.length === 0 ? { check: "missing" } : { check: "present" },
+      );
+      backend.statusResult = {
+        pendingEmbeddings: created.length,
+        oldestPendingAgeMs: 0,
+        totalFiles: 1,
+        embeddedFiles: 1,
+      };
+      created.push(backend);
+      return backend;
+    },
+  );
+
+  const first = await router.healthForNamespace("shared");
+  assert.equal(first.collectionState, "missing");
+  assert.equal(created.length, 1);
+
+  const second = await router.healthForNamespace("shared");
+  assert.equal(second.collectionState, "present", "collection recovery must re-probe");
+  assert.equal(created.length, 2, "missing-collection answers must not be cached");
+});
+
+function routerWithDegradedFirstProbe(
+  created: FakeBackend[],
+  firstCheck: CollectionState,
+  degradeFirst: (backend: FakeBackend) => void,
+): NamespaceSearchRouter {
+  return new NamespaceSearchRouter(
+    config(),
+    { storageFor: async (namespace: string) => ({ dir: `/tmp/remnic/${namespace}` }) },
+    () => {
+      const isFirst = created.length === 0;
+      const backend = new FakeBackend(false, [], { check: isFirst ? firstCheck : "present" });
+      backend.statusResult = {
+        pendingEmbeddings: 10,
+        oldestPendingAgeMs: 0,
+        totalFiles: 1,
+        embeddedFiles: 1,
+      };
+      if (isFirst) degradeFirst(backend);
+      created.push(backend);
+      return backend;
+    },
+  );
+}
+
+test("healthForNamespace does not cache an unknown-collection answer", async () => {
+  const created: FakeBackend[] = [];
+  const router = routerWithDegradedFirstProbe(created, "unknown", () => {});
+
+  const first = await router.healthForNamespace("shared");
+  assert.equal(first.collectionState, "unknown");
+
+  const second = await router.healthForNamespace("shared");
+  assert.equal(second.collectionState, "present", "recovery must be visible on the next probe");
+  assert.equal(created.length, 2, "unknown-collection answers must not be cached");
+});
+
+test("healthForNamespace does not cache an answer whose backend status failed", async () => {
+  const created: FakeBackend[] = [];
+  const router = routerWithDegradedFirstProbe(created, "present", (backend) => {
+    backend.statusResult = null;
+  });
+
+  const first = await router.healthForNamespace("shared");
+  assert.equal(first.pendingEmbeddings, null);
+
+  const second = await router.healthForNamespace("shared");
+  assert.equal(second.pendingEmbeddings, 10, "backlog data must be visible on the next probe");
+  assert.equal(created.length, 2, "answers without backlog data must not be cached");
+});
+
+test("healthForNamespace does not cache a backlog over the threshold", async () => {
+  const created: FakeBackend[] = [];
+  const router = routerWithDegradedFirstProbe(created, "present", (backend) => {
+    backend.statusResult = {
+      pendingEmbeddings: config().qmdEmbeddingBacklogThreshold + 1,
+      oldestPendingAgeMs: 0,
+      totalFiles: 1,
+      embeddedFiles: 1,
+    };
+  });
+
+  await router.healthForNamespace("shared");
+
+  const second = await router.healthForNamespace("shared");
+  assert.equal(second.pendingEmbeddings, 10, "a drained backlog must be visible on the next probe");
+  assert.equal(created.length, 2, "over-threshold answers must not be cached");
+});
+
+test("healthForNamespace dispose drops the health result cache", async () => {
+  const created: FakeBackend[] = [];
+  const router = new NamespaceSearchRouter(
+    config(),
+    { storageFor: async (namespace: string) => ({ dir: `/tmp/remnic/${namespace}` }) },
+    () => {
+      const backend = new FakeBackend(false, [], { check: "present" });
+      backend.statusResult = {
+        pendingEmbeddings: created.length + 1,
+        oldestPendingAgeMs: 0,
+        totalFiles: 1,
+        embeddedFiles: 1,
+      };
+      created.push(backend);
+      return backend;
+    },
+  );
+
+  const first = await router.healthForNamespace("shared");
+  assert.equal(created.length, 1);
+  assert.equal(first.pendingEmbeddings, 1);
+
+  await router.dispose();
+  const second = await router.healthForNamespace("shared");
+
+  assert.equal(created.length, 2, "dispose must drop the health result cache");
+  assert.equal(second.pendingEmbeddings, 2);
+});
+
+test("healthForNamespace disableSearchBackends drops the health result cache", async () => {
+  const created: FakeBackend[] = [];
+  const router = new NamespaceSearchRouter(
+    config(),
+    { storageFor: async (namespace: string) => ({ dir: `/tmp/remnic/${namespace}` }) },
+    () => {
+      const backend = new FakeBackend(false, [], { check: "present" });
+      backend.statusResult = {
+        pendingEmbeddings: created.length + 1,
+        oldestPendingAgeMs: 0,
+        totalFiles: 1,
+        embeddedFiles: 1,
+      };
+      created.push(backend);
+      return backend;
+    },
+  );
+
+  const first = await router.healthForNamespace("shared");
+  assert.equal(created.length, 1);
+  assert.equal(first.pendingEmbeddings, 1);
+
+  await router.disableSearchBackends();
+  const second = await router.healthForNamespace("shared");
+
+  assert.equal(created.length, 2, "disableSearchBackends must drop the health result cache");
+  assert.equal(second.pendingEmbeddings, 2);
+});
+
+
+
 test("ensureNamespaceCollection does not cache aborted namespace backend probes", async () => {
   const created: FakeBackend[] = [];
   const router = new NamespaceSearchRouter(