@remnic/core 9.25.6 → 9.26.0

package/src/orchestration/generic-recall-paths.ts

@@ -0,0 +1,119 @@
+import path from "node:path";
+import type { QmdSearchResult } from "../types.js";
+import { ALL_CATEGORY_DIRS } from "../utils/category-dir.js";
+import {
+  isActivityDigestPath,
+  isArtifactMemoryPath,
+  isMeetingRecordPath,
+} from "./orchestrator-helpers.js";
+
+export interface GenericRecallPathPolicy {
+  readonly memoryDir?: string;
+  readonly qmdCollection?: string;
+  readonly qmdColdCollection?: string;
+}
+
+type GenericRecallPathSource = "filesystem" | "qmd";
+
+const CATEGORY_MEMORY_ROOTS: Record<string, true> = Object.fromEntries(
+  ALL_CATEGORY_DIRS.map((directory) => [directory, true]),
+);
+const RESERVED_ARCHIVE_ROOTS: Record<string, true> = { archive: true, namespaces: true };
+
+function isQmdCollectionPrefix(
+  prefix: string,
+  collection: string | undefined,
+  source: GenericRecallPathSource,
+): boolean {
+  if (typeof collection !== "string" || collection.length === 0) return false;
+  if (prefix.startsWith(`${collection}--`)) return true;
+  if (prefix !== collection) return false;
+  if (source === "qmd") return true;
+  return (
+    !Object.hasOwn(CATEGORY_MEMORY_ROOTS, prefix) &&
+    !Object.hasOwn(RESERVED_ARCHIVE_ROOTS, prefix)
+  );
+}
+
+function normalizeQmdUriPath(relativePath: string, source: GenericRecallPathSource): string {
+  if (source !== "qmd" || !relativePath.startsWith("qmd://")) return relativePath;
+  try {
+    const parsed = new URL(relativePath);
+    if (parsed.protocol !== "qmd:" || !parsed.hostname) return relativePath;
+    const collection = decodeURIComponent(parsed.hostname);
+    const pathname = decodeURIComponent(parsed.pathname.replace(/^\/+/, ""));
+    return pathname ? `${collection}/${pathname}` : collection;
+  } catch {
+    return relativePath;
+  }
+}
+
+function stripQmdCollectionPrefix(
+  relativePath: string,
+  policy: GenericRecallPathPolicy,
+  source: GenericRecallPathSource,
+): string {
+  const normalized = path.posix.normalize(normalizeQmdUriPath(relativePath, source).replace(/\\/g, "/"));
+  const slashIndex = normalized.indexOf("/");
+  if (slashIndex <= 0 || slashIndex >= normalized.length - 1) return normalized;
+
+  const prefix = normalized.slice(0, slashIndex);
+  const matchesHotCollection = isQmdCollectionPrefix(prefix, policy.qmdCollection, source);
+  const matchesColdCollection = isQmdCollectionPrefix(prefix, policy.qmdColdCollection, source);
+  return matchesHotCollection || matchesColdCollection
+    ? normalized.slice(slashIndex + 1)
+    : normalized;
+}
+
+export function isTopLevelArchivePath(
+  filePath: string,
+  policy: GenericRecallPathPolicy = {},
+  source: GenericRecallPathSource = "filesystem",
+): boolean {
+  const isAbsolutePath = path.isAbsolute(filePath);
+  const relative =
+    policy.memoryDir && isAbsolutePath
+      ? path.relative(policy.memoryDir, path.resolve(filePath))
+      : filePath;
+  const normalized = !isAbsolutePath
+    ? stripQmdCollectionPrefix(relative, policy, source)
+    : relative;
+  return /^(?:archive|namespaces[\\/][^\\/]+[\\/]archive)(?:[\\/]|$)/i.test(normalized);
+}
+
+export function isGenericRecallExcludedPath(
+  filePath: string,
+  policy: GenericRecallPathPolicy = {},
+  source: GenericRecallPathSource = "filesystem",
+): boolean {
+  return (
+    isArtifactMemoryPath(filePath) ||
+    isActivityDigestPath(filePath, policy.memoryDir) ||
+    isTopLevelArchivePath(filePath, policy, source) ||
+    isMeetingRecordPath(filePath)
+  );
+}
+
+export function filterRecallCandidates(
+  candidates: QmdSearchResult[],
+  options: {
+    namespacesEnabled: boolean;
+    recallNamespaces: string[];
+    resolveNamespace: (path: string) => string;
+    limit: number;
+    pathPolicy?: GenericRecallPathPolicy;
+    memoryRoot?: string;
+  },
+): QmdSearchResult[] {
+  const pathPolicy =
+    options.pathPolicy ??
+    (options.memoryRoot === undefined ? undefined : { memoryDir: options.memoryRoot });
+  const scopedByNamespace = options.namespacesEnabled
+    ? candidates.filter((r) =>
+        options.recallNamespaces.includes(r.namespace ?? options.resolveNamespace(r.path)),
+      )
+    : candidates;
+  return scopedByNamespace
+    .filter((r) => !isGenericRecallExcludedPath(r.path, pathPolicy, "qmd"))
+    .slice(0, Math.max(0, options.limit));
+}

package/src/orchestration/namespace-read-fanout.ts

@@ -475,16 +475,4 @@ export class NamespaceReadFanoutCoordinator {
     return lists.flat();
   }
 
-  async readArchivedMemoriesForNamespaces(
-    namespaces: string[],
-  ): Promise<MemoryFile[]> {
-    const uniq = Array.from(new Set(namespaces.filter(Boolean)));
-    const lists = await Promise.all(
-      uniq.map(async (ns) => {
-        const sm = await this.deps.storageRouter.storageFor(ns);
-        return sm.readArchivedMemories();
-      }),
-    );
-    return lists.flat();
-  }
 }

package/src/orchestration/orchestrator-helpers.ts

@@ -641,21 +641,6 @@ export function isActivityDigestPath(filePath: string, memoryRoot?: string): boo
   return ACTIVITY_DIGEST_ANYWHERE.test(filePath);
 }
 
-/**
- * Paths that dedicated surfaces own and generic recall must never inject:
- * artifacts, activity digests (issue #1899), and meeting records (issue #1900).
- * One predicate shared by every recall filter site so the exclusion cannot drift.
- * Explicit search paths (memory_search, activity search) do not apply this
- * filter, so those surfaces still read them.
- */
-export function isGenericRecallExcludedPath(filePath: string, memoryRoot?: string): boolean {
-  return (
-    isArtifactMemoryPath(filePath) ||
-    isActivityDigestPath(filePath, memoryRoot) ||
-    isMeetingRecordPath(filePath)
-  );
-}
-
 export function buildCompressionGuidelinesMarkdown(
   events: MemoryActionEvent[],
   generatedAtIso: string = new Date().toISOString(),
@@ -663,26 +648,6 @@ export function buildCompressionGuidelinesMarkdown(
   return buildCompressionGuidelinesMarkdownV2(events, generatedAtIso);
 }
 
-export function filterRecallCandidates(
-  candidates: QmdSearchResult[],
-  options: {
-    namespacesEnabled: boolean;
-    recallNamespaces: string[];
-    resolveNamespace: (path: string) => string;
-    limit: number;
-    /** Memory root for top-level activity-digest detection. */
-    memoryRoot?: string;
-  },
-): QmdSearchResult[] {
-  const scopedByNamespace = options.namespacesEnabled
-    ? candidates.filter((r) =>
-        options.recallNamespaces.includes(r.namespace ?? options.resolveNamespace(r.path)),
-      )
-    : candidates;
-  return scopedByNamespace
-    .filter((r) => !isGenericRecallExcludedPath(r.path, options.memoryRoot))
-    .slice(0, Math.max(0, options.limit));
-}
 
 export function applyQueryAwareCandidateFilter(
   candidates: QmdSearchResult[],

package/src/orchestration/recall-internal.ts

@@ -100,7 +100,7 @@ import {
   type QueryAwarePrefilter,
   type RecallInvocationOptions,
 } from "../orchestrator.js";
-import { isGenericRecallExcludedPath } from "./orchestrator-helpers.js";
+import { isGenericRecallExcludedPath } from "./generic-recall-paths.js";
 
 import type {
   RecallSectionAppendOptions,
@@ -4165,8 +4165,7 @@ export class RecallInternalCoordinator {
         memoryResults = memoryResults.filter((r) =>
           recallNamespaces.includes(r.namespace ?? this.deps.namespaceFromPath(r.path)));
       }
-      // Artifacts + activity digests are dedicated surfaces, never generic recall.
-      memoryResults = memoryResults.filter((r) => !isGenericRecallExcludedPath(r.path, this.deps.config.memoryDir));
+      memoryResults = memoryResults.filter((r) => !isGenericRecallExcludedPath(r.path, this.deps.config, "qmd"));
 
       const isFullModeGraphAssist =
         graphCaps.multiGraphMemory &&
@@ -4297,6 +4296,7 @@ export class RecallInternalCoordinator {
           }
         }
       }
+      memoryResults = memoryResults.filter((r) => !isGenericRecallExcludedPath(r.path, this.deps.config, "qmd"));
 
       // Apply mandatory recall safety filters before deadline-bound scoring
       // enrichment. If scoring times out, we must fall back to this filtered
@@ -4550,7 +4550,7 @@ export class RecallInternalCoordinator {
                 recallNamespaces,
                 resolveNamespace: (p) => this.deps.namespaceFromPath(p),
                 limit: embeddingFetchLimit,
-                memoryRoot: this.deps.config.memoryDir,
+                pathPolicy: this.deps.config,
               },
             );
             const boostedScoped = await this.deps.boostSearchResults(
@@ -4745,7 +4745,7 @@ export class RecallInternalCoordinator {
                 recallNamespaces,
                 resolveNamespace: (p) => this.deps.namespaceFromPath(p),
                 limit: embeddingFetchLimit,
-                memoryRoot: this.deps.config.memoryDir,
+                pathPolicy: this.deps.config,
               },
             );
             const boostedScoped = await this.deps.boostSearchResults(
@@ -4868,7 +4868,7 @@ export class RecallInternalCoordinator {
             typeof asOfMs === "number" && Number.isFinite(asOfMs);
           const activeMemories = memories.filter(
             (m) => {
-              if (isGenericRecallExcludedPath(m.path, this.deps.config.memoryDir)) return false;
+              if (isGenericRecallExcludedPath(m.path, this.deps.config)) return false;
               const status = m.frontmatter.status;
               if (!status || status === "active") return true;
               if (status === "superseded") {

package/src/orchestration/recall-search-pipeline.test.ts

@@ -78,13 +78,12 @@ function scopedCandidatesStub(
     seed.filter((r) => candidatePaths.has(r.path)).slice(0, Math.max(0, limit));
 }
 
-function fallbackCoordinator(seed: QmdSearchResult[], archived: MemoryFile[]): RecallSearchPipelineCoordinator {
-  const readArchivedMemoriesForNamespaces: RecallSearchPipelineDeps["readArchivedMemoriesForNamespaces"] =
-    async () => archived;
+function queryAwareFallbackCoordinator(
+  seed: QmdSearchResult[],
+): RecallSearchPipelineCoordinator {
   const namespaceFromPath: RecallSearchPipelineDeps["namespaceFromPath"] = () => "default";
   const deps = {
     searchScopedMemoryCandidates: scopedCandidatesStub(seed),
-    readArchivedMemoriesForNamespaces,
     namespaceFromPath,
     config: { memoryDir: "/mem" },
   } as unknown as RecallSearchPipelineDeps;
@@ -107,48 +106,85 @@ const SEED_PATHS = [MEETING_RECORD, "facts/a.md", "artifacts/y.md"];
 const SEED_WITH_NON_RECALLABLE = SEED_PATHS.map((p) => result("default", p));
 const PREFILTER = prefilter(SEED_PATHS);
 
-test("archive fallback excludes non-recallable seeds on the archived-empty early return", async () => {
-  const coordinator = fallbackCoordinator(SEED_WITH_NON_RECALLABLE, []);
-  // limit 10 > filtered seed count, tokens non-empty, archived empty → the
-  // `return scopedSeedResults` early path at the archived-empty guard.
-  const out = await coordinator.searchLongTermArchiveFallback("quarterly report", ["default"], 10, PREFILTER);
+test("query-aware fallback filters non-recallable scoped candidates", async () => {
+  const coordinator = queryAwareFallbackCoordinator(SEED_WITH_NON_RECALLABLE);
+  const out = await coordinator.searchQueryAwareFallback("quarterly report", 10, PREFILTER);
   assert.deepEqual(
     out.map((r) => r.path),
     ["facts/a.md"],
-    "meeting record + artifact must not leak through the direct-caller early return",
+    "meeting records and artifacts must not leak through query-aware fallback",
   );
 });
 
-test("archive fallback excludes non-recallable seeds on the empty-tokens early return", async () => {
-  const coordinator = fallbackCoordinator(SEED_WITH_NON_RECALLABLE, [memory("facts/z.md", "archived")]);
-  // Empty prompt → zero tokens → the `return scopedSeedResults` early path
-  // before archive scoring is even attempted.
-  const out = await coordinator.searchLongTermArchiveFallback("", ["default"], 10, PREFILTER);
+test("query-aware fallback filters non-recallable scoped candidates for an empty query", async () => {
+  const coordinator = queryAwareFallbackCoordinator(SEED_WITH_NON_RECALLABLE);
+  const out = await coordinator.searchQueryAwareFallback("", 10, PREFILTER);
   assert.deepEqual(out.map((r) => r.path), ["facts/a.md"]);
 });
 
-test("archive fallback applies the cap to FILTERED seeds, not raw non-recallable hits", async () => {
-  // The scoped search returns a single non-recallable hit that would fill a cap
-  // of 1 if counted raw; the method must count only recallable seeds, so it does
-  // not early-return that hit and (archive empty) yields nothing.
-  const coordinator = fallbackCoordinator([result("default", MEETING_RECORD)], []);
-  const out = await coordinator.searchLongTermArchiveFallback("quarterly", ["default"], 1, prefilter([MEETING_RECORD]));
+test("query-aware fallback applies the cap after filtering", async () => {
+  const coordinator = queryAwareFallbackCoordinator([result("default", MEETING_RECORD)]);
+  const out = await coordinator.searchQueryAwareFallback("quarterly", 1, prefilter([MEETING_RECORD]));
   assert.deepEqual(out.map((r) => r.path), []);
 });
 
-test("finding 1 — overfetches scoped seeds before exclusion so a capped fetch still yields recallable hits", async () => {
+test("query-aware fallback excludes root archive candidates before its cap", async () => {
+  const archivePath = "archive/2026-02-23/fact-archived.md";
+  const coordinator = queryAwareFallbackCoordinator([
+    result("default", archivePath),
+    result("default", "facts/a.md"),
+  ]);
+
+  const out = await coordinator.searchQueryAwareFallback(
+    "quarterly",
+    1,
+    prefilter([archivePath, "facts/a.md"]),
+  );
+
+  assert.deepEqual(out.map((r) => r.path), ["facts/a.md"]);
+});
+
+test("query-aware fallback retains non-active candidates for historical filtering", async () => {
+  let optionsSeen: { allowArchived?: boolean } | undefined;
+  const searchScopedMemoryCandidates: RecallSearchPipelineDeps["searchScopedMemoryCandidates"] = async (
+    candidatePaths,
+    _query,
+    limit,
+    options,
+  ) => {
+    optionsSeen = options;
+    return [result("default", "facts/superseded.md")]
+      .filter((candidate) => candidatePaths.has(candidate.path))
+      .slice(0, limit);
+  };
+  const coordinator = new RecallSearchPipelineCoordinator({
+    config: { memoryDir: "/mem" },
+    namespaceFromPath: () => "default",
+    searchScopedMemoryCandidates,
+  } as unknown as RecallSearchPipelineDeps);
+
+  const out = await coordinator.searchQueryAwareFallback(
+    "historical incident",
+    1,
+    prefilter(["facts/superseded.md"]),
+  );
+
+  assert.deepEqual(out.map((candidate) => candidate.path), ["facts/superseded.md"]);
+  assert.deepEqual(optionsSeen, { allowArchived: true });
+});
+
+test("query-aware fallback overfetches scoped candidates before exclusion", async () => {
   // The scoped candidate order puts the NON-recallable meeting record FIRST;
   // capping the scoped fetch to the caller's limit (1) BEFORE the generic-recall
   // exclusion would return only the meeting record, which is then filtered out —
   // dropping facts/a.md with no refill. Overfetching the full candidate set first
   // and excluding afterward must still surface the recallable hit at the cap.
-  const coordinator = fallbackCoordinator(
-    [result("default", MEETING_RECORD), result("default", "facts/a.md")],
-    [],
-  );
-  const out = await coordinator.searchLongTermArchiveFallback(
+  const coordinator = queryAwareFallbackCoordinator([
+    result("default", MEETING_RECORD),
+    result("default", "facts/a.md"),
+  ]);
+  const out = await coordinator.searchQueryAwareFallback(
     "quarterly",
-    ["default"],
     1,
     prefilter([MEETING_RECORD, "facts/a.md"]),
   );
@@ -159,6 +195,7 @@ test("finding 1 — overfetches scoped seeds before exclusion so a capped fetch
   );
 });
 
+
 function hotSeedCoordinator(seed: QmdSearchResult[]): RecallSearchPipelineCoordinator {
   const deps = {
     config: { memoryDir: "/mem", qmdSearchStrategy: "lex", searchBackend: "qmd" },
@@ -250,3 +287,132 @@ test("finding 1 (round-5) — excluded seeds must not fill the merge cap and sta
     "excluded seeds ranked ahead must not fill the fetchLimit cap and starve the recallable primary hit",
   );
 });
+
+test("QMD widening continues past archive-only pages", async () => {
+  const archived = Array.from({ length: 45 }, (_, index) =>
+    resultScored("default", `archive/2026-07-25/archive-${index}.md`, 2),
+  );
+  const live = resultScored("default", "facts/live.md", 1);
+  const ranked = [...archived, live];
+  const coordinator = new RecallSearchPipelineCoordinator({
+    config: { memoryDir: "/mem", qmdSearchStrategy: "lex", searchBackend: "qmd" },
+    qmd: {},
+    namespaceFromPath: () => "default",
+    searchAcrossNamespaces: async ({ maxResults }: { maxResults: number }) =>
+      ranked.slice(0, maxResults),
+  } as unknown as RecallSearchPipelineDeps);
+
+  const out = await coordinator.fetchQmdMemoryResultsWithArtifactTopUp(
+    "archive-starved",
+    20,
+    20,
+    {
+      namespacesEnabled: false,
+      recallNamespaces: ["default"],
+      resolveNamespace: () => "default",
+      queryAwarePrefilter: {
+        candidatePaths: null,
+        temporalFromDate: null,
+        matchedTags: [],
+        expandedTags: [],
+        combination: "none",
+        filteredToFullSearch: false,
+      },
+    },
+  );
+
+  assert.deepEqual(out.map((candidate) => candidate.path), ["facts/live.md"]);
+});
+
+test("QMD hybrid top-up runs when primary results are archive-only", async () => {
+  const archived = Array.from({ length: 20 }, (_, index) =>
+    resultScored("default", `archive/2026-07-25/archive-${index}.md`, 2),
+  );
+  const live = resultScored("default", "facts/hybrid-live.md", 1);
+  let hybridCalls = 0;
+  const coordinator = new RecallSearchPipelineCoordinator({
+    config: {
+      memoryDir: "/mem",
+      qmdSearchStrategy: "hybrid",
+      searchBackend: "qmd",
+    },
+    qmd: {
+      search: async (
+        _query: string,
+        _collection: string,
+        maxResults: number,
+      ) => archived.slice(0, maxResults),
+      hybridSearch: async () => {
+        hybridCalls += 1;
+        return [live];
+      },
+    },
+    namespaceFromPath: () => "default",
+  } as unknown as RecallSearchPipelineDeps);
+
+  const out = await coordinator.fetchQmdMemoryResultsWithArtifactTopUp(
+    "archive-starved-hybrid",
+    2,
+    2,
+    {
+      namespacesEnabled: false,
+      recallNamespaces: ["default"],
+      resolveNamespace: () => "default",
+      collection: "openclaw-engram",
+      queryAwarePrefilter: {
+        candidatePaths: null,
+        temporalFromDate: null,
+        matchedTags: [],
+        expandedTags: [],
+        combination: "none",
+        filteredToFullSearch: false,
+      },
+    },
+  );
+
+  assert.ok(hybridCalls > 0, "archive-only primary pages must trigger hybrid top-up");
+  assert.deepEqual(out.map((candidate) => candidate.path), ["facts/hybrid-live.md"]);
+});
+
+test("QMD stops after an underfilled archive-only page", async () => {
+  const archived = [resultScored("default", "archive/2026-07-25/only.md", 2)];
+  let searchCalls = 0;
+  const coordinator = new RecallSearchPipelineCoordinator({
+    config: {
+      memoryDir: "/mem",
+      qmdSearchStrategy: "hybrid",
+      searchBackend: "qmd",
+    },
+    qmd: {
+      search: async () => {
+        searchCalls += 1;
+        return archived;
+      },
+      hybridSearch: async () => [],
+    },
+    namespaceFromPath: () => "default",
+  } as unknown as RecallSearchPipelineDeps);
+
+  const out = await coordinator.fetchQmdMemoryResultsWithArtifactTopUp(
+    "archive-underfilled",
+    20,
+    20,
+    {
+      namespacesEnabled: false,
+      recallNamespaces: ["default"],
+      resolveNamespace: () => "default",
+      collection: "openclaw-engram",
+      queryAwarePrefilter: {
+        candidatePaths: null,
+        temporalFromDate: null,
+        matchedTags: [],
+        expandedTags: [],
+        combination: "none",
+        filteredToFullSearch: false,
+      },
+    },
+  );
+
+  assert.deepEqual(out, []);
+  assert.equal(searchCalls, 1, "an underfilled raw page must not trigger widening");
+});