@remnic/core 9.7.15 → 9.9.0

package/src/orchestration/recall-search-pipeline.test.ts

@@ -0,0 +1,70 @@
+import assert from "node:assert/strict";
+import os from "node:os";
+import path from "node:path";
+import { mkdtemp, rm } from "node:fs/promises";
+import test from "node:test";
+
+import { parseConfig } from "../config.js";
+import { RecallSearchPipelineCoordinator } from "./recall-search-pipeline.js";
+import type { RecallSearchPipelineDeps } from "./recall-search-pipeline.js";
+import type { MemoryFile, QmdSearchResult } from "../types.js";
+import type { StorageManager } from "../index.js";
+
+function result(namespace: string, path: string): QmdSearchResult {
+  return { docid: `${namespace}:${path}`, namespace, path, score: 1, snippet: path };
+}
+
+function memory(path: string, status: string): MemoryFile {
+  return {
+    path,
+    content: path,
+    frontmatter: {
+      status,
+      memoryKind: "fact",
+      created: "2026-07-19T00:00:00.000Z",
+      updated: "2026-07-19T00:00:00.000Z",
+    } as unknown as MemoryFile["frontmatter"],
+  };
+}
+
+async function makeCoordinator(memoryDir: string): Promise<RecallSearchPipelineCoordinator> {
+  const config = parseConfig({
+    openaiApiKey: "sk-test",
+    memoryDir,
+    workspaceDir: path.join(memoryDir, "workspace"),
+    namespaces: true,
+  });
+  const deps = {
+    config,
+    storage: {} as StorageManager,
+    readQmdResultMemory: async (
+      resultPath: string,
+      _fallbackStorage: StorageManager,
+      _recallNamespaces: readonly string[],
+      preferredNamespace?: string,
+    ) => memory(resultPath, preferredNamespace === "private" ? "forgotten" : "active"),
+  } as unknown as RecallSearchPipelineDeps;
+  return new RecallSearchPipelineCoordinator(deps);
+}
+
+test("recall safety keeps same relative path distinct across namespaces", async () => {
+  const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-recall-map-"));
+  try {
+    const coordinator = await makeCoordinator(memoryDir);
+    const results = [result("shared", "facts/same.md"), result("private", "facts/same.md")];
+
+    const loaded = await coordinator.loadSearchResultMemoryMap(results, undefined, {
+      recallNamespaces: ["shared", "private"],
+    });
+    const safe = coordinator.filterSearchResultsByRecallSafety(results, loaded.memoryByPath);
+
+    assert.equal(loaded.memoryByPath.size, 2, "each namespace must retain its own loaded memory");
+    assert.deepEqual(
+      safe.map((candidate) => candidate.namespace),
+      ["shared"],
+      "status filtering must use the memory belonging to each result namespace",
+    );
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});

package/src/orchestration/recall-search-pipeline.ts

@@ -18,6 +18,7 @@
 
 import { createHash } from "node:crypto";
 import path from "node:path";
+import { abortError } from "../abort-error.js";
 import { type CapabilitySet, type GraphConstructionCapabilitySet, resolveCapabilities, resolveConversationContextCapabilities, resolveGraphConstructionCapabilities, resolveIndexingCapabilities, resolveMemoryLifecycleCapabilities, resolveNamespaceCapabilities, resolvePipelineProcessingCapabilities, resolveQmdCapabilities, resolveRecallEnhancementCapabilities } from "../capabilities.js";
 import { EmbeddingFallback } from "../embedding-fallback.js";
 import { StorageManager } from "../index.js";
@@ -29,7 +30,7 @@ import { qmdCollectionPathParts } from "./qmd-result-resolver.js";
 import type { GraphRecallExpandedEntry } from "../recall-state.js";
 import { getDefaultArchiveScoring, memoryFileToScoreItem } from "../recall/archive-scoring.js";
 import { RelevanceStore } from "../relevance.js";
-import { RerankCache, rerankLocalOrNoop } from "../rerank.js";
+import { RerankCache, rerankLocalOrNoop, reorderByRankedKeys } from "../rerank.js";
 import type { SearchBackend, SearchDegradation, SearchExecutionOptions, SearchQueryOptions } from "../search/port.js";
 import { SecureStoreLockedError } from "../secure-store/index.js";
 import { isPathInsideStorageRoot } from "../storage-paths.js";
@@ -38,6 +39,14 @@ import { shouldFilterSupersededFromRecall } from "../temporal-supersession.js";
 import { isValidAsOf, isValidityExpiredNow } from "../temporal-validity.js";
 import type { TrustStageResultItem } from "../trust-score-stage.js";
 import type { MemoryFile, PluginConfig, QmdSearchResult, RecallPlanMode } from "../types.js";
+import {
+  dedupeResultsByNamespace,
+  hasMemoryForResult,
+  markResultKey,
+  memoryForResult,
+  memoryMapKey,
+  resultHasKey,
+} from "../recall-memory-map.js";
 import { type UtilityRuntimeValues, applyUtilityRankingRuntimeDelta } from "../utility-runtime.js";
 import {
   computeArtifactCandidateFetchLimit,
@@ -56,10 +65,12 @@ export interface RecallSearchPipelineDeps {
   applyMemoryWorthRerank(
     results: QmdSearchResult[],
     namespaces: string[],
+    preloadedFrontmatter?: ReadonlyMap<string, MemoryFile>,
   ): Promise<QmdSearchResult[]>;
   applyTrustScoreRerank(
     results: QmdSearchResult[],
     namespaces: string[],
+    preloadedFrontmatter?: ReadonlyMap<string, MemoryFile>,
   ): Promise<{
     results: QmdSearchResult[];
     trustByPath: Map<string, TrustStageResultItem> | null;
@@ -174,13 +185,12 @@ export interface RecallSearchPipelineDeps {
   namespaceFromPath(p: string): string;
   readonly negatives: NegativeExampleStore;
   readonly qmd: SearchBackend;
-  readArchivedMemoriesForNamespaces(
-    namespaces: string[],
-  ): Promise<MemoryFile[]>;
+  readArchivedMemoriesForNamespaces(namespaces: string[]): Promise<MemoryFile[]>;
   readQmdResultMemory(
     resultPath: string,
     fallbackStorage: StorageManager,
     recallNamespaces?: readonly string[],
+    preferredNamespace?: string,
   ): Promise<MemoryFile | null>;
   readonly relevance: RelevanceStore;
   readonly rerankCache: RerankCache;
@@ -575,40 +585,24 @@ export class RecallSearchPipelineCoordinator {
           lastHybridResultCount = hybridResults.length;
           lastHybridTopUpUsed = hybridResults.length > 0;
           if (hybridResults.length > 0) {
-            const mergedByPath = new Map<string, QmdSearchResult>();
-            for (const result of [...primaryResults, ...hybridResults]) {
-              const key = result.path || result.docid;
-              const existing = mergedByPath.get(key);
-              if (!existing || result.score > existing.score) {
-                mergedByPath.set(key, {
-                  ...result,
-                  transport: result.transport ?? "hybrid",
-                  snippet: result.snippet || existing?.snippet || "",
-                });
-              }
-            }
-            mergedResults = [...mergedByPath.values()]
-              .sort((a, b) => b.score - a.score)
-              .slice(0, fetchLimit);
+            // Dedup by composite (namespace, path) so a fanout hit and a
+            // seed/hybrid hit for the same memory never inject twice (#2020).
+            mergedResults = dedupeResultsByNamespace(
+              [...primaryResults, ...hybridResults],
+              this.deps.namespaceFromPath,
+              fetchLimit,
+              { transportFallback: "hybrid" },
+            );
           }
         }
       }
 
       if (scopedSeedResults.length > 0) {
-        const mergedByPath = new Map<string, QmdSearchResult>();
-        for (const result of [...scopedSeedResults, ...mergedResults]) {
-          const key = result.path || result.docid;
-          const existing = mergedByPath.get(key);
-          if (!existing || result.score > existing.score) {
-            mergedByPath.set(key, {
-              ...result,
-              snippet: result.snippet || existing?.snippet || "",
-            });
-          }
-        }
-        mergedResults = [...mergedByPath.values()]
-          .sort((a, b) => b.score - a.score)
-          .slice(0, fetchLimit);
+        mergedResults = dedupeResultsByNamespace(
+          [...scopedSeedResults, ...mergedResults],
+          this.deps.namespaceFromPath,
+          fetchLimit,
+        );
       }
 
       const filteredResults = filterRecallCandidates(mergedResults, {
@@ -728,22 +722,12 @@ export class RecallSearchPipelineCoordinator {
       snippet: r.snippet,
     }));
 
-    const mergedByPath = new Map<string, QmdSearchResult>();
-    for (const result of [...scopedSeedResults, ...scored]) {
-      const key = result.path || result.docid;
-      const existing = mergedByPath.get(key);
-      if (!existing || result.score > existing.score) {
-        mergedByPath.set(key, {
-          ...result,
-          snippet: result.snippet || existing?.snippet || "",
-        });
-      }
-    }
-
-    return [...mergedByPath.values()]
-      .filter((result) => !isArtifactMemoryPath(result.path))
-      .sort((a, b) => b.score - a.score)
-      .slice(0, cappedLimit);
+    return dedupeResultsByNamespace(
+      [...scopedSeedResults, ...scored],
+      this.deps.namespaceFromPath,
+      cappedLimit,
+      { filter: (result) => !isArtifactMemoryPath(result.path) },
+    );
   }
 
   async applyColdFallbackPipeline(options: {
@@ -802,7 +786,7 @@ export class RecallSearchPipelineCoordinator {
     const runColdStepWithinDeadline = async <T>(
       label: string,
       fallback: T,
-      task: () => Promise<T>,
+      task: (stepSignal: AbortSignal) => Promise<T>,
       // Invoked when the deadline abandons this step (before it started or
       // while it runs), so callers can report the abandonment and gate off
       // late observer callbacks (#1536, cursor round-6 on #1544).
@@ -810,47 +794,48 @@ export class RecallSearchPipelineCoordinator {
     ): Promise<T> => {
       throwIfRecallAborted(options.abortSignal);
       const remainingMs = deadlineRemainingMs();
-      if (remainingMs === 0) {
+      // Step-level deadline abort (#1907): compose the caller's request signal with
+      // a per-step controller so a disconnect OR this step's deadline stops the task.
+      const stepController = new AbortController();
+      const stepSignal = options.abortSignal
+        ? AbortSignal.any([options.abortSignal, stepController.signal])
+        : stepController.signal;
+      const abandonToDeadline = () => {
+        stepController.abort(abortError(`cold-tier recall ${label} deadline exceeded`));
         try {
           onDeadline?.();
         } catch {
           // Observers must never break recall.
         }
         log.debug(`cold-tier recall ${label} skipped: shared assembly deadline expired`);
+      };
+      if (remainingMs === 0) {
+        abandonToDeadline();
         return fallback;
       }
-      if (remainingMs === null) return task();
-
+      if (remainingMs === null) return task(stepSignal);
       let timeoutHandle: NodeJS.Timeout | undefined;
       let timedOut = false;
-      const taskPromise = task().catch((err) => {
+      const taskPromise = task(stepSignal).catch((err) => {
         if (timedOut) {
           log.debug(`cold-tier recall ${label} failed after deadline: ${err}`);
           return fallback;
         }
         throw err;
       });
-
       try {
         return await Promise.race<T>([
           taskPromise,
           new Promise<T>((resolve) => {
             timeoutHandle = setTimeout(() => {
               timedOut = true;
-              try {
-                onDeadline?.();
-              } catch {
-                // Observers must never break recall.
-              }
-              log.debug(
-                `cold-tier recall ${label} skipped: shared assembly deadline expired`,
-              );
+              abandonToDeadline();
               resolve(fallback);
             }, remainingMs);
           }),
         ]);
       } finally {
-        if (timeoutHandle) clearTimeout(timeoutHandle);
+        clearTimeout(timeoutHandle);
       }
     };
 
@@ -893,7 +878,7 @@ export class RecallSearchPipelineCoordinator {
         longTerm = await runColdStepWithinDeadline(
           "qmd lookup",
           [],
-          () =>
+          (stepSignal) =>
             this.deps.fetchQmdMemoryResultsWithArtifactTopUp(
               options.prompt,
               coldFetchLimit,
@@ -905,7 +890,7 @@ export class RecallSearchPipelineCoordinator {
                 collection: coldCollection,
                 queryAwarePrefilter: options.queryAwarePrefilter,
                 searchOptions: this.deps.buildConfiguredQmdSearchOptions(options.prompt),
-                abortSignal: options.abortSignal,
+                abortSignal: stepSignal,
                 onDegradation: (degradation) => {
                   if (coldQmdObserverActive) {
                     options.onDegradation?.(degradation);
@@ -927,17 +912,14 @@ export class RecallSearchPipelineCoordinator {
       }
     }
     if (longTerm.length === 0) {
-      // Deadline-aware abort: terminate the scoring worker when the shared
-      // assembly deadline wins, not just when the caller aborts (#1674).
-      const da = new AbortController();
-      if (options.abortSignal?.aborted) da.abort();
-      else options.abortSignal?.addEventListener("abort", () => da.abort(), { once: true });
+      // The archive-scan scoring worker terminates when EITHER the caller
+      // aborts or this step's own deadline wins — both flow through the
+      // step signal injected by runColdStepWithinDeadline (#1674, #1907).
       longTerm = await runColdStepWithinDeadline(
         "archive scan", [],
-        () => this.deps.searchLongTermArchiveFallback(
+        (stepSignal) => this.deps.searchLongTermArchiveFallback(
           options.prompt, options.recallNamespaces, options.recallResultLimit,
-          options.queryAwarePrefilter, da.signal),
-        () => da.abort(),
+          options.queryAwarePrefilter, stepSignal),
       );
       if (longTerm.length > 0) {
         log.debug("cold-tier recall source=archive-scan");
@@ -983,19 +965,12 @@ export class RecallSearchPipelineCoordinator {
           continue;
         }
         if (path.isAbsolute(result.path)) {
-          const resolvedPath = path.resolve(result.path);
-          if (
-            recallRoots.some((recallRoot) =>
-              isPathInsideStorageRoot(recallRoot, resolvedPath),
-            )
-          ) {
+          if (recallRoots.some((recallRoot) => isPathInsideStorageRoot(recallRoot, path.resolve(result.path)))) {
             scopedResults.push(result);
           }
           continue;
         }
-        if (options.recallNamespaces.includes(this.deps.namespaceFromPath(result.path))) {
-          scopedResults.push(result);
-        }
+        if (options.recallNamespaces.includes(result.namespace ?? this.deps.namespaceFromPath(result.path))) scopedResults.push(result);
       }
       results = scopedResults;
     }
@@ -1089,12 +1064,14 @@ export class RecallSearchPipelineCoordinator {
     }
 
     if (caps.rerank && this.deps.config.rerankProvider === "local") {
+      // (namespace, path) id so same-relative-path cross-namespace hits are not collapsed (#2020).
+      const rerankId = (r: QmdSearchResult): string => `${r.namespace ?? ""}|${r.path}`;
       const ranked = await rerankLocalOrNoop({
         query: options.prompt,
         candidates: results
           .slice(0, this.deps.config.rerankMaxCandidates)
           .map((r) => ({
-            id: r.path,
+            id: rerankId(r),
             snippet: r.snippet || r.path,
           })),
         local: this.deps.fastLlmForRerank,
@@ -1106,17 +1083,7 @@ export class RecallSearchPipelineCoordinator {
         cacheTtlMs: this.deps.config.rerankCacheTtlMs,
       });
       if (ranked && ranked.length > 0) {
-        const byPath = new Map(results.map((r) => [r.path, r]));
-        const reordered: QmdSearchResult[] = [];
-        for (const p of ranked) {
-          const it = byPath.get(p);
-          if (it) reordered.push(it);
-        }
-        const rankedSet = new Set(ranked);
-        for (const r of results) {
-          if (!rankedSet.has(r.path)) reordered.push(r);
-        }
-        results = reordered;
+        results = reorderByRankedKeys(results, ranked, rerankId);
       }
     }
     if (caps.rerank && this.deps.config.rerankProvider === "cloud") {
@@ -1131,7 +1098,7 @@ export class RecallSearchPipelineCoordinator {
     // Fail-open on lookup errors.
     if (caps.recallTrustScore && results.length > 0) {
       try {
-        const trustOutcome = await this.deps.applyTrustScoreRerank(results, options.recallNamespaces);
+        const trustOutcome = await this.deps.applyTrustScoreRerank(results, options.recallNamespaces, boostInput.memoryByPath);
         results = trustOutcome.results;
         if (options.trustByPathSink) options.trustByPathSink.trustByPath = trustOutcome.trustByPath;
       } catch (err) {
@@ -1141,7 +1108,7 @@ export class RecallSearchPipelineCoordinator {
       }
     } else if (caps.recallMemoryWorthFilter && results.length > 0) {
       try {
-        results = await this.deps.applyMemoryWorthRerank(results, options.recallNamespaces);
+        results = await this.deps.applyMemoryWorthRerank(results, options.recallNamespaces, boostInput.memoryByPath);
       } catch (err) {
         log.debug("memory-worth filter (cold) failed open", {
           error: (err as Error).message,
@@ -1189,14 +1156,15 @@ export class RecallSearchPipelineCoordinator {
     const unreadablePaths = new Set<string>();
 
     const markChecked = (result: QmdSearchResult): void => {
-      if (result.path) checkedPaths.add(result.path);
+      if (result.path) markResultKey(checkedPaths, result);
     };
     const markUnreadable = (result: QmdSearchResult, err: unknown): void => {
       if (!result.path) return;
-      checkedPaths.add(result.path);
-      unreadablePaths.add(result.path);
+      markChecked(result);
+      markResultKey(unreadablePaths, result);
       log.warn("recall safety filter dropped unreadable secure-store candidate", {
         path: result.path,
+        namespace: result.namespace,
         error: (err as Error).message,
       });
     };
@@ -1216,27 +1184,28 @@ export class RecallSearchPipelineCoordinator {
           };
         }
         await Promise.all(
-          results.slice(offset, offset + batchSize).map(async (r) => {
-            if (!r.path) return;
-            if (memoryByPath.has(r.path)) {
-              markChecked(r);
+        results.slice(offset, offset + batchSize).map(async (r) => {
+          if (!r.path) return;
+          if (hasMemoryForResult(memoryByPath, r)) {
+            markChecked(r);
+            return;
+          }
+          try {
+            const mem = await this.deps.readQmdResultMemory(
+              r.path,
+              this.deps.storage,
+              options?.recallNamespaces,
+              r.namespace,
+            );
+            markChecked(r);
+            if (mem) memoryByPath.set(memoryMapKey(r), mem);
+          } catch (err) {
+            if (err instanceof SecureStoreLockedError) {
+              markUnreadable(r, err);
               return;
             }
-            try {
-              const mem = await this.deps.readQmdResultMemory(
-                r.path,
-                this.deps.storage,
-                options?.recallNamespaces,
-              );
-              markChecked(r);
-              if (mem) memoryByPath.set(r.path, mem);
-            } catch (err) {
-              if (err instanceof SecureStoreLockedError) {
-                markUnreadable(r, err);
-                return;
-              }
-              throw err;
-            }
+            throw err;
+          }
           }),
         );
       }
@@ -1246,7 +1215,7 @@ export class RecallSearchPipelineCoordinator {
 
     for (const result of results) {
       if (!result.path) continue;
-      if (memoryByPath.has(result.path)) {
+      if (hasMemoryForResult(memoryByPath, result)) {
         markChecked(result);
         continue;
       }
@@ -1258,9 +1227,10 @@ export class RecallSearchPipelineCoordinator {
           result.path,
           this.deps.storage,
           options?.recallNamespaces,
+          result.namespace,
         );
         markChecked(result);
-        if (mem) memoryByPath.set(result.path, mem);
+        if (mem) memoryByPath.set(memoryMapKey(result), mem);
       } catch (err) {
         if (err instanceof SecureStoreLockedError) {
           markUnreadable(result, err);
@@ -1292,11 +1262,11 @@ export class RecallSearchPipelineCoordinator {
     let blockedPathFilteredCount = 0;
     const filtered: QmdSearchResult[] = [];
     for (const r of results) {
-      if (r.path && options?.blockedPaths?.has(r.path)) {
+      if (options?.blockedPaths && resultHasKey(options.blockedPaths, r)) {
         blockedPathFilteredCount += 1;
         continue;
       }
-      const memory = memoryByPath.get(r.path);
+      const memory = memoryForResult(memoryByPath, r);
       if (memory) {
         // Review-lifecycle statuses never enter active recall injection
         // (forgotten, pending_review, rejected, quarantined). Superseded and
@@ -1451,17 +1421,20 @@ export class RecallSearchPipelineCoordinator {
     );
     const candidateResults = loaded.completed
       ? results
-      : results.filter((result) => !result.path || loaded.checkedPaths.has(result.path));
+      : results.filter((result) => !result.path || resultHasKey(loaded.checkedPaths, result));
     if (!loaded.completed) {
       log.debug(
         `recall safety filter stopped before validating all candidates (${candidateResults.length}/${results.length} eligible)`,
       );
     }
-    const blockedPaths = new Set(loaded.unreadablePaths);
+    // Always block secure-store-locked candidates that failed to load: they
+    // must never re-enter ranking/injection without a loaded memory, whether or
+    // not dropUnresolved is set. Keyed by composite (namespace, path).
+    const blockedPaths = new Set<string>(loaded.unreadablePaths);
     if (options?.dropUnresolved === true) {
-      for (const resultPath of loaded.checkedPaths) {
-        if (!loaded.memoryByPath.has(resultPath)) {
-          blockedPaths.add(resultPath);
+      for (const result of candidateResults) {
+        if (result.path && resultHasKey(loaded.checkedPaths, result) && !hasMemoryForResult(loaded.memoryByPath, result)) {
+          markResultKey(blockedPaths, result);
         }
       }
     }
@@ -1570,7 +1543,7 @@ export class RecallSearchPipelineCoordinator {
     const boosted: QmdSearchResult[] = [];
     const recencyWeight = this.deps.effectiveRecencyWeight();
     for (const r of safeResults) {
-      const memory = memoryByPath.get(r.path);
+      const memory = memoryForResult(memoryByPath, r);
       let score = r.score;
 
       if (memory) {