@remnic/core 9.25.6 → 9.26.0

package/src/orchestration/recall-search-pipeline.ts

@@ -28,7 +28,6 @@ import { NamespaceStorageRouter } from "../namespaces/storage.js";
 import { NegativeExampleStore } from "../negative.js";
 import { qmdCollectionPathParts } from "./qmd-result-resolver.js";
 import type { GraphRecallExpandedEntry } from "../recall-state.js";
-import { getDefaultArchiveScoring, memoryFileToScoreItem } from "../recall/archive-scoring.js";
 import { RelevanceStore } from "../relevance.js";
 import { RerankCache, rerankLocalOrNoop, reorderByRankedKeys } from "../rerank.js";
 import type { SearchBackend, SearchDegradation, SearchExecutionOptions, SearchQueryOptions } from "../search/port.js";
@@ -55,11 +54,11 @@ import {
   computeQmdHybridFetchLimit,
   filterRecallCandidates,
   throwIfRecallAborted,
-  tokenizeRecallQuery,
   type QmdRecallSnapshot,
   type QueryAwarePrefilter,
 } from "../orchestrator.js";
-import { isActivityDigestPath, isGenericRecallExcludedPath } from "./orchestrator-helpers.js";
+import { isActivityDigestPath } from "./orchestrator-helpers.js";
+import { isGenericRecallExcludedPath, isTopLevelArchivePath } from "./generic-recall-paths.js";
 
 export interface RecallSearchPipelineDeps {
   applyMemoryWorthRerank(
@@ -185,7 +184,6 @@ export interface RecallSearchPipelineDeps {
   namespaceFromPath(p: string): string;
   readonly negatives: NegativeExampleStore;
   readonly qmd: SearchBackend;
-  readArchivedMemoriesForNamespaces(namespaces: string[]): Promise<MemoryFile[]>;
   readQmdResultMemory(
     resultPath: string,
     fallbackStorage: StorageManager,
@@ -215,9 +213,8 @@ export interface RecallSearchPipelineDeps {
     searchOptions?: SearchQueryOptions;
     execution?: SearchExecutionOptions;
   }): Promise<QmdSearchResult[]>;
-  searchLongTermArchiveFallback(
+  searchQueryAwareFallback(
     prompt: string,
-    recallNamespaces: string[],
     limit: number,
     queryAwarePrefilter?: QueryAwarePrefilter,
     abortSignal?: AbortSignal,
@@ -422,26 +419,19 @@ export class RecallSearchPipelineCoordinator {
       ? await this.deps.searchScopedMemoryCandidates(
           queryAwarePrefilter.candidatePaths,
           prompt,
-          // Overfetch the FULL scoped candidate set BEFORE the generic-recall
-          // exclusion `filterRecallCandidates` applies below (L464). Capping to
-          // `qmdFetchLimit` here lets excluded paths (artifacts, activity digests,
-          // meeting records) consume the caller's budget and drop legitimate hits
-          // with no refill — the hot-path mirror of the archive-fallback fix. The
-          // candidate set is an already-bounded prefilter, so its size is the
-          // natural ceiling; the post-exclusion cap is applied by filterRecallCandidates.
+          // Exclude dedicated surfaces before capping the bounded prefilter set.
           queryAwarePrefilter.candidatePaths.size,
           { allowArchived: options.collection !== undefined },
         )
       : [];
     // Drop generic-recall-excluded records before the fetchLimit cap so they can't starve valid memories (#1995).
-    const recallable = (r: QmdSearchResult) => !isGenericRecallExcludedPath(r.path, this.deps.config.memoryDir);
+    const recallable = (r: QmdSearchResult) => !isGenericRecallExcludedPath(r.path, this.deps.config, "qmd");
 
     let fetchLimit = Math.max(qmdFetchLimit, qmdHybridFetchLimit);
     const maxFetchLimit = Math.min(
       320,
       Math.max(fetchLimit, qmdFetchLimit * 5),
     );
-    const MAX_ATTEMPTS = 2;
     const qmdRecallBudgetMs = this.deps.config.recallEnrichmentDeadlineMs ?? 25_000;
     const qmdRecallBudgetEnabled = qmdRecallBudgetMs > 0;
     const startedAtMs = Date.now();
@@ -475,7 +465,7 @@ export class RecallSearchPipelineCoordinator {
       recallNamespaces: options.recallNamespaces,
       resolveNamespace: options.resolveNamespace,
       limit: qmdFetchLimit,
-      memoryRoot: this.deps.config.memoryDir,
+      pathPolicy: this.deps.config,
     });
     const emitDebugSnapshot = async (
       results: QmdSearchResult[],
@@ -508,7 +498,7 @@ export class RecallSearchPipelineCoordinator {
       return [];
     }
 
-    for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt += 1) {
+    for (;;) {
       throwIfRecallAborted(options.abortSignal);
       if (
         qmdRecallBudgetEnabled &&
@@ -554,10 +544,15 @@ export class RecallSearchPipelineCoordinator {
       lastHybridTopUpUsed = false;
       lastHybridTopUpSkippedReason = undefined;
       let mergedResults = primaryResults;
+      const primaryRecallableCount = primaryResults.filter(recallable).length;
+      const primaryIncludesArchivePath = primaryResults.some((result) =>
+        isTopLevelArchivePath(result.path, this.deps.config, "qmd"),
+      );
 
-      // Backfill with hybrid results only when primary retrieval underfills.
+      // Backfill with hybrid results when the recallable primary page underfills.
       if (
-        primaryResults.length < qmdFetchLimit &&
+        primaryRecallableCount < qmdFetchLimit &&
+        (primaryResults.length < qmdFetchLimit || primaryIncludesArchivePath) &&
         (!qmdRecallBudgetEnabled ||
           Date.now() - startedAtMs < qmdRecallBudgetMs)
       ) {
@@ -620,7 +615,7 @@ export class RecallSearchPipelineCoordinator {
         recallNamespaces: options.recallNamespaces,
         resolveNamespace: options.resolveNamespace,
         limit: fetchLimit,
-        memoryRoot: this.deps.config.memoryDir,
+        pathPolicy: this.deps.config,
       });
 
       if (filteredResults.length >= qmdFetchLimit) {
@@ -631,11 +626,9 @@ export class RecallSearchPipelineCoordinator {
       if (filteredResults.length > bestFiltered.length) {
         bestFiltered = filteredResults;
       }
-      if (mergedResults.length === 0) {
-        await emitDebugSnapshot(filteredResults, fetchLimit);
-        return filteredResults;
-      }
-      if (mergedResults.length < fetchLimit && filteredResults.length > 0) {
+      // A full raw page may filter down to nothing (for example, archive-only
+      // hits). Keep widening until the backend itself underfills the page.
+      if (primaryResults.length < fetchLimit) {
         await emitDebugSnapshot(filteredResults, fetchLimit);
         return filteredResults;
       }
@@ -679,20 +672,18 @@ export class RecallSearchPipelineCoordinator {
   }
 
   /**
-   * Long-term fallback retrieval.
-   * Searches archived memories only, and is invoked only when hot recall returns zero hits.
+   * Cold fallback retains qualifying query-aware candidates. Archive records
+   * belong to dedicated surfaces and never re-enter generic recall.
    */
-  async searchLongTermArchiveFallback(
+  async searchQueryAwareFallback(
     prompt: string,
-    recallNamespaces: string[],
     limit: number,
     queryAwarePrefilter?: QueryAwarePrefilter,
     abortSignal?: AbortSignal,
   ): Promise<QmdSearchResult[]> {
     throwIfRecallAborted(abortSignal);
     const cappedLimit = Math.max(0, limit);
-    if (cappedLimit === 0) return [];
-    if (queryAwarePrefilter?.candidatePaths?.size === 0) return [];
+    if (cappedLimit === 0 || queryAwarePrefilter?.candidatePaths?.size === 0) return [];
 
     const candidatePaths = queryAwarePrefilter?.candidatePaths;
     const scopedSeedResults = (
@@ -700,57 +691,12 @@ export class RecallSearchPipelineCoordinator {
         ? await this.deps.searchScopedMemoryCandidates(
             candidatePaths,
             prompt,
-            // Overfetch the FULL scoped candidate set BEFORE the generic-recall
-            // exclusion below. Capping to `cappedLimit` first lets excluded paths
-            // (artifacts, activity digests, meeting records) consume the caller's
-            // budget and drop legitimate hits with no refill — e.g. a prefilter of
-            // [meetingRecord, facts/a.md] at limit 1 would return nothing. The
-            // candidate set is an already-bounded prefilter, so its size is the
-            // natural ceiling; the post-exclusion cap is applied below.
             candidatePaths.size,
             { allowArchived: true },
           )
         : []
-    ).filter((result) => !isGenericRecallExcludedPath(result.path, this.deps.config.memoryDir));
-    // Non-recallable seed paths (artifacts, activity digests, meeting records) are
-    // excluded above AFTER an unconstrained candidate fetch, so the cap here counts
-    // only recallable seeds — and EVERY early-return / direct-caller path below
-    // returns already-filtered, budget-honoring seeds.
-    if (scopedSeedResults.length >= cappedLimit) {
-      return scopedSeedResults.slice(0, cappedLimit);
-    }
-
-    const tokens = Array.from(new Set(tokenizeRecallQuery(prompt)));
-    if (tokens.length === 0) return scopedSeedResults;
-
-    throwIfRecallAborted(abortSignal);
-    const archivedMemories =
-      await this.deps.readArchivedMemoriesForNamespaces(recallNamespaces);
-    if (archivedMemories.length === 0) return scopedSeedResults;
-
-    // Issue #1674: off-load the CPU-bound archive-scoring loop to a
-    // worker_threads pool so concurrent recall requests run on separate
-    // cores instead of serializing on the main JS thread. The pure scoring
-    // function is identical to the old inline loop — only the execution
-    // context changed. Aborts are checked at the boundaries (before submit
-    // and after result); the worker's work is bounded by the file count.
-    throwIfRecallAborted(abortSignal);
-    const scoring = getDefaultArchiveScoring();
-    const scoredResults = await scoring.score(archivedMemories.map(memoryFileToScoreItem), tokens, abortSignal);
-    throwIfRecallAborted(abortSignal);
-    const scored: QmdSearchResult[] = scoredResults.map((r) => ({
-      docid: r.docid,
-      path: r.path,
-      score: r.score,
-      snippet: r.snippet,
-    }));
-
-    return dedupeResultsByNamespace(
-      [...scopedSeedResults, ...scored],
-      this.deps.namespaceFromPath,
-      cappedLimit,
-      { filter: (result) => !isGenericRecallExcludedPath(result.path, this.deps.config.memoryDir) },
-    );
+    ).filter((result) => !isGenericRecallExcludedPath(result.path, this.deps.config, "qmd"));
+    return scopedSeedResults.slice(0, cappedLimit);
   }
 
   async applyColdFallbackPipeline(options: {
@@ -935,17 +881,19 @@ export class RecallSearchPipelineCoordinator {
       }
     }
     if (longTerm.length === 0) {
-      // The archive-scan scoring worker terminates when EITHER the caller
-      // aborts or this step's own deadline wins — both flow through the
-      // step signal injected by runColdStepWithinDeadline (#1674, #1907).
       longTerm = await runColdStepWithinDeadline(
-        "archive scan", [],
-        (stepSignal) => this.deps.searchLongTermArchiveFallback(
-          options.prompt, options.recallNamespaces, options.recallResultLimit,
-          options.queryAwarePrefilter, stepSignal),
+        "query-aware fallback",
+        [],
+        (stepSignal) =>
+          this.deps.searchQueryAwareFallback(
+            options.prompt,
+            options.recallResultLimit,
+            options.queryAwarePrefilter,
+            stepSignal,
+          ),
       );
       if (longTerm.length > 0) {
-        log.debug("cold-tier recall source=archive-scan");
+        log.debug("cold-tier recall source=query-aware-fallback");
       }
     }
     if (longTerm.length === 0) return [];
@@ -997,9 +945,8 @@ export class RecallSearchPipelineCoordinator {
       }
       results = scopedResults;
     }
-    // Dedicated-surface isolation: generic recall must exclude artifacts and
-    // activity digests (both remain readable via explicit search).
-    results = results.filter((r) => !isGenericRecallExcludedPath(r.path, this.deps.config.memoryDir));
+    // Dedicated-surface isolation keeps generic recall out of artifacts, activity digests, and archives.
+    results = results.filter((r) => !isGenericRecallExcludedPath(r.path, this.deps.config, "qmd"));
     if (results.length === 0) return [];
 
     const isFullModeGraphAssist =
@@ -1022,6 +969,9 @@ export class RecallSearchPipelineCoordinator {
       });
       results = merged;
     }
+    results = results.filter(
+      (result) => !isGenericRecallExcludedPath(result.path, this.deps.config, "qmd"),
+    );
 
     const boostInput = await this.deps.filterSearchResultsForRecall(
       results,

package/src/orchestrator.ts

@@ -134,11 +134,11 @@ import {
   computeSemanticDedupScope,
   qmdCollectionNamespaceFromPrefix as computeQmdCollectionNamespaceFromPrefix,
 } from "./orchestration/orchestrator-namespace-scope.js";
+import { isGenericRecallExcludedPath } from "./orchestration/generic-recall-paths.js";
 import {
   abortRecallError,
   buildCompressionGuidelinesMarkdown,
   buildQmdIntentHint,
-  isGenericRecallExcludedPath,
   mergeArtifactRecallCandidates,
   tokenizeRecallQuery,
   type BulkImportBatchIngestResult,
@@ -163,7 +163,6 @@ export {
   computeQmdHybridFetchLimit,
   defaultWorkspaceDir,
   filterHourlySummaryMarkdownForLocalDay,
-  filterRecallCandidates,
   formatDateInTimeZone,
   isArtifactMemoryPath,
   lifecycleRecallScoreAdjustment,
@@ -198,6 +197,7 @@ export {
   type RecallInvocationOptions,
   type RecallModeDecision,
 } from "./orchestration/orchestrator-helpers.js";
+export { filterRecallCandidates } from "./orchestration/generic-recall-paths.js";
 
 export { hasIdentityRecoveryIntent, resolveEffectiveIdentityInjectionMode } from "./orchestration/recall-result-formatter.js";
 import {
@@ -541,7 +541,6 @@ import type {
   RecallSectionConfig,
   RecallTierExplain,
 } from "./types.js";
-import { disposeDefaultArchiveScoring, getDefaultArchiveScoring, memoryFileToScoreItem } from "./recall/archive-scoring.js";
 
 // Issue #1526 seam 15: ExtractionRunResult moved to orchestration/extraction-run.ts.
 export type { ExtractionRunResult } from "./orchestration/extraction-run.js";
@@ -997,8 +996,6 @@ export class Orchestrator {
       if (this.conversationQmd && this.conversationQmd !== this.qmd) {
         await (this.conversationQmd as { dispose?: () => void | Promise<void> }).dispose?.();
       }
-      // Issue #1674: terminate archive-scoring worker threads on destroy.
-      await disposeDefaultArchiveScoring();
     } finally {
       if (bufferFlushError !== undefined) {
         log.warn(
@@ -2483,7 +2480,7 @@ export class Orchestrator {
     if (!paths) return null;
     const scoped = new Set<string>();
     for (const memoryPath of paths) {
-      if (!memoryPath || isGenericRecallExcludedPath(memoryPath, this.config.memoryDir)) continue;
+      if (!memoryPath || isGenericRecallExcludedPath(memoryPath, this.config, "qmd")) continue;
       if (
         resolveNamespaceCapabilities(this.config).namespaces &&
         !recallNamespaces.includes(this.namespaceFromPath(memoryPath))
@@ -3600,16 +3597,14 @@ export class Orchestrator {
     );
   }
 
-  private async searchLongTermArchiveFallback(
+  private async searchQueryAwareFallback(
     prompt: string,
-    recallNamespaces: string[],
     limit: number,
     queryAwarePrefilter?: QueryAwarePrefilter,
     abortSignal?: AbortSignal,
   ): Promise<QmdSearchResult[]> {
-    return this.recallSearchPipelineCoordinator.searchLongTermArchiveFallback(
+    return this.recallSearchPipelineCoordinator.searchQueryAwareFallback(
       prompt,
-      recallNamespaces,
       limit,
       queryAwarePrefilter,
       abortSignal,
@@ -3968,11 +3963,4 @@ export class Orchestrator {
       namespaces,
     );
   }
-  private async readArchivedMemoriesForNamespaces(
-    namespaces: string[],
-  ): Promise<MemoryFile[]> {
-    return this.namespaceReadFanoutCoordinator.readArchivedMemoriesForNamespaces(
-      namespaces,
-    );
-  }
 }

package/src/recall-cold-deadline.test.ts

@@ -4,7 +4,7 @@
  * Pins the task-level cancellation contract of runColdStepWithinDeadline
  * inside RecallSearchPipelineCoordinator.applyColdFallbackPipeline:
  *   - when the per-step deadline wins, the losing cold task's INJECTED signal
- *     is aborted (so an orphaned archive scan stops cooperatively) and the
+ *     is aborted so an orphaned query-aware fallback stops cooperatively; the
  *     step returns its fallback (fail-open);
  *   - a task-level deadline must NOT abort the request-level signal;
  *   - a task that completes normally (no deadline) never aborts the injected
@@ -37,24 +37,22 @@ async function coldConfig() {
 }
 
 /**
- * Minimal deps for the archive-scan branch. qmd.isAvailable() === false forces
- * the cold-qmd block to be skipped, so control reaches the archive-scan step;
- * an empty archive result returns [] before any namespace/storage-router work,
- * keeping the stub surface tight. The cast is a deliberate test double.
+ * Minimal dependencies for the query-aware fallback. QMD is unavailable, so
+ * the cold-QMD branch is skipped and the injected fallback step is observed.
  */
 function coldDeps(
   config: PluginConfig,
-  searchLongTermArchiveFallback: RecallSearchPipelineDeps["searchLongTermArchiveFallback"],
+  searchQueryAwareFallback: RecallSearchPipelineDeps["searchQueryAwareFallback"],
 ): RecallSearchPipelineDeps {
   return {
     config,
     qmd: { isAvailable: () => false },
     namespaceFromPath: () => "default",
-    searchLongTermArchiveFallback,
+    searchQueryAwareFallback,
   } as unknown as RecallSearchPipelineDeps;
 }
 
-test("#1907: cold archive-scan deadline aborts the injected step signal, returns fallback, leaves the request signal intact", async () => {
+test("#1907: query-aware fallback deadline aborts the injected step signal, returns fallback, leaves the request signal intact", async () => {
   const { memoryDir, config } = await coldConfig();
   try {
     const stuck = Promise.withResolvers<QmdSearchResult[]>();
@@ -65,7 +63,7 @@ test("#1907: cold archive-scan deadline aborts the injected step signal, returns
     // so the late completion leaks "late" here and the assertion below fails.
     const lateWrites: string[] = [];
     const abandonedTaskSettled = Promise.withResolvers<void>();
-    const deps = coldDeps(config, async (_prompt, _ns, _limit, _prefilter, abortSignal) => {
+    const deps = coldDeps(config, async (_prompt, _limit, _prefilter, abortSignal) => {
       capturedStepSignal = abortSignal;
       const late = await stuck.promise; // does not settle before the deadline wins
       if (!abortSignal?.aborted) {
@@ -87,7 +85,7 @@ test("#1907: cold archive-scan deadline aborts the injected step signal, returns
     });
 
     assert.deepEqual(result, [], "the step returns its fallback when the deadline wins");
-    assert.ok(capturedStepSignal, "archive scan received an injected step signal");
+    assert.ok(capturedStepSignal, "query-aware fallback received an injected step signal");
     assert.equal(
       capturedStepSignal!.aborted,
       true,
@@ -119,7 +117,7 @@ test("#1907: a cold step that completes before its deadline never aborts the inj
   const { memoryDir, config } = await coldConfig();
   try {
     let capturedStepSignal: AbortSignal | undefined;
-    const deps = coldDeps(config, async (_prompt, _ns, _limit, _prefilter, abortSignal) => {
+    const deps = coldDeps(config, async (_prompt, _limit, _prefilter, abortSignal) => {
       capturedStepSignal = abortSignal;
       return []; // resolves immediately, well within the deadline
     });
@@ -135,8 +133,8 @@ test("#1907: a cold step that completes before its deadline never aborts the inj
       abortSignal: requestController.signal,
     });
 
-    assert.deepEqual(result, [], "empty archive returns []");
-    assert.ok(capturedStepSignal, "archive scan received an injected step signal");
+    assert.deepEqual(result, [], "empty query-aware fallback returns []");
+    assert.ok(capturedStepSignal, "query-aware fallback received an injected step signal");
     assert.equal(
       capturedStepSignal!.aborted,
       false,