@remnic/core 9.47.1 → 9.48.1

package/src/orchestration/recall-internal.ts

@@ -46,7 +46,10 @@ import {
   boundRecallContextComposition, composeRecallContext, contextBudgetForFooter,
   formatCuriosityFooter, selectCuriosityQuestion,
 } from "../recall-context-composition.js";
-import { createRecallSectionMetricRecorder } from "../recall-qos.js";
+import {
+  createBoundedCoreSectionRunner,
+  createRecallSectionMetricRecorder,
+} from "../recall-qos.js";
 import { buildRecallQueryPolicy } from "../recall-query-policy.js";
 import { type GraphRecallExpandedEntry, type LastRecallSnapshot } from "../recall-state.js";
 import {
@@ -74,7 +77,7 @@ import { searchTrustZoneRecords } from "../trust-zones.js";
 import type { IdentityInjectionMode, MemoryFile, QmdSearchResult, RecallPlanMode } from "../types.js";
 import { type VerifiedEpisodeResult, compareVerifiedEpisodeResults, searchVerifiedEpisodes } from "../verified-recall.js";
 import { type WorkProductLedgerSearchResult, searchWorkProductLedgerEntries } from "../work-product-ledger.js";
-import { abortError } from "../abort-error.js";
+import { abortError, isAbortError, raceAbort } from "../abort-error.js";
 import {
   applyQueryAwareCandidateFilter,
   filterRecallCandidates,
@@ -147,7 +150,13 @@ export class RecallInternalCoordinator {
       profileTraceClosed = true;
       this.deps.profiler.endTrace(profileTraceId); // persists to JSONL file
     };
+    // The CONFIGURED core budget, which is what the sections that run outside the
+    // bounded runner report in their metrics: nothing enforces a deadline for them,
+    // so reporting a derived figure would claim a bound that does not exist. The
+    // bounded runner below derives and reports its own effective budget per
+    // section, against the request budget still left when that section starts.
     const recallSectionDeadlineMs = this.deps.config.recallCoreDeadlineMs ?? 75_000;
+    const recallOuterTimeoutMs = this.deps.config.recallOuterTimeoutMs ?? 75_000;
     const enrichmentSectionDeadlineMs =
       this.deps.config.recallEnrichmentDeadlineMs ?? 25_000;
     // Wrap entire recall body in try/finally so profiling trace is always closed,
@@ -206,6 +215,14 @@ export class RecallInternalCoordinator {
       timings,
       logger: log,
     });
+    const runBoundedCoreSection = createBoundedCoreSectionRunner({
+      configuredDeadlineMs: recallSectionDeadlineMs,
+      outerDeadlineAtMs:
+        recallOuterTimeoutMs > 0 ? recallStart + recallOuterTimeoutMs : null,
+      parentSignal: options.abortSignal,
+      record: recordRecallSectionMetric,
+      logger: log,
+    });
     const promptHash = createHash("sha256").update(prompt).digest("hex");
     const traceId = createHash("sha256")
       .update(`${sessionKey ?? "default"}:${recallStart}:${promptHash}`)
@@ -985,41 +1002,48 @@ export class RecallInternalCoordinator {
         });
         return null;
       }
-      const t0 = Date.now();
-      const transcriptEntries = sessionKey
-        ? await readRecentEntityTranscriptEntries(
-            this.deps.transcript.readRecent(
-              entityRecentTranscriptLookbackHours,
-              sessionKey,
-            ),
+      return await runBoundedCoreSection(
+        "entityRetrieval",
+        null as string | null,
+        async (sectionSignal) => {
+          // The transcript store is shared and takes no signal of its own, so the
+          // section stops WAITING on a stalled read rather than pretending to
+          // interrupt it — otherwise a cancelled section sits in this await.
+          const transcriptEntries = sessionKey
+            ? await raceAbort(
+                readRecentEntityTranscriptEntries(
+                  this.deps.transcript.readRecent(
+                    entityRecentTranscriptLookbackHours,
+                    sessionKey,
+                  ),
+                  recentTurns,
+                ),
+                sectionSignal,
+                "entity recall aborted",
+              )
+            : [];
+          return await buildEntityRecallSection({
+            config: this.deps.config,
+            storage: profileStorage,
+            namespaceStorage: (namespace) => this.deps.getStorage(namespace),
+            query: retrievalQuery,
+            recallNamespaces,
             recentTurns,
-          )
-        : [];
-      const section = await buildEntityRecallSection({
-        config: this.deps.config,
-        storage: profileStorage,
-        namespaceStorage: (namespace) => this.deps.getStorage(namespace),
-        query: retrievalQuery,
-        recallNamespaces,
-        recentTurns,
-        maxHints,
-        maxSupportingFacts,
-        maxRelatedEntities,
-        maxChars,
-        transcriptEntries,
-      }).catch((err) => {
-        log.warn(`entity retrieval build failed: ${err}`);
-        return null;
-      });
-      recordRecallSectionMetric({
-        section: "entityRetrieval",
-        priority: "core",
-        durationMs: Date.now() - t0,
-        deadlineMs: recallSectionDeadlineMs,
-        source: "fresh",
-        success: true,
-      });
-      return section;
+            maxHints,
+            maxSupportingFacts,
+            maxRelatedEntities,
+            maxChars,
+            transcriptEntries,
+            abortSignal: sectionSignal,
+          }).catch((err) => {
+            // Cancellation is the deadline/abort contract working; let the bounded
+            // runner record it as cancelled rather than as an empty section.
+            if (isAbortError(err)) throw err;
+            log.warn(`entity retrieval build failed: ${err}`);
+            return null;
+          });
+        },
+      );
     })();
 
     // 1b. Knowledge Index (v7.0)
@@ -1130,7 +1154,6 @@ export class RecallInternalCoordinator {
       )
         return [];
       if (!resolvePresentationCapabilities(this.deps.config).verbatimArtifacts) return [];
-      const t0 = Date.now();
       const targetCount = computeArtifactRecallLimit(
         recallMode,
         recallResultLimit,
@@ -1148,25 +1171,23 @@ export class RecallInternalCoordinator {
         });
         return [];
       }
-      const results = await this.deps.recallArtifactsAcrossNamespaces(
-        retrievalQuery,
-        recallNamespaces,
-        targetCount,
+      return await runBoundedCoreSection(
+        "artifacts",
+        [] as MemoryFile[],
+        async (sectionSignal) => {
+          const results = await this.deps.recallArtifactsAcrossNamespaces(
+            retrievalQuery,
+            recallNamespaces,
+            targetCount,
+            { abortSignal: sectionSignal },
+          );
+          return lifecycleCaps.extractionScopeClassification
+            ? results.filter((artifact) =>
+                canRecallToolScopedMemory(artifact.frontmatter, options.sourceConnector),
+              )
+            : results;
+        },
       );
-
-      recordRecallSectionMetric({
-        section: "artifacts",
-        priority: "core",
-        durationMs: Date.now() - t0,
-        deadlineMs: recallSectionDeadlineMs,
-        source: "fresh",
-        success: true,
-      });
-      return lifecycleCaps.extractionScopeClassification
-        ? results.filter((artifact) =>
-            canRecallToolScopedMemory(artifact.frontmatter, options.sourceConnector),
-          )
-        : results;
     })();
 
     const objectiveStatePromise = (async (): Promise<string | null> => {

package/src/orchestration/recall-search-pipeline.ts

@@ -40,6 +40,7 @@ import {
   fetchActiveArtifactsForNamespace as fetchActiveArtifactsForNamespaceHelper,
   searchEmbeddingFallback as searchEmbeddingFallbackHelper,
   searchQueryAwareFallback as searchQueryAwareFallbackHelper,
+  type ArtifactRecallOptions,
 } from "./recall-search-prefilter.js";
 import { shouldFilterSupersededFromRecall } from "../temporal-supersession.js";
 import { isValidAsOf, isValidityExpiredNow } from "../temporal-validity.js";
@@ -251,8 +252,15 @@ export class RecallSearchPipelineCoordinator {
     namespace: string,
     prompt: string,
     targetCount: number,
+    options: ArtifactRecallOptions = {},
   ): Promise<MemoryFile[]> {
-    return fetchActiveArtifactsForNamespaceHelper(this.deps, namespace, prompt, targetCount);
+    return fetchActiveArtifactsForNamespaceHelper(
+      this.deps,
+      namespace,
+      prompt,
+      targetCount,
+      options,
+    );
   }
 
   async buildQueryAwarePrefilter(

package/src/orchestration/recall-search-prefilter.ts

@@ -42,11 +42,22 @@ export interface PrefilterAndArtifactDeps {
   ): Promise<QmdSearchResult[]>;
 }
 
+/**
+ * Cancellation for the artifact recall path (issue #2291).  Artifact recall
+ * re-scans and re-tokenizes every stored artifact, up to `MAX_ATTEMPTS` times
+ * with a growing fetch limit; on a large or slow memory tree that is minutes of
+ * work that used to keep running after the caller had already given up.
+ */
+export interface ArtifactRecallOptions {
+  abortSignal?: AbortSignal;
+}
+
 export async function fetchActiveArtifactsForNamespace(
   deps: Pick<PrefilterAndArtifactDeps, "storageRouter" | "resolveArtifactSourceStatuses">,
   namespace: string,
   prompt: string,
   targetCount: number,
+  options: ArtifactRecallOptions = {},
 ): Promise<MemoryFile[]> {
   const storage = await deps.storageRouter.storageFor(namespace);
   let fetchLimit = computeArtifactCandidateFetchLimit(targetCount);
@@ -55,7 +66,10 @@ export async function fetchActiveArtifactsForNamespace(
   let bestFiltered: MemoryFile[] = [];
 
   for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt += 1) {
-    const rawResults = await storage.searchArtifacts(prompt, fetchLimit);
+    throwIfRecallAborted(options.abortSignal);
+    const rawResults = await storage.searchArtifacts(prompt, fetchLimit, {
+      abortSignal: options.abortSignal,
+    });
     const sourceIds = Array.from(
       new Set(
         rawResults
@@ -65,6 +79,9 @@ export async function fetchActiveArtifactsForNamespace(
           ),
       ),
     );
+    // The status snapshot rebuild reads every memory in the namespace; do not
+    // start one for a recall that has already been abandoned.
+    throwIfRecallAborted(options.abortSignal);
     const sourceStatus =
       sourceIds.length > 0
         ? await deps.resolveArtifactSourceStatuses(storage, sourceIds)

package/src/orchestrator.ts

@@ -120,6 +120,7 @@ import { ConsolidationRunCoordinator } from "./orchestration/consolidation-run.j
 import { ExtractionPersistCoordinator } from "./orchestration/extraction-persist.js";
 import { RecallInternalCoordinator } from "./orchestration/recall-internal.js";
 import { RecallSearchPipelineCoordinator } from "./orchestration/recall-search-pipeline.js";
+import type { ArtifactRecallOptions } from "./orchestration/recall-search-prefilter.js";
 import { TurnIngestionCoordinator, type TurnIngestionOptions } from "./orchestration/turn-ingestion.js";
 import { RecallIntrospectionCoordinator } from "./orchestration/recall-introspection.js";
 import { OrchestratorInitCoordinator } from "./orchestration/orchestrator-init.js";
@@ -2460,11 +2461,10 @@ export class Orchestrator {
     namespace: string,
     prompt: string,
     targetCount: number,
+    options: ArtifactRecallOptions = {},
   ): Promise<MemoryFile[]> {
     return this.recallSearchPipelineCoordinator.fetchActiveArtifactsForNamespace(
-      namespace,
-      prompt,
-      targetCount,
+      namespace, prompt, targetCount, options,
     );
   }
 
@@ -2472,11 +2472,10 @@ export class Orchestrator {
     prompt: string,
     recallNamespaces: string[],
     targetCount: number,
+    options: ArtifactRecallOptions = {},
   ): Promise<MemoryFile[]> {
     return this.namespaceReadFanoutCoordinator.recallArtifactsAcrossNamespaces(
-      prompt,
-      recallNamespaces,
-      targetCount,
+      prompt, recallNamespaces, targetCount, options,
     );
   }
 

package/src/recall-qos.test.ts

@@ -1,7 +1,12 @@
 import test from "node:test";
 import assert from "node:assert/strict";
+import { throwIfAborted } from "./abort-error.js";
 import {
+  createBoundedCoreSectionRunner,
   createRecallSectionMetricRecorder,
+  resolveRecallCoreSectionDeadlineMs,
+  runRecallSectionWithinDeadline,
+  yieldToEventLoop,
   type RecallSectionMetric,
 } from "./recall-qos.js";
 
@@ -83,3 +88,287 @@ test("recall section metric recorder respects timing overrides and logs enrichme
     success: true,
   });
 });
+
+test("timed-out section metric reports the breach instead of a plain duration", () => {
+  const timings: Record<string, string> = {};
+  const calls: Array<{ level: "info" | "debug"; payload: unknown[] }> = [];
+  const recorder = createRecallSectionMetricRecorder({
+    timings,
+    logger: {
+      info: (_message: string, ...payload: unknown[]) => calls.push({ level: "info", payload }),
+      debug: (_message: string, ...payload: unknown[]) => calls.push({ level: "debug", payload }),
+    },
+  });
+
+  recorder({
+    section: "entityRetrieval",
+    priority: "core",
+    durationMs: 75_004,
+    deadlineMs: 75_000,
+    source: "timeout",
+    success: false,
+  });
+
+  assert.equal(timings.entityRetrieval, "timeout(75004ms)");
+  // A breach is not a success, so it must not be logged as one.
+  assert.equal(calls[0]?.level, "debug");
+});
+
+test("recall section deadline degrades to the fallback and cancels the section", async () => {
+  let observedSignal: AbortSignal | undefined;
+  let cancelledDuringWork = false;
+
+  // A real timer is the subject here, not a guessed wait: the section below
+  // never settles on its own, so the deadline always wins regardless of load.
+  const outcome = await runRecallSectionWithinDeadline({
+    deadlineMs: 15,
+    fallback: "degraded",
+    run: async (sectionSignal) => {
+      observedSignal = sectionSignal;
+      const cancelled = Promise.withResolvers<void>();
+      sectionSignal.addEventListener("abort", () => {
+        cancelledDuringWork = true;
+        cancelled.resolve();
+      });
+      await cancelled.promise;
+      return "late value";
+    },
+  });
+
+  assert.equal(outcome.timedOut, true);
+  assert.equal(outcome.value, "degraded");
+  assert.equal(observedSignal?.aborted, true);
+  assert.equal(cancelledDuringWork, true);
+});
+
+test("recall section deadline returns the real value when the section finishes in time", async () => {
+  const outcome = await runRecallSectionWithinDeadline({
+    deadlineMs: 5_000,
+    fallback: null as string | null,
+    run: async (sectionSignal) => {
+      assert.equal(sectionSignal.aborted, false);
+      return "section body";
+    },
+  });
+
+  assert.equal(outcome.timedOut, false);
+  assert.equal(outcome.value, "section body");
+});
+
+test("a zero deadline disables the bound rather than failing every section", async () => {
+  const released = Promise.withResolvers<void>();
+  const pending = runRecallSectionWithinDeadline({
+    deadlineMs: 0,
+    fallback: "degraded",
+    run: async () => {
+      await released.promise;
+      return "unbounded value";
+    },
+  });
+
+  // Two full event-loop turns: any 0ms timer would already have fired, so a
+  // still-pending section proves the bound was disabled rather than instant.
+  for (let turn = 0; turn < 2; turn += 1) {
+    const turned = Promise.withResolvers<void>();
+    setImmediate(turned.resolve);
+    await turned.promise;
+  }
+  released.resolve();
+
+  const outcome = await pending;
+  assert.equal(outcome.timedOut, false);
+  assert.equal(outcome.value, "unbounded value");
+});
+
+test("a caller abort propagates into the section signal", async () => {
+  const caller = new AbortController();
+  let cancelledDuringWork = false;
+
+  const pending = runRecallSectionWithinDeadline({
+    deadlineMs: 5_000,
+    fallback: "degraded",
+    parentSignal: caller.signal,
+    run: async (sectionSignal) => {
+      const cancelled = Promise.withResolvers<void>();
+      sectionSignal.addEventListener("abort", () => {
+        cancelledDuringWork = true;
+        cancelled.resolve();
+      });
+      await cancelled.promise;
+      return "cancelled";
+    },
+  });
+
+  caller.abort();
+  const outcome = await pending;
+
+  assert.equal(cancelledDuringWork, true);
+  assert.equal(outcome.timedOut, false);
+  assert.equal(outcome.value, "cancelled");
+});
+
+test("an already-aborted caller signal cancels the section before it does work", async () => {
+  const caller = new AbortController();
+  caller.abort();
+
+  const outcome = await runRecallSectionWithinDeadline({
+    deadlineMs: 5_000,
+    fallback: "degraded",
+    parentSignal: caller.signal,
+    run: async (sectionSignal) => {
+      assert.equal(sectionSignal.aborted, true);
+      return "observed";
+    },
+  });
+
+  assert.equal(outcome.value, "observed");
+});
+
+test("a section rejection surfaces to the caller instead of being swallowed", async () => {
+  await assert.rejects(
+    runRecallSectionWithinDeadline({
+      deadlineMs: 5_000,
+      fallback: "degraded",
+      run: async () => {
+        throw new Error("section exploded");
+      },
+    }),
+    /section exploded/,
+  );
+});
+
+test("the default core section budget is capped below the remaining request budget", () => {
+  // Both settings default to 75s. Taken at face value the section deadline could
+  // never fire first, so degradation would be unreachable on default config.
+  assert.equal(
+    resolveRecallCoreSectionDeadlineMs({
+      configuredCoreDeadlineMs: 75_000,
+      remainingOuterMs: 75_000,
+    }),
+    60_000,
+  );
+});
+
+test("the core section budget shrinks with the request budget already spent", () => {
+  // A section that starts 20s into a 75s request must finish inside the 55s left,
+  // not inside the original ceiling — otherwise the request is aborted wholesale.
+  assert.equal(
+    resolveRecallCoreSectionDeadlineMs({
+      configuredCoreDeadlineMs: 75_000,
+      remainingOuterMs: 55_000,
+    }),
+    44_000,
+  );
+});
+
+test("a request already over budget degrades its sections immediately", () => {
+  assert.equal(
+    resolveRecallCoreSectionDeadlineMs({
+      configuredCoreDeadlineMs: 75_000,
+      remainingOuterMs: 0,
+    }),
+    1,
+  );
+  assert.equal(
+    resolveRecallCoreSectionDeadlineMs({
+      configuredCoreDeadlineMs: 75_000,
+      remainingOuterMs: -5_000,
+    }),
+    1,
+  );
+});
+
+test("an explicitly lowered core section budget is honored exactly", () => {
+  assert.equal(
+    resolveRecallCoreSectionDeadlineMs({
+      configuredCoreDeadlineMs: 5_000,
+      remainingOuterMs: 75_000,
+    }),
+    5_000,
+  );
+});
+
+test("a zero budget on either side keeps the configured value", () => {
+  assert.equal(
+    resolveRecallCoreSectionDeadlineMs({ configuredCoreDeadlineMs: 0, remainingOuterMs: 75_000 }),
+    0,
+  );
+  // No request ceiling means nothing to reserve headroom against.
+  assert.equal(
+    resolveRecallCoreSectionDeadlineMs({ configuredCoreDeadlineMs: 75_000, remainingOuterMs: null }),
+    75_000,
+  );
+});
+
+test("a bounded core section resolves its budget from the budget left when it starts", async () => {
+  const metrics: RecallSectionMetric[] = [];
+  const clockMs = 1_000_000;
+  const runSection = createBoundedCoreSectionRunner({
+    configuredDeadlineMs: 75_000,
+    // 75s request that started 20s ago: 55s left, so the section gets 80% of 55s.
+    outerDeadlineAtMs: clockMs + 55_000,
+    record: (metric) => metrics.push(metric),
+    logger: { warn: () => {} },
+    now: () => clockMs,
+  });
+
+  await runSection("entityRetrieval", null as string | null, async () => "section body");
+
+  assert.equal(metrics[0]?.deadlineMs, 44_000);
+});
+
+test("a bounded core section degrades on cancellation rather than rejecting", async () => {
+  // The phase this section belongs to is awaited through a race the caller's
+  // abort also wins, so a rejection here would be an unhandled rejection.
+  const metrics: RecallSectionMetric[] = [];
+  const caller = new AbortController();
+  const runSection = createBoundedCoreSectionRunner({
+    configuredDeadlineMs: 5_000,
+    outerDeadlineAtMs: null,
+    parentSignal: caller.signal,
+    record: (metric) => metrics.push(metric),
+    logger: { warn: () => {} },
+  });
+
+  caller.abort();
+  const value = await runSection("artifacts", [] as string[], async (sectionSignal) => {
+    throwIfAborted(sectionSignal, "artifact search aborted");
+    return ["should not be reached"];
+  });
+
+  assert.deepEqual(value, []);
+  assert.equal(metrics[0]?.section, "artifacts");
+  assert.equal(metrics[0]?.success, false);
+  assert.match(metrics[0]?.timing ?? "", /^cancelled\(\d+ms\)$/);
+});
+
+test("a bounded core section still surfaces a non-abort failure", async () => {
+  const runSection = createBoundedCoreSectionRunner({
+    configuredDeadlineMs: 5_000,
+    outerDeadlineAtMs: null,
+    record: () => {},
+    logger: { warn: () => {} },
+  });
+
+  await assert.rejects(
+    runSection("artifacts", [] as string[], async () => {
+      throw new Error("artifact tier unreadable");
+    }),
+    /artifact tier unreadable/,
+  );
+});
+
+test("yieldToEventLoop lets a pending timer run", async () => {
+  // The guarantee corpus scans depend on: without a yield the timer that carries
+  // a section deadline cannot fire while the scan holds the loop.
+  let timerFired = false;
+  const timer = setTimeout(() => {
+    timerFired = true;
+  }, 0);
+
+  await yieldToEventLoop();
+  await yieldToEventLoop();
+  clearTimeout(timer);
+
+  assert.equal(timerFired, true);
+});