@remnic/core 9.47.1 → 9.48.0

package/src/artifact-search.test.ts

@@ -0,0 +1,110 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { selectArtifactMatches, tokenizeArtifactSearchText } from "./artifact-search.js";
+import type { MemoryFile } from "./types.js";
+
+function artifact(id: string, content: string, tags: string[] = []): MemoryFile {
+  return {
+    path: `artifacts/${id}.md`,
+    content,
+    frontmatter: {
+      id,
+      category: "artifact",
+      created: "2026-08-01T00:00:00.000Z",
+      updated: "2026-08-01T00:00:00.000Z",
+      ...(tags.length > 0 ? { tags } : {}),
+    },
+  } as unknown as MemoryFile;
+}
+
+test("artifact scan ranks by query token overlap and applies the result cap", async () => {
+  const matches = await selectArtifactMatches(
+    async () => [
+      artifact("a", "the deploy runbook covers rollback"),
+      artifact("b", "unrelated grocery list"),
+      artifact("c", "runbook", ["deploy", "rollback"]),
+    ],
+    "deploy rollback runbook",
+    2,
+  );
+
+  assert.deepEqual(
+    matches.map((m) => m.frontmatter.id),
+    ["a", "c"],
+  );
+});
+
+test("a stopword-only query never reads the artifact tier", async () => {
+  // Reading the tier is a full recursive filesystem scan on a cold cache, so a
+  // query that cannot match anything must not pay for it (issue #2291).
+  let loaded = 0;
+  const matches = await selectArtifactMatches(
+    async () => {
+      loaded += 1;
+      return [artifact("a", "the and of")];
+    },
+    "the and of",
+    5,
+  );
+
+  assert.deepEqual(matches, []);
+  assert.equal(loaded, 0);
+});
+
+test("an already-aborted caller never reads the artifact tier", async () => {
+  const aborted = new AbortController();
+  aborted.abort();
+  let loaded = 0;
+
+  await assert.rejects(
+    selectArtifactMatches(
+      async () => {
+        loaded += 1;
+        return [artifact("a", "deploy runbook")];
+      },
+      "deploy runbook",
+      5,
+      { abortSignal: aborted.signal },
+    ),
+    (err: unknown) => err instanceof Error && err.name === "AbortError",
+  );
+  assert.equal(loaded, 0);
+});
+
+test("artifact scan stops at the caller's signal instead of scanning the tier", async () => {
+  // Yield/abort checkpoints land every 256 documents, so the corpus must cross
+  // that boundary for cancellation to be observable mid-scan (issue #2291).
+  const artifacts = Array.from({ length: 600 }, (_unused, index) =>
+    artifact(`doc-${index}`, "deploy runbook rollback"),
+  );
+  const aborted = new AbortController();
+  let scannedBeforeAbort = 0;
+  const artifactsWithProbe = artifacts.map((memory) => ({
+    get content() {
+      scannedBeforeAbort += 1;
+      if (scannedBeforeAbort === 300) aborted.abort();
+      return memory.content;
+    },
+    path: memory.path,
+    frontmatter: memory.frontmatter,
+  })) as unknown as MemoryFile[];
+
+  await assert.rejects(
+    selectArtifactMatches(async () => artifactsWithProbe, "deploy runbook", 5, {
+      abortSignal: aborted.signal,
+    }),
+    (err: unknown) => err instanceof Error && err.name === "AbortError",
+  );
+  assert.ok(
+    scannedBeforeAbort < artifacts.length,
+    `scan stopped early (read ${scannedBeforeAbort} of ${artifacts.length})`,
+  );
+});
+
+test("artifact tokenizer drops stopwords and single characters", () => {
+  assert.deepEqual(tokenizeArtifactSearchText("The a deploy-runbook v2"), [
+    "deploy",
+    "runbook",
+    "v2",
+  ]);
+});

package/src/artifact-search.ts

@@ -0,0 +1,115 @@
+/**
+ * Verbatim-artifact keyword scan — extracted from `storage.ts` so the scan can
+ * be made interruptible without growing that file (issue #2291).
+ *
+ * The scan re-tokenizes every stored artifact, and `fetchActiveArtifactsForNamespace`
+ * repeats it with a growing fetch limit. On a large artifact tier that is long
+ * enough to starve the event loop, which would stop a recall section deadline
+ * from ever firing — so the loop yields periodically and observes cancellation.
+ */
+
+import { throwIfAborted } from "./abort-error.js";
+import { yieldToEventLoop } from "./recall-qos.js";
+import type { MemoryFile } from "./types.js";
+
+const ARTIFACT_SEARCH_STOPWORDS = new Set([
+  "a",
+  "an",
+  "and",
+  "are",
+  "as",
+  "at",
+  "be",
+  "but",
+  "by",
+  "for",
+  "from",
+  "has",
+  "have",
+  "i",
+  "in",
+  "is",
+  "it",
+  "of",
+  "on",
+  "or",
+  "that",
+  "the",
+  "this",
+  "to",
+  "was",
+  "were",
+  "with",
+]);
+
+/**
+ * Artifacts scored between two yields. Small enough that a deadline observes a
+ * sub-millisecond delay on a normal tier, large enough that the yields cost
+ * nothing measurable relative to tokenizing that many documents.
+ */
+const ARTIFACT_SCAN_YIELD_INTERVAL = 256;
+
+export function tokenizeArtifactSearchText(input: string): string[] {
+  return input
+    .toLowerCase()
+    .split(/[^a-z0-9]+/i)
+    .map((t) => t.trim())
+    .filter((t) => t.length >= 2)
+    .filter((t) => !ARTIFACT_SEARCH_STOPWORDS.has(t));
+}
+
+export interface ArtifactSearchOptions {
+  abortSignal?: AbortSignal;
+}
+
+/**
+ * Rank the artifact tier by how many query tokens each document contains, best
+ * first.
+ *
+ * `loadArtifacts` is a thunk rather than a value so the two cheap early exits —
+ * an already-cancelled caller, and a query with no searchable tokens — happen
+ * BEFORE the tier is read. Reading it can mean a full recursive filesystem scan
+ * on a cold cache, which a stopword-only query must never pay for.
+ *
+ * Yields to the event loop every `ARTIFACT_SCAN_YIELD_INTERVAL` documents and
+ * checks the caller's signal there, so a pending recall section deadline can fire
+ * mid-scan and an abandoned recall stops scanning instead of running to the end.
+ */
+export async function selectArtifactMatches(
+  loadArtifacts: () => Promise<MemoryFile[]>,
+  query: string,
+  maxResults: number,
+  options: ArtifactSearchOptions = {},
+): Promise<MemoryFile[]> {
+  throwIfAborted(options.abortSignal, "artifact search aborted");
+  const tokens = tokenizeArtifactSearchText(query);
+  if (tokens.length === 0) return [];
+
+  const artifacts = await loadArtifacts();
+  // The scan below never awaits real I/O, so this is the last chance to observe a
+  // signal that fired while the tier was being read.
+  throwIfAborted(options.abortSignal, "artifact search aborted");
+  const hits: Array<{ score: number; memory: MemoryFile }> = [];
+  let scanned = 0;
+  for (const memory of artifacts) {
+    scanned += 1;
+    if (scanned % ARTIFACT_SCAN_YIELD_INTERVAL === 0) {
+      await yieldToEventLoop();
+      throwIfAborted(options.abortSignal, "artifact search aborted");
+    }
+    const indexedTokens = new Set(
+      tokenizeArtifactSearchText(`${memory.content} ${(memory.frontmatter.tags ?? []).join(" ")}`),
+    );
+    let score = 0;
+    for (const token of tokens) {
+      if (indexedTokens.has(token)) score += 1;
+    }
+    if (score > 0) {
+      hits.push({ score, memory });
+    }
+  }
+  // Ties keep corpus order: the comparator returns 0 for equal scores rather
+  // than an arbitrary sign, so repeated scans produce the same top-N.
+  hits.sort((a, b) => b.score - a.score);
+  return hits.slice(0, maxResults).map((h) => h.memory);
+}

package/src/entity-recall-cancellation.ts

@@ -0,0 +1,58 @@
+/**
+ * Cancellation checkpoints for entity recall (issue #2291).
+ *
+ * Entity recall reads every entity and memory file for the recalled namespaces and
+ * then makes several synchronous passes over the result. Both halves need the same
+ * two things at every boundary, in the same shape, with the same abort message —
+ * so they live here rather than being restated at each of a dozen call sites.
+ */
+
+import { throwIfAborted } from "./abort-error.js";
+import { yieldToEventLoop } from "./recall-qos.js";
+
+const ABORT_MESSAGE = "entity recall aborted";
+
+/**
+ * Entities/memories processed between two yields during an index build. Matches
+ * the artifact scan's interval: large enough that the yields are free relative to
+ * the work, small enough that a section deadline is observed promptly.
+ */
+export const ENTITY_SCAN_YIELD_INTERVAL = 256;
+
+/** Stop at the caller's signal without yielding. */
+export function checkEntityRecallAbort(abortSignal?: AbortSignal): void {
+  throwIfAborted(abortSignal, ABORT_MESSAGE);
+}
+
+/**
+ * Hand the loop a macrotask, then stop if the caller gave up while it ran.
+ *
+ * A section deadline is a timer, so a synchronous pass over the corpus must be
+ * preceded by one of these or the deadline cannot fire during it.
+ */
+export async function yieldEntityRecallScan(abortSignal?: AbortSignal): Promise<void> {
+  await yieldToEventLoop();
+  throwIfAborted(abortSignal, ABORT_MESSAGE);
+}
+
+/**
+ * The same yield/abort pair, but only every `ENTITY_SCAN_YIELD_INTERVAL` items —
+ * for loops where yielding per item would cost more than the work itself.
+ */
+export async function yieldEntityRecallScanEvery(
+  processed: number,
+  abortSignal?: AbortSignal,
+): Promise<void> {
+  if (processed % ENTITY_SCAN_YIELD_INTERVAL !== 0) return;
+  await yieldEntityRecallScan(abortSignal);
+}
+
+/**
+ * Exit an entity-recall path with "no section", unless the caller has actually
+ * cancelled — in which case throw, so the bounded runner records a breach rather
+ * than a successful empty section.
+ */
+export function entityRecallSectionAbsent(abortSignal?: AbortSignal): null {
+  throwIfAborted(abortSignal, ABORT_MESSAGE);
+  return null;
+}

package/src/entity-retrieval.ts

@@ -8,6 +8,12 @@ import { compareEntityTimestamps, normalizeEntityName, type StorageManager } fro
 import { normalizeEntityText, resolveRequestedEntitySectionKeys } from "./entity-schema.js";
 import type { EntityStructuredSection, MemoryFile, PluginConfig, TranscriptEntry } from "./types.js";
 import { containsPhrase } from "./entity-retrieval-boundaries.js";
+import {
+  checkEntityRecallAbort,
+  entityRecallSectionAbsent,
+  yieldEntityRecallScan,
+  yieldEntityRecallScanEvery,
+} from "./entity-recall-cancellation.js";
 
 const ENTITY_INDEX_VERSION = 3;
 const RECENT_TRANSCRIPT_LOOKBACK_HOURS = 24;
@@ -81,6 +87,13 @@ export interface BuildEntityRecallSectionOptions {
   maxRelatedEntities: number;
   maxChars: number;
   transcriptEntries: TranscriptEntry[];
+  /**
+   * Cancels the build at its next checkpoint (issue #2291).  Entity recall reads
+   * every entity and memory file across the recalled namespaces, which is
+   * minutes of work on a large or slow store; without this the scan kept running
+   * after the recall that asked for it had been abandoned.
+   */
+  abortSignal?: AbortSignal;
 }
 
 function tokenize(value: string): string[] {
@@ -523,7 +536,9 @@ async function buildEntityMentionIndex(
   namespaceStorage?: (namespace: string) => Promise<StorageManager>,
   nativeChunksOverride?: NativeKnowledgeChunk[],
   resolvedStorages?: StorageManager[],
+  abortSignal?: AbortSignal,
 ): Promise<EntityMentionIndex> {
+  checkEntityRecallAbort(abortSignal);
   const storages = resolvedStorages ?? await resolveEntityIndexStorages(
     storage,
     config,
@@ -539,6 +554,9 @@ async function buildEntityMentionIndex(
     Promise.all(storages.map((scopedStorage) => scopedStorage.readAllMemories())),
     nativeChunksOverride ? Promise.resolve(nativeChunksOverride) : readNativeChunks(config, recallNamespaces),
   ]);
+  // The bulk reads above cannot be interrupted mid-flight; stop before spending
+  // the (larger) indexing pass over their results.
+  checkEntityRecallAbort(abortSignal);
   // Pair each entity with the storage that owns it (#1534): canonical ids must
   // reflect that store's aliases and historical migration mappings, never another
   // namespace's.
@@ -547,8 +565,14 @@ async function buildEntityMentionIndex(
   );
   const memories = memorySets.flat();
 
+  // The two passes over the corpus below never await real I/O, so without an
+  // explicit yield they hold the event loop and no recall section deadline can
+  // fire during them (issue #2291).
+  let scanned = 0;
   const entities = new Map<string, EntityMentionIndexEntry>();
   for (const { entity, storage: entityStorage } of entityRecords) {
+    scanned += 1;
+    await yieldEntityRecallScanEvery(scanned, abortSignal);
     const canonicalId =
       typeof entityStorage.normalizeEntityName === "function"
         ? entityStorage.normalizeEntityName(entity.name, entity.type)
@@ -589,7 +613,10 @@ async function buildEntityMentionIndex(
     });
   }
 
+  scanned = 0;
   for (const memory of memories) {
+    scanned += 1;
+    await yieldEntityRecallScanEvery(scanned, abortSignal);
     const entityRef = typeof memory.frontmatter.entityRef === "string" ? memory.frontmatter.entityRef : "";
     if (!entityRef) continue;
     const entry = entities.get(entityRef);
@@ -600,6 +627,11 @@ async function buildEntityMentionIndex(
     }
   }
 
+  // Each remaining phase is another synchronous pass over the whole index
+  // (alias map, native-chunk merge, sort, two full serializations). Yield between
+  // them so a section deadline can fire between passes instead of only after all
+  // of them (issue #2291).
+  await yieldEntityRecallScan(abortSignal);
   const aliasIndex = buildAliasIndex([...entities.values()]);
   for (const chunk of nativeChunks) {
     const existingPseudo = entities.get(nativePseudoCanonicalId(chunk));
@@ -620,8 +652,11 @@ async function buildEntityMentionIndex(
     entities.set(pseudoEntry.canonicalId, pseudoEntry);
   }
 
+  await yieldEntityRecallScan(abortSignal);
   const sortedEntities = [...entities.values()].sort((left, right) => left.name.localeCompare(right.name));
+  await yieldEntityRecallScan(abortSignal);
   const previousEntities = previousIndex ? JSON.stringify(previousIndex.entities) : "";
+  await yieldEntityRecallScan(abortSignal);
   const nextEntities = JSON.stringify(sortedEntities);
   const entityStatusVersionAfter = shouldPersistIndex ? storage.getMemoryStatusVersion() : undefined;
   const canPersistIndex = shouldPersistIndex && entityStatusVersionBefore === entityStatusVersionAfter;
@@ -635,6 +670,10 @@ async function buildEntityMentionIndex(
     entities: sortedEntities,
   };
   if (canPersistIndex) {
+    // A build the caller has abandoned must not leave a persisted index behind:
+    // the write is I/O the timed-out recall no longer needs, and the next recall
+    // is already competing for the same disk.
+    checkEntityRecallAbort(abortSignal);
     await writeEntityIndexState(storage, index);
   }
   return index;
@@ -1020,6 +1059,11 @@ function formatEntityHintSection(
 }
 
 export async function buildEntityRecallSection(options: BuildEntityRecallSectionOptions): Promise<string | null> {
+  // Checkpoints sit at the real I/O boundaries below (persisted-index read,
+  // native-chunk read, storage resolution, index build).  The per-candidate
+  // formatting that follows never awaits real I/O, so a deadline that fires
+  // during a scan is observed here (issue #2291).
+  checkEntityRecallAbort(options.abortSignal);
   const prefixedMode = detectEntityQueryMode(options.query);
   const persistedIndex = prefixedMode
     ? null
@@ -1029,6 +1073,7 @@ export async function buildEntityRecallSection(options: BuildEntityRecallSection
       options.recallNamespaces,
       options.namespaceStorage,
     );
+  checkEntityRecallAbort(options.abortSignal);
   let nativeChunks: NativeKnowledgeChunk[] | undefined;
   if (
     !prefixedMode &&
@@ -1037,7 +1082,7 @@ export async function buildEntityRecallSection(options: BuildEntityRecallSection
   ) {
     nativeChunks = await readNativeChunks(options.config, options.recallNamespaces);
     if (!hasFreshNativeExplicitMention(persistedIndex, nativeChunks, options.query)) {
-      return null;
+      return entityRecallSectionAbsent(options.abortSignal);
     }
   }
   const namespaceScoped =
@@ -1066,6 +1111,7 @@ export async function buildEntityRecallSection(options: BuildEntityRecallSection
   let index: EntityMentionIndex | undefined = namespaceCacheKey
     ? namespaceEntityIndexCache.get(namespaceCacheKey)
     : undefined;
+  checkEntityRecallAbort(options.abortSignal);
   if (!index) {
     index = await buildEntityMentionIndex(
       options.storage,
@@ -1074,21 +1120,27 @@ export async function buildEntityRecallSection(options: BuildEntityRecallSection
       options.namespaceStorage,
       nativeChunks,
       resolvedStorages,
+      options.abortSignal,
     );
     if (namespaceCacheKey) rememberNamespaceEntityIndex(namespaceCacheKey, index);
   }
+  // Candidate resolution is another synchronous pass over every entity and alias,
+  // reached even on a cache hit that did no I/O at all — so yield here too, or a
+  // cached corpus could still hold the loop past the section budget (issue #2291).
+  await yieldEntityRecallScan(options.abortSignal);
   const explicitCandidates = resolveExplicitCandidates(index, options.query);
+  await yieldEntityRecallScan(options.abortSignal);
   const queryCandidates = prefixedMode
     ? explicitCandidates
     : resolveLanguageIndependentExplicitCandidates(index, options.query);
   const mode = prefixedMode ?? (queryCandidates.length > 0 ? "direct" : null);
-  if (!mode) return null;
+  if (!mode) return entityRecallSectionAbsent(options.abortSignal);
 
   const candidates = queryCandidates.length > 0
     ? queryCandidates
     : resolveRecentTurnCandidates(index, options.transcriptEntries, options.recentTurns);
 
-  if (candidates.length === 0) return null;
+  if (candidates.length === 0) return entityRecallSectionAbsent(options.abortSignal);
 
   const queryTokens = tokenize(options.query);
   const candidateLimit = queryCandidates.length === 0 && mode === "follow_up"
@@ -1121,7 +1173,9 @@ export async function buildEntityRecallSection(options: BuildEntityRecallSection
   );
 
   const section = formatEntityHintSection(enriched, queryTokens, mode, options.maxRelatedEntities, options.maxChars);
-  if (!section) return null;
+  // `entityRecallSectionAbsent` so a cancelled build is not recorded as a
+  // successful section that simply found no entities.
+  if (!section) return entityRecallSectionAbsent(options.abortSignal);
   return section;
 }
 

package/src/orchestration/namespace-read-fanout.ts

@@ -27,6 +27,7 @@ import { mergeArtifactRecallCandidates, tokenizeRecallQuery } from "./orchestrat
 import { qmdCollectionPathParts } from "./qmd-result-resolver.js";
 import { qmdCollectionNamespaceFromPrefix as computeQmdCollectionNamespaceFromPrefix } from "./orchestrator-namespace-scope.js";
 import { resolveNamespaceFromStorageDir } from "../scopes/scope-plan.js";
+import type { ArtifactRecallOptions } from "./recall-search-prefilter.js";
 import type { SearchBackend, SearchExecutionOptions, SearchQueryOptions } from "../search/port.js";
 import type { MemoryFile, PluginConfig, QmdSearchResult } from "../types.js";
 import {
@@ -41,6 +42,7 @@ export interface NamespaceReadFanoutDeps {
     namespace: string,
     prompt: string,
     targetCount: number,
+    options?: ArtifactRecallOptions,
   ): Promise<MemoryFile[]>;
   loadNamespaceStorageDirHintsFromCatalog(): void;
   readonly namespaceCatalog: NamespaceCatalog;
@@ -338,12 +340,13 @@ export class NamespaceReadFanoutCoordinator {
     prompt: string,
     recallNamespaces: string[],
     targetCount: number,
+    options: ArtifactRecallOptions = {},
   ): Promise<MemoryFile[]> {
     if (targetCount <= 0) return [];
     const namespaces = Array.from(new Set(recallNamespaces));
     const filteredByNamespace = await Promise.all(
       namespaces.map((namespace) =>
-        this.deps.fetchActiveArtifactsForNamespace(namespace, prompt, targetCount),
+        this.deps.fetchActiveArtifactsForNamespace(namespace, prompt, targetCount, options),
       ),
     );
 

package/src/orchestration/recall-internal-deps.ts

@@ -35,6 +35,7 @@ import type { VerifiedEpisodeResult } from "../verified-recall.js";
 import type { WorkProductLedgerSearchResult } from "../work-product-ledger.js";
 import type { GraphRecallRankedResult, GraphRecallShadowComparison } from "./graph-recall-coordinator.js";
 import type { RecallRerankCoordinator, RecallResultPartitionSink } from "./recall-rerank-coordinator.js";
+import type { ArtifactRecallOptions } from "./recall-search-prefilter.js";
 import type { RecallSectionAppendOptions, RecallSectionBuckets } from "./recall-section-coordinator.js";
 
 export interface RecallInternalDeps {
@@ -339,6 +340,7 @@ export interface RecallInternalDeps {
     prompt: string,
     recallNamespaces: string[],
     targetCount: number,
+    options?: ArtifactRecallOptions,
   ): Promise<MemoryFile[]>;
   recallAssemblyClockMs(): number;
   recordLastGraphRecallSnapshot(options: {