@remnic/core 9.45.5 → 9.46.0

package/src/orchestration/recall-internal.ts

@@ -102,6 +102,7 @@ import {
   type RecallResultPartitionSink,
 } from "./recall-rerank-coordinator.js";
 import type { RecallInternalDeps } from "./recall-internal-deps.js";
+import { resolveCompositeProfileStorage } from "./recall-profile-storage.js";
 
 export class RecallInternalCoordinator {
   constructor(
@@ -686,154 +687,10 @@ export class RecallInternalCoordinator {
     const profileStorages = await Promise.all(
       profileStorageNamespaces.map((namespace) => this.deps.storageRouter.storageFor(namespace)),
     );
-    const emptyProfileStorage = new Proxy(
-      { dir: path.join(this.deps.config.memoryDir, ".empty-scope-profile") } as any,
-      {
-        get(target, prop: string | symbol) {
-          if (prop in target) return target[prop];
-          if (prop === "readProfile") return async () => "";
-          if (
-            prop === "readQuestions" ||
-            prop === "listEntityNames" ||
-            prop === "readContinuityIncidents"
-          )
-            return async () => [];
-          if (
-            prop === "readIdentityAnchor" ||
-            prop === "readIdentityImprovementLoops"
-          )
-            return async () => "";
-          if (prop === "readEntity" || prop === "readMemoryByPath")
-            return async () => null;
-          return async () => [];
-        },
-      },
-    );
-    const profileStorage =
-      profileStorages.length <= 1
-        ? profileStorages[0] ?? emptyProfileStorage
-        : new Proxy(profileStorages[0] as any, {
-            get(target, prop: string | symbol) {
-              if (prop === "readProfile") {
-                return async () => {
-                  for (const storage of profileStorages) {
-                    const profile = await storage.readProfile();
-                    if (profile.trim().length > 0) return profile;
-                  }
-                  return "";
-                };
-              }
-              if (prop === "readQuestions") {
-                return async (...args: any[]) => {
-                  const merged: any[] = [];
-                  const seen = new Set<string>();
-                  const priorityOf = (question: any): number => {
-                    const priority = Number(question?.priority ?? 0);
-                    return Number.isFinite(priority) ? priority : 0;
-                  };
-                  for (const storage of profileStorages) {
-                    const questions = await (storage.readQuestions as any)(...args);
-                    for (const question of questions) {
-                      const key = typeof question === "string" ? question : JSON.stringify(question);
-                      if (seen.has(key)) continue;
-                      seen.add(key);
-                      merged.push(question);
-                    }
-                  }
-                  return merged.sort(
-                    (left, right) =>
-                      priorityOf(right) - priorityOf(left) ||
-                      String(left?.id ?? "").localeCompare(String(right?.id ?? "")),
-                  );
-                };
-              }
-              if (prop === "readIdentityAnchor") {
-                return async () => {
-                  for (const storage of profileStorages) {
-                    const anchor = (await storage.readIdentityAnchor()) ?? "";
-                    if (anchor.trim().length > 0) return anchor;
-                  }
-                  return "";
-                };
-              }
-              if (prop === "readIdentityImprovementLoops") {
-                return async () => {
-                  const sections: string[] = [];
-                  const seen = new Set<string>();
-                  for (const storage of profileStorages) {
-                    const loops = ((await storage.readIdentityImprovementLoops()) ?? "").trim();
-                    if (!loops || seen.has(loops)) continue;
-                    seen.add(loops);
-                    sections.push(loops);
-                  }
-                  return sections.join("\n\n");
-                };
-              }
-              if (prop === "readContinuityIncidents") {
-                return async (...args: any[]) => {
-                  const limit = typeof args[0] === "number" && Number.isFinite(args[0]) ? Math.max(0, args[0]) : undefined;
-                  const incidents: any[] = [];
-                  const seen = new Set<string>();
-                  const incidentTime = (incident: any): number => {
-                    const raw = incident?.updatedAt ?? incident?.openedAt ?? incident?.createdAt;
-                    const parsed = typeof raw === "string" ? Date.parse(raw) : Number.NaN;
-                    return Number.isFinite(parsed) ? parsed : 0;
-                  };
-                  for (const storage of profileStorages) {
-                    for (const incident of await (storage.readContinuityIncidents as any)(...args)) {
-                      const key = JSON.stringify(incident);
-                      if (seen.has(key)) continue;
-                      seen.add(key);
-                      incidents.push(incident);
-                    }
-                  }
-                  incidents.sort(
-                    (left, right) =>
-                      incidentTime(right) - incidentTime(left) ||
-                      String(left?.id ?? "").localeCompare(String(right?.id ?? "")),
-                  );
-                  return limit === undefined ? incidents : incidents.slice(0, limit);
-                };
-              }
-              if (prop === "listEntityNames") {
-                return async (...args: any[]) => {
-                  const names = new Set<string>();
-                  for (const storage of profileStorages) {
-                    for (const name of await (storage.listEntityNames as any)(...args)) names.add(name);
-                  }
-                  return [...names];
-                };
-              }
-              if (prop === "readEntity" || prop === "readMemoryByPath") {
-                return async (...args: any[]) => {
-                  for (const storage of profileStorages) {
-                    const value = await (storage as any)[prop](...args);
-                    if (value) return value;
-                  }
-                  return null;
-                };
-              }
-              if (prop === "readAllMemories") {
-                return async (...args: any[]) => {
-                  const memories: any[] = [];
-                  const seen = new Set<string>();
-                  for (const storage of profileStorages) {
-                    for (const memory of await (storage.readAllMemories as any)(...args)) {
-                      const key = String(memory?.path ?? memory?.frontmatter?.id ?? JSON.stringify(memory));
-                      if (seen.has(key)) continue;
-                      seen.add(key);
-                      memories.push(memory);
-                    }
-                  }
-                  return memories;
-                };
-              }
-              return target[prop];
-            },
-          });
-    const profileStorageDirs = Array.from(
-      new Set(profileStorages.map((storage) => storage.dir).filter((dir): dir is string => typeof dir === "string" && dir.length > 0)),
-    );
+    const { profileStorage, profileStorageDirs } = resolveCompositeProfileStorage({
+      profileStorages,
+      memoryDir: this.deps.config.memoryDir,
+    });
 
     // --- Phase 1: Launch ALL independent data fetches in parallel ---
     throwIfRecallAborted(options.abortSignal);

package/src/orchestration/recall-profile-storage.ts

@@ -0,0 +1,194 @@
+/**
+ * Composite profile storage proxy for recall orchestration — extracted from
+ * recall-internal.ts (issue #1526, seam 18).
+ */
+
+import path from "node:path";
+import type { StorageManager } from "../index.js";
+
+export interface CompositeProfileStorageOptions {
+  profileStorages: StorageManager[];
+  memoryDir: string;
+}
+
+export interface CompositeProfileStorageResult {
+  profileStorage: StorageManager;
+  profileStorageDirs: string[];
+}
+
+/**
+ * Creates a composite profile storage proxy over multiple namespace storage instances.
+ * When multiple storages exist, methods aggregate results across all instances.
+ * When storages is empty, returns a safe empty fallback storage.
+ */
+export function resolveCompositeProfileStorage(
+  options: CompositeProfileStorageOptions,
+): CompositeProfileStorageResult {
+  const { profileStorages, memoryDir } = options;
+
+  const emptyProfileStorage = new Proxy(
+    { dir: path.join(memoryDir, ".empty-scope-profile") } as unknown as StorageManager,
+    {
+      get(target, prop: string | symbol) {
+        if (prop in target) return Reflect.get(target, prop);
+        if (prop === "readProfile") return async () => "";
+        if (
+          prop === "readQuestions" ||
+          prop === "listEntityNames" ||
+          prop === "readContinuityIncidents"
+        )
+          return async () => [];
+        if (
+          prop === "readIdentityAnchor" ||
+          prop === "readIdentityImprovementLoops"
+        )
+          return async () => "";
+        if (prop === "readEntity" || prop === "readMemoryByPath")
+          return async () => null;
+        return async () => [];
+      },
+    },
+  );
+
+  const profileStorage =
+    profileStorages.length <= 1
+      ? profileStorages[0] ?? emptyProfileStorage
+      : new Proxy(profileStorages[0] as StorageManager, {
+          get(target, prop: string | symbol) {
+            if (prop === "readProfile") {
+              return async () => {
+                for (const storage of profileStorages) {
+                  const profile = await storage.readProfile();
+                  if (profile.trim().length > 0) return profile;
+                }
+                return "";
+              };
+            }
+            if (prop === "readQuestions") {
+              return async (...args: unknown[]) => {
+                const merged: unknown[] = [];
+                const seen = new Set<string>();
+                const priorityOf = (question: unknown): number => {
+                  const priority = Number((question as Record<string, unknown>)?.priority ?? 0);
+                  return Number.isFinite(priority) ? priority : 0;
+                };
+                for (const storage of profileStorages) {
+                  const readFn = (storage as unknown as Record<string, (...a: unknown[]) => Promise<unknown[]>>).readQuestions;
+                  const questions = await readFn.call(storage, ...args);
+                  for (const question of questions) {
+                    const key = typeof question === "string" ? question : JSON.stringify(question);
+                    if (seen.has(key)) continue;
+                    seen.add(key);
+                    merged.push(question);
+                  }
+                }
+                return merged.sort(
+                  (left, right) =>
+                    priorityOf(right) - priorityOf(left) ||
+                    String((left as Record<string, unknown>)?.id ?? "").localeCompare(String((right as Record<string, unknown>)?.id ?? "")),
+                );
+              };
+            }
+            if (prop === "readIdentityAnchor") {
+              return async () => {
+                for (const storage of profileStorages) {
+                  const anchor = (await storage.readIdentityAnchor()) ?? "";
+                  if (anchor.trim().length > 0) return anchor;
+                }
+                return "";
+              };
+            }
+            if (prop === "readIdentityImprovementLoops") {
+              return async () => {
+                const sections: string[] = [];
+                const seen = new Set<string>();
+                for (const storage of profileStorages) {
+                  const loops = ((await storage.readIdentityImprovementLoops()) ?? "").trim();
+                  if (!loops || seen.has(loops)) continue;
+                  seen.add(loops);
+                  sections.push(loops);
+                }
+                return sections.join("\n\n");
+              };
+            }
+            if (prop === "readContinuityIncidents") {
+              return async (...args: unknown[]) => {
+                const limit = typeof args[0] === "number" && Number.isFinite(args[0]) ? Math.max(0, args[0]) : undefined;
+                const incidents: unknown[] = [];
+                const seen = new Set<string>();
+                const incidentTime = (incident: unknown): number => {
+                  const incObj = incident as Record<string, unknown>;
+                  const raw = incObj?.updatedAt ?? incObj?.openedAt ?? incObj?.createdAt;
+                  const parsed = typeof raw === "string" ? Date.parse(raw) : Number.NaN;
+                  return Number.isFinite(parsed) ? parsed : 0;
+                };
+                for (const storage of profileStorages) {
+                  const readFn = (storage as unknown as Record<string, (...a: unknown[]) => Promise<unknown[]>>).readContinuityIncidents;
+                  for (const incident of await readFn.call(storage, ...args)) {
+                    const key = JSON.stringify(incident);
+                    if (seen.has(key)) continue;
+                    seen.add(key);
+                    incidents.push(incident);
+                  }
+                }
+                incidents.sort(
+                  (left, right) =>
+                    incidentTime(right) - incidentTime(left) ||
+                    String((left as Record<string, unknown>)?.id ?? "").localeCompare(String((right as Record<string, unknown>)?.id ?? "")),
+                );
+                return limit === undefined ? incidents : incidents.slice(0, limit);
+              };
+            }
+            if (prop === "listEntityNames") {
+              return async (...args: unknown[]) => {
+                const names = new Set<string>();
+                for (const storage of profileStorages) {
+                  const listFn = (storage as unknown as Record<string, (...a: unknown[]) => Promise<string[]>>).listEntityNames;
+                  for (const name of await listFn.call(storage, ...args)) names.add(name);
+                }
+                return [...names];
+              };
+            }
+            if (prop === "readEntity" || prop === "readMemoryByPath") {
+              return async (...args: unknown[]) => {
+                for (const storage of profileStorages) {
+                  const methodFn = (storage as unknown as Record<string, (...a: unknown[]) => Promise<unknown>>)[prop as string];
+                  if (methodFn) {
+                    const value = await methodFn.call(storage, ...args);
+                    if (value) return value;
+                  }
+                }
+                return null;
+              };
+            }
+            if (prop === "readAllMemories") {
+              return async (...args: unknown[]) => {
+                const memories: unknown[] = [];
+                const seen = new Set<string>();
+                for (const storage of profileStorages) {
+                  const readFn = (storage as unknown as Record<string, (...a: unknown[]) => Promise<unknown[]>>).readAllMemories;
+                  for (const memory of await readFn.call(storage, ...args)) {
+                    const memObj = memory as Record<string, unknown>;
+                    const key = String(memObj?.path ?? (memObj?.frontmatter as Record<string, unknown>)?.id ?? JSON.stringify(memory));
+                    if (seen.has(key)) continue;
+                    seen.add(key);
+                    memories.push(memory);
+                  }
+                }
+                return memories;
+              };
+            }
+            return Reflect.get(target, prop);
+          },
+        });
+
+  const profileStorageDirs = Array.from(
+    new Set(
+      profileStorages
+        .map((storage) => storage.dir)
+        .filter((dir): dir is string => typeof dir === "string" && dir.length > 0),
+    ),
+  );
+
+  return { profileStorage, profileStorageDirs };
+}

package/src/orchestration/recall-search-pipeline.ts

@@ -34,7 +34,13 @@ import { RerankCache, rerankLocalOrNoop, reorderByRankedKeys } from "../rerank.j
 import type { SearchBackend, SearchDegradation, SearchExecutionOptions, SearchQueryOptions } from "../search/port.js";
 import { SecureStoreLockedError } from "../secure-store/index.js";
 import { isPathInsideStorageRoot } from "../storage-paths.js";
-import { extractTagsFromPrompt, isTemporalQuery, queryByDateRangeAsync, queryByTagsAsync, readIndexSnapshotAsync, recencyWindowFromPrompt, resolvePromptTagPrefilterAsync } from "../temporal-index.js";
+import { extractTagsFromPrompt, isTemporalQuery, queryByDateRangeAsync, queryByTagsAsync, readIndexSnapshotAsync, recencyWindowFromPrompt } from "../temporal-index.js";
+import {
+  buildQueryAwarePrefilter as buildQueryAwarePrefilterHelper,
+  fetchActiveArtifactsForNamespace as fetchActiveArtifactsForNamespaceHelper,
+  searchEmbeddingFallback as searchEmbeddingFallbackHelper,
+  searchQueryAwareFallback as searchQueryAwareFallbackHelper,
+} from "./recall-search-prefilter.js";
 import { shouldFilterSupersededFromRecall } from "../temporal-supersession.js";
 import { isValidAsOf, isValidityExpiredNow } from "../temporal-validity.js";
 import type { TrustStageResultItem } from "../trust-score-stage.js";
@@ -49,7 +55,6 @@ import {
 } from "../recall-memory-map.js";
 import { type UtilityRuntimeValues, applyUtilityRankingRuntimeDelta } from "../utility-runtime.js";
 import {
-  computeArtifactCandidateFetchLimit,
   lifecycleRecallScoreAdjustment,
   shouldFilterLifecycleRecallCandidate,
   computeQmdHybridFetchLimit,
@@ -247,158 +252,17 @@ export class RecallSearchPipelineCoordinator {
     prompt: string,
     targetCount: number,
   ): Promise<MemoryFile[]> {
-    const storage = await this.deps.storageRouter.storageFor(namespace);
-    let fetchLimit = computeArtifactCandidateFetchLimit(targetCount);
-    const maxFetchLimit = Math.min(800, Math.max(fetchLimit, targetCount * 8));
-    const MAX_ATTEMPTS = 4;
-    let bestFiltered: MemoryFile[] = [];
-
-    for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt += 1) {
-      const rawResults = await storage.searchArtifacts(prompt, fetchLimit);
-      const sourceIds = Array.from(
-        new Set(
-          rawResults
-            .map((a) => a.frontmatter.sourceMemoryId)
-            .filter(
-              (id): id is string => typeof id === "string" && id.length > 0,
-            ),
-        ),
-      );
-      const sourceStatus =
-        sourceIds.length > 0
-          ? await this.deps.resolveArtifactSourceStatuses(storage, sourceIds)
-          : new Map<string, "active" | "superseded" | "archived" | "missing">();
-
-      const filtered: MemoryFile[] = [];
-      for (const artifact of rawResults) {
-        const sourceId = artifact.frontmatter.sourceMemoryId;
-        if (!sourceId) {
-          filtered.push(artifact);
-          if (filtered.length >= targetCount) break;
-          continue;
-        }
-        const status = sourceStatus.get(sourceId) ?? "missing";
-        if (status !== "active") continue;
-        filtered.push(artifact);
-        if (filtered.length >= targetCount) break;
-      }
-
-      if (filtered.length >= targetCount) return filtered.slice(0, targetCount);
-      if (filtered.length > bestFiltered.length) {
-        bestFiltered = filtered;
-      }
-      if (rawResults.length === 0) return filtered;
-      if (rawResults.length < fetchLimit && filtered.length > 0)
-        return filtered;
-      if (fetchLimit >= maxFetchLimit) return filtered;
-
-      const growth = Math.max(targetCount * 2, 12);
-      fetchLimit = Math.min(maxFetchLimit, fetchLimit + growth);
-    }
-
-    return bestFiltered;
+    return fetchActiveArtifactsForNamespaceHelper(this.deps, namespace, prompt, targetCount);
   }
 
   async buildQueryAwarePrefilter(
     prompt: string,
     recallNamespaces: string[],
   ): Promise<QueryAwarePrefilter> {
-    if (!resolveIndexingCapabilities(this.deps.config).queryAwareIndexing || !prompt.trim()) {
-      return {
-        candidatePaths: null,
-        temporalFromDate: null,
-        matchedTags: [],
-        expandedTags: [],
-        combination: "none",
-        filteredToFullSearch: false,
-      };
-    }
-
-    const temporalFromDate = isTemporalQuery(prompt)
-      ? recencyWindowFromPrompt(prompt, Date.now())
-      : null;
-    // Read the temporal + tag prefilter indexes as ONE consistent snapshot
-    // (issue #1911, Codex Medium) so a concurrent async index mutation can't be
-    // observed half-applied (new temporal row, tag membership not yet written),
-    // which would drop the current memory from the tag prefilter.
-    const [rawTemporal, tagSignals] = await readIndexSnapshotAsync(
-      this.deps.config.memoryDir,
-      () =>
-        Promise.all([
-          temporalFromDate
-            ? queryByDateRangeAsync(this.deps.config.memoryDir, temporalFromDate)
-            : Promise.resolve<Set<string> | null>(null),
-          resolvePromptTagPrefilterAsync(this.deps.config.memoryDir, prompt).catch(
-            () => ({
-              matchedTags: extractTagsFromPrompt(prompt),
-              expandedTags: extractTagsFromPrompt(prompt),
-              paths: null,
-            }),
-          ),
-        ]),
-    );
-
-    const temporalCandidates = this.deps.scopeQueryAwarePaths(
-      rawTemporal,
-      recallNamespaces,
-    );
-    const tagCandidates = this.deps.scopeQueryAwarePaths(
-      tagSignals.paths,
-      recallNamespaces,
-    );
-    const maxCandidates = this.deps.config.queryAwareIndexingMaxCandidates;
-
-    let candidatePaths: Set<string> | null = null;
-    let combination: QueryAwarePrefilter["combination"] = "none";
-    let filteredToFullSearch = false;
-
-    if (
-      tagSignals.matchedTags.length > 0 &&
-      tagCandidates !== null &&
-      tagCandidates.size === 0
-    ) {
-      candidatePaths = tagCandidates;
-      combination = "tag";
-    } else if (temporalCandidates !== null && tagCandidates !== null) {
-      const intersection = new Set(
-        Array.from(temporalCandidates).filter((memoryPath) =>
-          tagCandidates.has(memoryPath),
-        ),
-      );
-      if (intersection.size > 0) {
-        candidatePaths = intersection;
-        combination = "intersection";
-      } else {
-        candidatePaths = new Set([...temporalCandidates, ...tagCandidates]);
-        combination = "union";
-      }
-    } else if (temporalCandidates !== null) {
-      candidatePaths = temporalCandidates;
-      combination = "temporal";
-    } else if (tagCandidates !== null) {
-      candidatePaths = tagCandidates;
-      combination = "tag";
-    }
-
-    if (
-      candidatePaths &&
-      maxCandidates > 0 &&
-      candidatePaths.size > maxCandidates
-    ) {
-      filteredToFullSearch = true;
-      candidatePaths = null;
-    }
-
-    return {
-      candidatePaths,
-      temporalFromDate,
-      matchedTags: tagSignals.matchedTags,
-      expandedTags: tagSignals.expandedTags,
-      combination,
-      filteredToFullSearch,
-    };
+    return buildQueryAwarePrefilterHelper(this.deps, prompt, recallNamespaces);
   }
 
+
   async fetchQmdMemoryResultsWithArtifactTopUp(
     prompt: string,
     qmdFetchLimit: number,
@@ -654,26 +518,7 @@ export class RecallSearchPipelineCoordinator {
     query: string,
     limit: number,
   ): Promise<QmdSearchResult[]> {
-    if (!resolveMemoryLifecycleCapabilities(this.deps.config).embeddingFallback) return [];
-    if (!(await this.deps.embeddingFallback.isAvailable())) return [];
-    const hits = await this.deps.embeddingFallback.search(query, limit);
-    if (hits.length === 0) return [];
-
-    const results: QmdSearchResult[] = [];
-    for (const hit of hits) {
-      const fullPath = path.isAbsolute(hit.path)
-        ? hit.path
-        : path.join(this.deps.config.memoryDir, hit.path);
-      const memory = await this.deps.storage.readMemoryByPath(fullPath);
-      if (!memory) continue;
-      results.push({
-        docid: hit.id,
-        path: fullPath,
-        score: hit.score,
-        snippet: memory.content.slice(0, 400).replace(/\n/g, " "),
-      });
-    }
-    return results;
+    return searchEmbeddingFallbackHelper(this.deps, query, limit);
   }
 
   /**
@@ -686,24 +531,10 @@ export class RecallSearchPipelineCoordinator {
     queryAwarePrefilter?: QueryAwarePrefilter,
     abortSignal?: AbortSignal,
   ): Promise<QmdSearchResult[]> {
-    throwIfRecallAborted(abortSignal);
-    const cappedLimit = Math.max(0, limit);
-    if (cappedLimit === 0 || queryAwarePrefilter?.candidatePaths?.size === 0) return [];
-
-    const candidatePaths = queryAwarePrefilter?.candidatePaths;
-    const scopedSeedResults = (
-      candidatePaths?.size
-        ? await this.deps.searchScopedMemoryCandidates(
-            candidatePaths,
-            prompt,
-            candidatePaths.size,
-            { allowArchived: true },
-          )
-        : []
-    ).filter((result) => !isGenericRecallExcludedPath(result.path, this.deps.config, "qmd"));
-    return scopedSeedResults.slice(0, cappedLimit);
+    return searchQueryAwareFallbackHelper(this.deps, prompt, limit, queryAwarePrefilter, abortSignal);
   }
 
+
   async applyColdFallbackPipeline(options: {
     prompt: string;
     recallNamespaces: string[];