@remnic/core 9.7.15 → 9.9.0

package/src/orchestration/namespace-read-fanout.ts

@@ -25,6 +25,7 @@ import { NamespaceSearchRouter } from "../namespaces/search.js";
 import { NamespaceStorageRouter } from "../namespaces/storage.js";
 import { mergeArtifactRecallCandidates, tokenizeRecallQuery } from "./orchestrator-helpers.js";
 import { qmdCollectionPathParts } from "./qmd-result-resolver.js";
+import { qmdCollectionNamespaceFromPrefix as computeQmdCollectionNamespaceFromPrefix } from "./orchestrator-namespace-scope.js";
 import { resolveNamespaceFromStorageDir } from "../scopes/scope-plan.js";
 import type { SearchBackend, SearchExecutionOptions, SearchQueryOptions } from "../search/port.js";
 import type { MemoryFile, PluginConfig, QmdSearchResult } from "../types.js";
@@ -59,7 +60,6 @@ export interface NamespaceReadFanoutDeps {
     configured: Set<string>,
   ): { namespace: string; identityToken: string; storageDir: string };
   readonly qmd: SearchBackend;
-  qmdCollectionNamespaceFromPrefix(collectionPrefix: string): string | null;
   rememberNamespaceStorageDirHint(namespace: string, storageDir?: string): void;
   storageDirMatchesNamespaceHint(namespace: string, storageDir: string): boolean;
   readonly storageRouter: NamespaceStorageRouter;
@@ -369,13 +369,16 @@ export class NamespaceReadFanoutCoordinator {
             ? this.deps.namespaceFromPath(memoryPath)
             : this.deps.config.defaultNamespace;
           const storage = await this.deps.storageRouter.storageFor(namespace);
-          return await storage.readMemoryByPath(memoryPath);
+          const memory = await storage.readMemoryByPath(memoryPath);
+          return memory ? { memory, namespace } : null;
         }),
       )
-    ).filter((memory): memory is MemoryFile => memory !== null);
+    ).filter(
+      (entry): entry is { memory: MemoryFile; namespace: string } => entry !== null,
+    );
 
     const results: QmdSearchResult[] = [];
-    for (const memory of memories) {
+    for (const { memory, namespace } of memories) {
       const status = memory.frontmatter.status ?? "active";
       if (!options?.allowArchived && status !== "active") continue;
 
@@ -395,6 +398,7 @@ export class NamespaceReadFanoutCoordinator {
 
       results.push({
         docid: memory.frontmatter.id,
+        namespace,
         path: memory.path,
         score,
         snippet: memory.content.slice(0, 400).replace(/\n/g, " "),
@@ -434,12 +438,14 @@ export class NamespaceReadFanoutCoordinator {
     if (!resolveNamespaceCapabilities(this.deps.config).namespaces) return this.deps.config.defaultNamespace;
     const parts = qmdCollectionPathParts(p);
     const collectionNamespace = parts
-      ? this.deps.qmdCollectionNamespaceFromPrefix(parts.collection)
+      ? computeQmdCollectionNamespaceFromPrefix(parts.collection, this.deps.config)
       : null;
     if (collectionNamespace) return collectionNamespace;
-    const m = p.match(/[\\/]+namespaces[\\/]+([^\\/]+)(?:[\\/]|$)/);
-    if (!m?.[1]) return this.deps.config.defaultNamespace;
-    return namespaceIdentityFromToken(m[1]) ?? m[1];
+    const pathParts = p.replaceAll("\\", "/").split("/");
+    const namespaceIndex = pathParts.indexOf("namespaces");
+    const namespaceToken = namespaceIndex >= 0 ? pathParts[namespaceIndex + 1] : undefined;
+    if (!namespaceToken) return this.deps.config.defaultNamespace;
+    return namespaceIdentityFromToken(namespaceToken) ?? namespaceToken;
   }
 
   storageDirNamespace(storageDir: string): string {

package/src/orchestration/orchestrator-helpers.ts

@@ -249,6 +249,13 @@ export interface RecallInvocationOptions {
    * against the caller's real context.
    */
   currentContextScopes?: readonly unknown[];
+  /**
+   * Wall-clock ms the caller waited to begin execution — per-principal
+   * semaphore queue + single-flight wait (issue #1906). Folded into
+   * recall-timings additively as `queueWaitMs`; omitted phases are simply
+   * absent, so existing timing consumers are unaffected.
+   */
+  queueWaitMs?: number;
 }
 
 export type QueryAwarePrefilter = {
@@ -612,7 +619,7 @@ export function filterRecallCandidates(
 ): QmdSearchResult[] {
   const scopedByNamespace = options.namespacesEnabled
     ? candidates.filter((r) =>
-        options.recallNamespaces.includes(options.resolveNamespace(r.path)),
+        options.recallNamespaces.includes(r.namespace ?? options.resolveNamespace(r.path)),
       )
     : candidates;
   return scopedByNamespace

package/src/orchestration/orchestrator-init.ts

@@ -174,13 +174,29 @@ export class OrchestratorInitCoordinator {
         promotionByOutcomeEnabled: resolveUtilityLearningCapabilities(this.deps.config).promotionByOutcome,
       });
 
-      // Initialize content-hash dedup index
+      // Initialize the content-hash dedup index from the corpus-AUTHORITATIVE
+      // rebuild (issue #1909 review round 12) rather than a raw, possibly-stale
+      // fact-hashes.txt load — the orchestrator dedup layer and StorageManager
+      // share this one instance so a crash before a deferred batch save cannot
+      // leave the restart's dedup blind to a durable fact.
       if (resolveRecallAuxiliaryCapabilities(this.deps.config).factDeduplication) {
-        this.deps.contentHashIndex = this.deps.storage.createContentHashIndex();
-        await this.deps.contentHashIndex.load();
-        log.info(
-          `content-hash dedup: loaded ${this.deps.contentHashIndex.size} hashes`,
-        );
+        try {
+          this.deps.contentHashIndex = await this.deps.storage.getAuthoritativeFactHashIndex();
+          log.info(
+            `content-hash dedup: rebuilt authoritative index with ${this.deps.contentHashIndex.size} hashes`,
+          );
+        } catch (err) {
+          // PR #2016: the locked corpus rebuild could not run at init (transient
+          // cross-process contention). Pre-warm with the shared instance instead
+          // of failing init; the dedup consumers gate a MISS on
+          // isFactContentHashAuthoritative() and confirm against the corpus until
+          // a later rebuild acquires the lock.
+          this.deps.contentHashIndex = await this.deps.storage.getSharedFactHashIndex();
+          log.warn(
+            `content-hash dedup: authoritative rebuild deferred at init (${err instanceof Error ? err.message : String(err)}); ` +
+              `using the shared index, will retry the locked rebuild on next use`,
+          );
+        }
       }
       await this.deps.transcript.initialize();
       await this.deps.summarizer.initialize();

package/src/orchestration/orchestrator-namespace-scope.ts

@@ -0,0 +1,71 @@
+import path from "node:path";
+import { log } from "../logger.js";
+import { resolveNamespaceCapabilities } from "../capabilities.js";
+import { namespaceIdentityFromToken } from "../namespaces/identity.js";
+import type { PluginConfig } from "../types.js";
+
+/**
+ * Compute the semantic-dedup path scope for a target storage (issue #1526 seam:
+ * extracted verbatim from Orchestrator.semanticDedupScopeFor). Reads only the
+ * plugin config + target storage dir; no `this` state.
+ */
+export function computeSemanticDedupScope(
+  targetStorage: { readonly dir: string },
+  config: PluginConfig,
+): {
+  pathPrefix?: string;
+  pathExcludePrefixes?: readonly string[];
+} {
+  if (!resolveNamespaceCapabilities(config).namespaces) return {};
+  const memoryDir = path.resolve(config.memoryDir);
+  const storageDir = path.resolve(targetStorage.dir);
+  if (storageDir === memoryDir) {
+    // Default namespace at legacy root. Include everything that isn't
+    // under `namespaces/*` (those belong to other namespaces).
+    return { pathExcludePrefixes: ["namespaces/"] };
+  }
+  let rel = path.relative(memoryDir, storageDir);
+  if (!rel || rel.startsWith("..")) {
+    // Round 12 fix (PR #399): when targetStorage.dir is outside memoryDir
+    // (custom namespace routing), toMemoryRelativePath() stores the absolute
+    // file path in the index rather than a memoryDir-relative path. Return the
+    // absolute storageDir as the pathPrefix so the search() filter still scopes
+    // the lookup to the correct tenant's files. Previously this returned {} (no
+    // scoping), which let high-similarity hits from other namespaces' absolute-
+    // path entries suppress writes in the target namespace — a cross-tenant
+    // dedup suppression path.
+    log.debug(
+      `semantic dedup: target storage dir ${storageDir} is outside memoryDir ${memoryDir}; scoping lookup to absolute path prefix`,
+    );
+    const absPrefix = storageDir.replace(/\\/g, "/");
+    return { pathPrefix: absPrefix.endsWith("/") ? absPrefix : `${absPrefix}/` };
+  }
+  rel = rel.replace(/\\/g, "/");
+  if (!rel.endsWith("/")) rel = `${rel}/`;
+  return { pathPrefix: rel };
+}
+
+/**
+ * Resolve the namespace identity implied by a QMD collection prefix (issue
+ * #1526 seam: extracted verbatim from Orchestrator.qmdCollectionNamespaceFromPrefix).
+ * Reads only the plugin config; no `this` state.
+ */
+export function qmdCollectionNamespaceFromPrefix(
+  collectionPrefix: string,
+  config: PluginConfig,
+): string | null {
+  const baseCollection = config.qmdCollection;
+  if (collectionPrefix === baseCollection) return config.defaultNamespace;
+  const namespaceSuffix = collectionPrefix.startsWith(`${baseCollection}--`)
+    ? collectionPrefix.slice(baseCollection.length + 2)
+    : "";
+  if (!namespaceSuffix) return null;
+
+  const decoded = namespaceIdentityFromToken(namespaceSuffix);
+  if (decoded !== null) return decoded || config.defaultNamespace;
+  if (namespaceSuffix.startsWith("ns--")) {
+    const legacyNamespace = namespaceSuffix.slice("ns--".length).trim();
+    return legacyNamespace || null;
+  }
+  return null;
+}

package/src/orchestration/persistence-index.ts

@@ -23,7 +23,8 @@ import { ContentHashIndex, StorageManager } from "../index.js";
 import { log } from "../logger.js";
 import { isActiveMemoryStatus } from "../memory-lifecycle-ledger-utils.js";
 import { stripCitationForTemplate } from "../source-attribution.js";
-import { clearIndexesAsync, indexMemoriesBatchAsync, indexesExistAsync } from "../temporal-index.js";
+import { clearIndexesAsync, indexesExistAsync } from "../temporal-index.js";
+import { indexMemoriesBatchAsync } from "../temporal-index-batch.js";
 import { normalizeSupersessionKey } from "../temporal-supersession.js";
 import type { MemoryFile, MemoryFrontmatter, PluginConfig } from "../types.js";
 import {
@@ -80,13 +81,23 @@ export class PersistenceIndexCoordinator {
 
     if (memory.frontmatter.contentHash) {
       index.removeByHash(memory.frontmatter.contentHash);
-      return;
+    } else {
+      log.warn(
+        `[${context}] removing hash for legacy memory ${memory.frontmatter.id ?? "(unknown)"} via content fallback - no contentHash in frontmatter`,
+      );
+      index.remove(memory.content);
     }
-
-    log.warn(
-      `[${context}] removing hash for legacy memory ${memory.frontmatter.id ?? "(unknown)"} via content fallback - no contentHash in frontmatter`,
-    );
-    index.remove(memory.content);
+    // PR #2016 threads SDzOP / SDzOR: the shared index above is category-
+    // agnostic, but StorageManager.hasFactContentHash answers from a fact-ONLY
+    // membership set. Removing from the shared index alone left that set holding
+    // the removed fact's hash, so hasFactContentHash returned a stale `true`
+    // until the next corpus rebuild and wearable / promotion callers skipped a
+    // valid write. Keep the fact-only set in lockstep with the shared removal.
+    // Round 11: no fact-hashes.ready marker exists — the fact-hash index is
+    // rebuilt from the corpus on every restart, so an archived/superseded
+    // memory's .md is gone by then and the removal lands regardless of whether
+    // this run's reconciling save published.
+    targetStorage.removeFactOnlyHashForMemory(memory);
   }
 
   /**
@@ -156,7 +167,16 @@ export class PersistenceIndexCoordinator {
       const incomingEntityNorm = entityRef
         ? normalizeSupersessionKey(entityRef)
         : undefined;
-      const all = await targetStorage.readAllMemories();
+      // #2016 cold-tier finding: the authoritative content-hash rebuild unions
+      // the hot and cold tiers, so a dedup hit can name an active fact whose
+      // only copy was demoted to cold/. Scan both tiers (hot first, so a hot
+      // copy is still preferred) or the corrected valid_at/invalid_at write is
+      // suppressed while the cold copy keeps stale bounds in recall.
+      const [hotMems, coldMems] = await Promise.all([
+        targetStorage.readAllMemories(),
+        targetStorage.readAllColdMemories(),
+      ]);
+      const all = coldMems.length === 0 ? hotMems : [...hotMems, ...coldMems];
       const existing = all.find((m) => {
         if (m.frontmatter.category !== "fact") return false;
         if ((m.frontmatter.status ?? "active") !== "active") return false;
@@ -263,6 +283,16 @@ export class PersistenceIndexCoordinator {
     }
   }
 
+  /**
+   * Persist the touched content-hash indexes via the removal-aware, lock-held
+   * RECONCILING save (issue #1909 review round 8 thread 3). Both append batches
+   * (extraction persist) and removal batches (archival / semantic consolidation)
+   * go through the SAME operation so they serialize against each other on the
+   * per-file lock: each reconciles `(on-disk \ removed) ∪ additions`, so a
+   * removal never resurrects a hash (a plain union would) and never clobbers a
+   * concurrent extraction's appended hash (a blind overwrite would). Untouched
+   * indexes dirty-short-circuit and are skipped entirely (finding 3).
+   */
   async saveContentHashIndexes(): Promise<void> {
     const indexes = new Set<ContentHashIndex>();
     if (this.deps.contentHashIndex) indexes.add(this.deps.contentHashIndex);
@@ -270,7 +300,32 @@ export class PersistenceIndexCoordinator {
       indexes.add(index);
     }
     for (const index of indexes) {
-      await index.save();
+      await index.saveMergingWithDisk();
+    }
+  }
+
+  /**
+   * Drive any deferred lock-timeout reconcile-save retries to completion at a
+   * lifecycle boundary (PR #2016 finding 3). Each per-index background retry is
+   * `unref`'d so it never keeps a long-lived host alive; a short-lived writer (a
+   * one-shot CLI) can exit before it fires, leaving a durable fact `.md` whose
+   * hash never reached disk for peers. `orchestrator.destroy()` calls this so the
+   * addition publishes before the process exits. Long-lived hosts only reach it
+   * at their own shutdown, so their in-flight retries keep running in the
+   * background until then. Best-effort: `flushReconcileRetry` is a no-op on a
+   * clean index and a permanently contended lock falls back to the
+   * corpus-rebuild-on-restart safety net.
+   */
+  async drainContentHashReconcileRetries(): Promise<void> {
+    const indexes = new Set<ContentHashIndex>();
+    if (this.deps.contentHashIndex) indexes.add(this.deps.contentHashIndex);
+    for (const index of this.deps.contentHashIndexesByStorageDir.values()) {
+      indexes.add(index);
+    }
+    for (const index of indexes) {
+      await index.flushReconcileRetry().catch((err) =>
+        log.warn(`content-hash index reconcile drain failed: ${err}`),
+      );
     }
   }
 

package/src/orchestration/qmd-result-resolver.ts

@@ -137,6 +137,7 @@ export class QmdResultResolver {
     resultPath: string,
     fallbackStorage: StorageManager,
     recallNamespaces: readonly string[] = [],
+    preferredNamespace?: string,
   ): Promise<MemoryFile | null> {
     const parts = qmdCollectionPathParts(resultPath);
     const fallbackStorageDir = storageDirOrNull(fallbackStorage);
@@ -234,6 +235,31 @@ export class QmdResultResolver {
       }
     }
 
+    if (preferredNamespace) {
+      try {
+        const preferredStorage = await this.storageFor(preferredNamespace);
+        for (const candidate of qmdResultPathCandidates(
+          preferredStorage.dir,
+          resultPath,
+        )) {
+          const memory = await preferredStorage.readMemoryByPath(candidate);
+          if (memory) return memory;
+        }
+      } catch (err) {
+        if (err instanceof SecureStoreLockedError) throw err;
+        log.debug("qmd preferred namespace path lookup failed open", {
+          path: resultPath,
+          namespace: preferredNamespace,
+          error: (err as Error).message,
+        });
+      }
+      // The caller supplied an explicit owning namespace. For a relative path
+      // the preferred store was the only correct lead, so a miss means the hit
+      // is stale/deleted — do NOT fall through to the default store and validate
+      // a same-relative-path memory from the wrong namespace (#2020). Absolute
+      // paths still fall through: the absolute branch resolves the true owner.
+      if (!path.isAbsolute(resultPath)) return null;
+    }
     if (path.isAbsolute(resultPath)) {
       if (!fallbackStorageDir) {
         return await fallbackStorage.readMemoryByPath(resultPath);
@@ -276,6 +302,7 @@ export class QmdResultResolver {
       result.path,
       fallbackStorage,
       recallNamespaces,
+      result.namespace,
     );
     if (!memory) return null;
 

package/src/orchestration/recall-entry.ts

@@ -14,16 +14,19 @@ import { type EvalShadowRecallRecord, recordEvalShadowRecall } from "../evals.js
 import { StorageManager } from "../index.js";
 import { log } from "../logger.js";
 import { resolvePrincipal } from "../namespaces/principal.js";
+import { type RecallSectionAppendOptions, type RecallSectionBuckets } from "./recall-section-coordinator.js";
 import { type RecallInvocationOptions, abortRecallError } from "./orchestrator-helpers.js";
 import { ProfilingCollector } from "../profiling.js";
-import type { TrustStageResultItem } from "../trust-score-stage.js";
+import { trustResultFor, type TrustStageResultItem } from "../trust-score-stage.js";
+import type { RecallResultFormatter } from "./recall-result-formatter.js";
 import type { IdentityInjectionMode, PluginConfig, QmdSearchResult } from "../types.js";
 
 export interface RecallEntryDeps {
   appendRecallSection(
-    sectionBuckets: Map<string, string[]>,
+    sectionBuckets: RecallSectionBuckets,
     sectionId: string,
     content: string,
+    options?: RecallSectionAppendOptions,
   ): boolean;
   readonly config: PluginConfig;
   enqueueDirectAnswerObservation(
@@ -36,12 +39,7 @@ export interface RecallEntryDeps {
   ): void;
   evalShadowWriteChain: Promise<void>;
   extractMemoryIdsFromResults(results: QmdSearchResult[]): string[];
-  formatQmdResults(
-    title: string,
-    results: QmdSearchResult[],
-    sessionKey?: string,
-    trustByPath?: Map<string, TrustStageResultItem> | null,
-  ): string;
+  recallResultFormatter: RecallResultFormatter;
   readonly initPromise: Promise<void> | null;
   lastRecallFailureAtMs: number;
   lastRecallFailureLogAtMs: number;
@@ -57,7 +55,6 @@ export interface RecallEntryDeps {
   ): Promise<string>;
   readonly storage: StorageManager;
   suppressedRecallFailures: number;
-  trackMemoryAccess(memoryIds: string[]): void;
   trackRecallBackgroundWrite(promise: Promise<void>, label: string): void;
 }
 
@@ -258,7 +255,7 @@ export class RecallEntryCoordinator {
   publishRecallResults(options: {
     title: string;
     results: QmdSearchResult[];
-    sectionBuckets: Map<string, string[]>;
+    sectionBuckets: RecallSectionBuckets;
     retrievalQuery: string;
     sessionKey: string | undefined;
     identityInjection?: {
@@ -270,26 +267,44 @@ export class RecallEntryCoordinator {
      * Issue #1577 — per-recall trust map. When present, quarantined items
      * are filtered from injection on EVERY recall path (hot QMD, embedding
      * fallback, cold archive, recent) so a faithfulness-contradicted memory
-     * cannot sneak in via a branch that bypasses trust scoring (review:
-     * fallback paths bypass trust). The map is also threaded to
-     * formatQmdResults for the epistemic hedge.
+     * cannot sneak in via a branch that bypasses trust scoring. The map is
+     * also threaded to formatQmdResultEntries for the epistemic hedge.
      */
     trustByPath?: Map<string, TrustStageResultItem> | null;
   }): void {
     const sectionId = "memories";
-    // Filter quarantined items from ALL recall paths so no branch can inject
-    // a hard-negative memory that trust scoring excluded on another path.
     const trustByPath = options.trustByPath ?? null;
     const injectable = trustByPath
-      ? options.results.filter((r) => !trustByPath.get(r.path)?.quarantined)
+      ? options.results.filter((r) => !trustResultFor(trustByPath, r)?.quarantined)
       : options.results;
-    const memoryIds = this.deps.extractMemoryIdsFromResults(injectable);
-    this.deps.trackMemoryAccess(memoryIds);
+    if (injectable.length === 0) return;
 
+    const formatted = this.deps.recallResultFormatter.formatQmdResultEntries(
+      options.title,
+      injectable,
+      options.sessionKey,
+      trustByPath,
+    );
     this.deps.appendRecallSection(
       options.sectionBuckets,
       sectionId,
-      this.deps.formatQmdResults(options.title, injectable, options.sessionKey, trustByPath),
+      formatted.heading,
     );
+    for (const [index, entry] of formatted.entries.entries()) {
+      const result = injectable[index];
+      if (!result) continue;
+      const memoryId = this.deps.extractMemoryIdsFromResults([result])[0];
+      this.deps.appendRecallSection(
+        options.sectionBuckets,
+        sectionId,
+        entry,
+        {
+          atomic: true,
+          ...(memoryId ? { memoryId } : {}),
+          ...(result.path ? { memoryPath: result.path } : {}),
+          ...(result.namespace ? { memoryNamespace: result.namespace } : {}),
+        },
+      );
+    }
   }
 }