@remnic/core 9.3.637 → 9.3.638

package/src/access-service.ts

@@ -54,7 +54,10 @@ import {
 } from "./memory-lifecycle-ledger-utils.js";
 import { getMemoryProjectionPath } from "./memory-projection-store.js";
 import { canReadNamespace, canWriteNamespace, defaultNamespaceForPrincipal, recallNamespacesForPrincipal, resolvePrincipal } from "./namespaces/principal.js";
+import { namespaceIdentityFromToken } from "./namespaces/identity.js";
 import { namespaceCollectionName } from "./namespaces/search.js";
+import { SecureStoreLockedError } from "./secure-store/index.js";
+import { isPathInsideStorageRoot } from "./storage-paths.js";
 import type { LastRecallSnapshot } from "./recall-state.js";
 import type {
   GraphRecallSnapshot,
@@ -168,6 +171,52 @@ import { formatProfileTraceAscii } from "./profiling.js";
 
 export class EngramAccessInputError extends Error {}
 
+function qmdCollectionPathParts(resultPath: string): {
+  collection: string;
+  relativePath: string;
+} | null {
+  if (!resultPath || nodePath.isAbsolute(resultPath)) return null;
+  const normalized = resultPath.replace(/\\/g, "/").replace(/^\/+/, "");
+  const slashIndex = normalized.indexOf("/");
+  if (slashIndex <= 0 || slashIndex >= normalized.length - 1) return null;
+  const collection = normalized.slice(0, slashIndex);
+  if (/^\d{4}-\d{2}-\d{2}$/.test(collection)) return null;
+  return {
+    collection,
+    relativePath: normalized.slice(slashIndex + 1),
+  };
+}
+
+function qmdResultPathCandidates(
+  storageDir: string,
+  resultPath: string,
+): string[] {
+  const candidates = new Set<string>();
+  const storageRoot = nodePath.resolve(storageDir);
+  const addCandidate = (candidate: string) => {
+    const resolved = nodePath.resolve(candidate);
+    if (isPathInsideStorageRoot(storageRoot, resolved)) {
+      candidates.add(resolved);
+    }
+  };
+  const addRelativeCandidates = (relativePath: string) => {
+    const normalized = relativePath.replace(/\\/g, "/").replace(/^\/+/, "");
+    if (!normalized) return;
+    addCandidate(nodePath.join(storageRoot, normalized));
+    if (/^\d{4}-\d{2}-\d{2}\//.test(normalized)) {
+      addCandidate(nodePath.join(storageRoot, "facts", normalized));
+    }
+  };
+
+  if (nodePath.isAbsolute(resultPath)) {
+    addCandidate(resultPath);
+  } else {
+    addRelativeCandidates(resultPath);
+  }
+
+  return [...candidates];
+}
+
 type AccessProfilingReportRequest = {
   format?: string;
   limit?: number;
@@ -1437,6 +1486,7 @@ export class EngramAccessService {
       queryLen: options.query.length,
       memoryIds,
       namespace,
+      recallNamespaces: [namespace],
       traceId: options.snapshot.traceId,
       plannerMode: options.normalizedMode,
       requestedMode:
@@ -1500,8 +1550,124 @@ export class EngramAccessService {
     const namespace = snapshot.namespace ? this.resolveNamespace(snapshot.namespace) : this.orchestrator.config.defaultNamespace;
     const storage = await this.orchestrator.getStorage(namespace);
     const storageDir = storage.dir;
+    const recallNamespaces = Array.from(
+      new Set(
+        [
+          namespace,
+          ...(Array.isArray(snapshot.recallNamespaces)
+            ? snapshot.recallNamespaces.map((ns) => this.resolveNamespace(ns))
+            : []),
+          this.orchestrator.config.defaultNamespace,
+          this.orchestrator.config.sharedNamespace,
+          ...(this.orchestrator.config.namespacePolicies ?? []).map((p) => p.name),
+        ].filter((ns): ns is string => typeof ns === "string" && ns.length > 0),
+      ),
+    );
     const results: EngramAccessMemorySummary[] = [];
     const seen = new Set<string>();
+    const collectionNamespaceFromPrefix = (collectionPrefix: string): string | null => {
+      const baseCollection = this.orchestrator.config.qmdCollection;
+      if (collectionPrefix === baseCollection) return this.orchestrator.config.defaultNamespace;
+      const namespaceSuffix = collectionPrefix.startsWith(`${baseCollection}--`)
+        ? collectionPrefix.slice(baseCollection.length + 2)
+        : "";
+      if (!namespaceSuffix) return null;
+
+      const decoded = namespaceIdentityFromToken(namespaceSuffix);
+      if (decoded !== null) return decoded || this.orchestrator.config.defaultNamespace;
+      if (namespaceSuffix.startsWith("ns--")) {
+        const legacyNamespace = namespaceSuffix.slice("ns--".length).trim();
+        return legacyNamespace || null;
+      }
+      return null;
+    };
+    const readResultPath = async (
+      memoryPath: string,
+    ): Promise<{ memory: MemoryFile; baseDir: string } | null> => {
+      const parts = qmdCollectionPathParts(memoryPath);
+      const coldCollection =
+        this.orchestrator.config.qmdColdCollection ?? "openclaw-engram-cold";
+      if (parts && parts.collection === coldCollection) {
+        const storages: Array<{ storage: StorageManager; dir: string }> = [];
+        const seenStorageDirs = new Set<string>();
+        const addStorage = (candidateStorage: StorageManager): void => {
+          const candidateDir = nodePath.resolve(candidateStorage.dir);
+          if (seenStorageDirs.has(candidateDir)) return;
+          seenStorageDirs.add(candidateDir);
+          storages.push({ storage: candidateStorage, dir: candidateDir });
+        };
+        addStorage(storage);
+        for (const recallNamespace of recallNamespaces) {
+          try {
+            addStorage(await this.orchestrator.getStorage(recallNamespace));
+          } catch {
+            continue;
+          }
+        }
+        for (const candidateStorage of storages) {
+          try {
+            const coldRoot = nodePath.join(candidateStorage.dir, "cold");
+            for (const candidatePath of qmdResultPathCandidates(
+              coldRoot,
+              parts.relativePath,
+            )) {
+              const memory =
+                await candidateStorage.storage.readMemoryByPath(candidatePath);
+              if (memory) return { memory, baseDir: candidateStorage.dir };
+            }
+          } catch (err) {
+            if (err instanceof SecureStoreLockedError) throw err;
+            continue;
+          }
+        }
+        return null;
+      }
+
+      const collectionNamespace = parts
+        ? collectionNamespaceFromPrefix(parts.collection)
+        : null;
+
+      if (parts && collectionNamespace) {
+        try {
+          const collectionStorage =
+            await this.orchestrator.getStorage(collectionNamespace);
+          for (const candidate of qmdResultPathCandidates(
+            collectionStorage.dir,
+            parts.relativePath,
+          )) {
+            const memory = await collectionStorage.readMemoryByPath(candidate);
+            if (memory) return { memory, baseDir: collectionStorage.dir };
+          }
+          return null;
+        } catch (err) {
+          if (err instanceof SecureStoreLockedError) throw err;
+          return null;
+        }
+      }
+
+      if (nodePath.isAbsolute(memoryPath)) {
+        const ownerStorage = await this.storageForAbsoluteRecallPath(
+          memoryPath,
+          namespace,
+          recallNamespaces,
+        );
+        if (!ownerStorage) return null;
+        for (const candidate of qmdResultPathCandidates(
+          ownerStorage.dir,
+          memoryPath,
+        )) {
+          const memory = await ownerStorage.storage.readMemoryByPath(candidate);
+          if (memory) return { memory, baseDir: ownerStorage.dir };
+        }
+        return null;
+      }
+
+      for (const candidate of qmdResultPathCandidates(storageDir, memoryPath)) {
+        const memory = await storage.readMemoryByPath(candidate);
+        if (memory) return { memory, baseDir: storageDir };
+      }
+      return null;
+    };
 
     // Pre-fetch raw excerpts once when `disclosure === "raw"` so we don't
     // hit the LCM archive per-result (issue #677 PR 2/4).  Excerpts are
@@ -1519,13 +1685,14 @@ export class EngramAccessService {
 
     for (const memoryPath of snapshot.resultPaths ?? []) {
       if (!memoryPath || seen.has(memoryPath)) continue;
-      const memory = await storage.readMemoryByPath(memoryPath);
-      if (!memory) continue;
+      const resolved = await readResultPath(memoryPath);
+      if (!resolved) continue;
+      const { memory, baseDir } = resolved;
       seen.add(memoryPath);
       results.push(
         this.serializeMemorySummary(
           memory,
-          storageDir,
+          baseDir,
           disclosure,
           // Attach the (possibly empty) raw excerpts to the first raw
           // result; subsequent results do not duplicate the array.
@@ -1552,6 +1719,58 @@ export class EngramAccessService {
     return results;
   }
 
+  private async storageForAbsoluteRecallPath(
+    memoryPath: string,
+    primaryNamespace: string,
+    recallNamespaces: readonly string[] = [],
+  ): Promise<{ storage: StorageManager; dir: string } | null> {
+    const resolvedPath = nodePath.resolve(memoryPath);
+    const memoryRoot = nodePath.resolve(this.orchestrator.config.memoryDir);
+    const namespacesRoot = nodePath.join(memoryRoot, "namespaces");
+    const configuredNamespaces = new Set<string>();
+    configuredNamespaces.add(primaryNamespace);
+    for (const namespace of recallNamespaces) {
+      configuredNamespaces.add(namespace);
+    }
+    configuredNamespaces.add(this.orchestrator.config.defaultNamespace);
+    configuredNamespaces.add(this.orchestrator.config.sharedNamespace);
+    if (isPathInsideStorageRoot(namespacesRoot, resolvedPath)) {
+      const relativeToNamespaces = nodePath.relative(namespacesRoot, resolvedPath);
+      const [namespaceSegment] = relativeToNamespaces.split(/[\\/]/);
+      if (namespaceSegment) {
+        configuredNamespaces.add(
+          namespaceIdentityFromToken(namespaceSegment) ?? namespaceSegment,
+        );
+      }
+    }
+    for (const policy of this.orchestrator.config.namespacePolicies ?? []) {
+      configuredNamespaces.add(policy.name);
+    }
+
+    const matches: Array<{ storage: StorageManager; dir: string }> = [];
+    for (const ns of configuredNamespaces) {
+      if (!ns) continue;
+      let candidateStorage: StorageManager;
+      try {
+        candidateStorage = await this.orchestrator.getStorage(ns);
+      } catch {
+        continue;
+      }
+      const candidateRoot = nodePath.resolve(candidateStorage.dir);
+      if (!isPathInsideStorageRoot(candidateRoot, resolvedPath)) continue;
+      if (
+        candidateRoot === memoryRoot &&
+        isPathInsideStorageRoot(namespacesRoot, resolvedPath)
+      ) {
+        continue;
+      }
+      matches.push({ storage: candidateStorage, dir: candidateRoot });
+    }
+
+    matches.sort((a, b) => b.dir.length - a.dir.length);
+    return matches[0] ?? null;
+  }
+
   /**
    * Fetch raw transcript excerpts from the LCM archive for `disclosure ===
    * "raw"` recalls (issue #677 PR 2/4).  Returns `null` for non-raw recall

package/src/graph.ts

@@ -517,6 +517,8 @@ export class GraphIndex {
        * Default `false` (floor from config is applied).
        */
       includeLowConfidence?: boolean;
+      /** Absolute deadline in ms-since-epoch for post-retrieval assembly. */
+      deadlineAtMs?: number;
     }
   ): Promise<
     Array<{
@@ -545,15 +547,22 @@ export class GraphIndex {
     const floor =
       opts?.includeLowConfidence === true ? 0 : clampConfidenceFloor(this.cfg.graphTraversalConfidenceFloor);
     const iterations = clampPageRankIterations(this.cfg.graphTraversalPageRankIterations);
+    const deadlineAtMs = opts?.deadlineAtMs;
+    const deadlineExpired = (): boolean =>
+      typeof deadlineAtMs === "number" && Date.now() >= deadlineAtMs;
 
     try {
+      if (deadlineExpired()) return [];
       const allEdges = await this.loadEdgesCached();
+      if (deadlineExpired()) return [];
 
       // Build adjacency index: from → edges, to → edges (bidirectional for entity/time, directional for causal).
       // Edges below the confidence floor are pruned at index time so neither
       // direct activation nor downstream BFS expansion can re-introduce them.
       const adj = new Map<string, GraphEdge[]>();
-      for (const edge of allEdges) {
+      for (let i = 0; i < allEdges.length; i += 1) {
+        if ((i & 1023) === 0 && deadlineExpired()) return [];
+        const edge = allEdges[i];
         const conf = readEdgeConfidence(edge);
         if (conf < floor) continue;
         if (!adj.has(edge.from)) adj.set(edge.from, []);
@@ -583,13 +592,28 @@ export class GraphIndex {
         frontier.set(`${seed}\0${seed}`, { node: seed, seed, activation: 1 });
         reachedBySeed.set(seed, new Set([seed]));
       }
+      const finalizeScores = () =>
+        Array.from(scores.entries())
+          .map(([p, score]) => ({
+            path: p,
+            score,
+            seed: provenance.get(p)?.seed ?? "",
+            hopDepth: provenance.get(p)?.hopDepth ?? 0,
+            decayedWeight: provenance.get(p)?.decayedWeight ?? 0,
+            graphType: provenance.get(p)?.graphType ?? "entity",
+            edgeConfidence: provenance.get(p)?.edgeConfidence ?? 1,
+          }))
+          .sort((a, b) => b.score - a.score);
 
       for (let hop = 0; hop < steps && frontier.size > 0; hop++) {
+        if (deadlineExpired()) return finalizeScores();
         const nextFrontier = new Map<string, { node: string; seed: string; activation: number }>();
 
         for (const { node, seed: sourceSeed, activation } of frontier.values()) {
           const edges = adj.get(node) ?? [];
-          for (const edge of edges) {
+          for (let i = 0; i < edges.length; i += 1) {
+            if ((i & 1023) === 0 && deadlineExpired()) return finalizeScores();
+            const edge = edges[i];
             const neighbor = edge.to === node ? edge.from : edge.to;
             const conf = readEdgeConfidence(edge);
             // Defense in depth: the adjacency build already drops sub-floor
@@ -645,15 +669,17 @@ export class GraphIndex {
       // edges, weighted by edge confidence. Damping is fixed at the
       // canonical 0.85 so the ranking stays comparable across queries;
       // the `iterations` knob bounds compute, not behavior shape.
-      if (iterations > 0 && scores.size > 1) {
+      if (!deadlineExpired() && iterations > 0 && scores.size > 1) {
         applyPageRankRefinement(scores, adj, {
           iterations,
           floor,
           damping: 0.85,
+          deadlineAtMs,
         });
       }
 
       // Apply lateral inhibition if enabled (Synapse-inspired competitive suppression)
+      if (deadlineExpired()) return finalizeScores();
       if (this.cfg.graphLateralInhibitionEnabled && scores.size > 1) {
         const inhibited = applyLateralInhibition(scores, {
           beta: this.cfg.graphLateralInhibitionBeta,
@@ -664,17 +690,7 @@ export class GraphIndex {
         }
       }
 
-      return Array.from(scores.entries())
-        .map(([p, score]) => ({
-          path: p,
-          score,
-          seed: provenance.get(p)?.seed ?? "",
-          hopDepth: provenance.get(p)?.hopDepth ?? 0,
-          decayedWeight: provenance.get(p)?.decayedWeight ?? 0,
-          graphType: provenance.get(p)?.graphType ?? "entity",
-          edgeConfidence: provenance.get(p)?.edgeConfidence ?? 1,
-        }))
-        .sort((a, b) => b.score - a.score);
+      return finalizeScores();
     } catch (err) {
       const { log } = await import("./logger.js");
       log.warn(`[graph] spreadingActivation error: ${err}`);
@@ -725,11 +741,13 @@ export function clampPageRankIterations(raw: unknown): number {
 export function applyPageRankRefinement(
   scores: Map<string, number>,
   adj: Map<string, GraphEdge[]>,
-  opts: { iterations: number; floor: number; damping: number }
+  opts: { iterations: number; floor: number; damping: number; deadlineAtMs?: number }
 ): void {
-  const { iterations, floor, damping } = opts;
+  const { iterations, floor, damping, deadlineAtMs } = opts;
   if (iterations <= 0 || scores.size === 0) return;
   const safeDamping = Math.min(1, Math.max(0, damping));
+  const deadlineExpired = (): boolean =>
+    typeof deadlineAtMs === "number" && Date.now() >= deadlineAtMs;
 
   // Pre-compute confidence-weighted out-edge totals for normalization.
   // Done once per refinement, not per iteration, since adjacency is
@@ -748,6 +766,7 @@ export function applyPageRankRefinement(
   };
   const outboundTotal = new Map<string, number>();
   for (const [node, edges] of adj.entries()) {
+    if (deadlineExpired()) return;
     if (!scores.has(node)) continue; // only candidate nodes redistribute
     let sum = 0;
     for (const edge of edges) {
@@ -758,6 +777,7 @@ export function applyPageRankRefinement(
   }
 
   for (let i = 0; i < iterations; i += 1) {
+    if (deadlineExpired()) return;
     const next = new Map<string, number>();
     // Teleport / damping floor: every node retains `(1 - damping) * score`
     // of its current activation so dangling nodes do not bleed to zero.
@@ -776,7 +796,9 @@ export function applyPageRankRefinement(
         next.set(node, (next.get(node) ?? 0) + safeDamping * score);
         continue;
       }
-      for (const edge of outEdges) {
+      for (let edgeIndex = 0; edgeIndex < outEdges.length; edgeIndex += 1) {
+        if ((edgeIndex & 1023) === 0 && deadlineExpired()) return;
+        const edge = outEdges[edgeIndex];
         if (!eligible(edge, node)) continue;
         const conf = readEdgeConfidence(edge);
         const neighbor = edge.to === node ? edge.from : edge.to;