@remnic/core 10.0.6 → 10.0.8

package/src/harmonic-source-lookup.ts

@@ -0,0 +1,142 @@
+import path from "node:path";
+import { existsSync } from "node:fs";
+import { mkdir, realpath } from "node:fs/promises";
+import { throwIfAborted } from "./abort-error.js";
+import { readJsonFile, writeJsonFileAtomic } from "./json-store.js";
+import { readProjectedMemoryPaths } from "./memory-projection-batch.js";
+import { SecureStoreLockedError } from "./secure-store/secure-fs.js";
+import { isPathInsideStorageRoot, isSafeLegacyPathSegment } from "./storage-paths.js";
+import { isRecord } from "./store-contract.js";
+import type { StorageManager } from "./storage.js";
+import type { MemoryFile } from "./types.js";
+import { RECALL_FALLBACK_DIRS } from "./utils/category-dir.js";
+
+const LOOKUP_TTL_MS = 10 * 60_000;
+const MAX_CACHE_ENTRIES = 4096;
+type PathEntry = { path: string | null; expiresAt: number };
+
+/** Direct and cached probes must retain the walk's realpath containment check. */
+export async function readHarmonicSourceMemory(
+  storage: StorageManager,
+  rootReal: string,
+  filePath: string,
+  abortSignal?: AbortSignal,
+): Promise<MemoryFile | null> {
+  throwIfAborted(abortSignal, "harmonic retrieval aborted");
+  if (!existsSync(filePath)) return null;
+  try {
+    if (!isPathInsideStorageRoot(rootReal, await realpath(filePath))) return null;
+    return await storage.readMemoryByPath(filePath);
+  } catch (err) {
+    throwIfAborted(abortSignal, "harmonic retrieval aborted");
+    if (err instanceof SecureStoreLockedError) throw err;
+    return null;
+  }
+}
+
+function isCorpusPath(memoryDir: string, filePath: string): boolean {
+  if (!isPathInsideStorageRoot(memoryDir, filePath)) return false;
+  const [dir] = path.relative(memoryDir, filePath).split(path.sep);
+  return dir === "cold" || RECALL_FALLBACK_DIRS.includes(dir);
+}
+
+/** A cache miss is not authoritative: unreadable/invalid cache state keeps the walk. */
+async function readPathCache(
+  filePath: string, rebuiltAt: string, corpusVersion: string, nowMs: number,
+): Promise<Map<string, PathEntry>> {
+  try {
+    const raw = await readJsonFile(filePath);
+    if (!isRecord(raw) || raw.rebuiltAt !== rebuiltAt || raw.corpusVersion !== corpusVersion || !Array.isArray(raw.entries)) return new Map();
+    const entries = new Map<string, PathEntry>();
+    for (const row of raw.entries) {
+      if (!Array.isArray(row) || row.length !== 3) continue;
+      const [id, storedPath, expiresAt] = row;
+      if (typeof id !== "string" || (storedPath !== null && typeof storedPath !== "string")) continue;
+      if (typeof expiresAt !== "number" || !Number.isFinite(expiresAt) || expiresAt <= nowMs || expiresAt > nowMs + LOOKUP_TTL_MS) continue;
+      entries.set(id, { path: storedPath, expiresAt });
+    }
+    return entries;
+  } catch {
+    return new Map();
+  }
+}
+
+/** Resolve only unresolved ids; persist paths/absence, never memory content. */
+export async function readUnresolvedHarmonicSources(options: {
+  storage: StorageManager;
+  rootReal: string;
+  rebuiltAt: string;
+  sourceMemoryIds: string[];
+  nowMs: number;
+  abortSignal?: AbortSignal;
+}): Promise<Map<string, MemoryFile>> {
+  const { storage, rootReal, rebuiltAt, nowMs, abortSignal } = options;
+  throwIfAborted(abortSignal, "harmonic retrieval aborted");
+  const corpusVersion = storage.getCorpusScanVersion();
+  const stateDir = path.join(storage.dir, "state");
+  const cachePath = path.join(stateDir, "harmonic-source-paths.json");
+  const stateReal = await realpath(stateDir).catch(() => null);
+  const safeCache = stateReal !== null && isPathInsideStorageRoot(rootReal, stateReal);
+  const cache = safeCache ? await readPathCache(cachePath, rebuiltAt, corpusVersion, nowMs) : new Map<string, PathEntry>();
+  const found = new Map<string, MemoryFile>();
+  const pending = new Set<string>();
+  for (const id of options.sourceMemoryIds) {
+    throwIfAborted(abortSignal, "harmonic retrieval aborted");
+    const entry = cache.get(id);
+    if (entry?.path === null) continue;
+    if (entry?.path) {
+      const candidate = path.resolve(storage.dir, entry.path);
+      const memory = isCorpusPath(storage.dir, candidate)
+        ? await readHarmonicSourceMemory(storage, rootReal, candidate, abortSignal) : null;
+      if (memory?.frontmatter.id === id) {
+        found.set(id, memory);
+        continue;
+      }
+    }
+    pending.add(id);
+  }
+  if (pending.size === 0) return found;
+
+  // Opaque legacy ids previously used the full scan, including edited projected files.
+  const inspectProjected = [...pending].some((id) => !isSafeLegacyPathSegment(id));
+  const projectedPaths = readProjectedMemoryPaths(storage.dir);
+  const walked = [
+    ...(await storage.collectActiveMemoryPaths({ abortSignal, propagateReadErrors: true })),
+    ...(await storage.collectColdMemoryPaths({ abortSignal, propagateReadErrors: true })),
+  ];
+  for (const filePath of walked) {
+    throwIfAborted(abortSignal, "harmonic retrieval aborted");
+    // Existing projected files need no parse unless their filename names a wanted id.
+    // New/imported files must be keyed by frontmatter id, not their basename (#3188).
+    if (!inspectProjected && !pending.has(path.basename(filePath, ".md")) && projectedPaths?.has(filePath)) continue;
+    const memory = await readHarmonicSourceMemory(storage, rootReal, filePath, abortSignal);
+    if (!memory) continue;
+    const id = memory.frontmatter.id;
+    if (pending.has(id)) {
+      found.set(id, memory);
+      cache.set(id, { path: path.relative(storage.dir, filePath), expiresAt: nowMs + LOOKUP_TTL_MS });
+    }
+  }
+  for (const id of pending) {
+    if (!found.has(id)) cache.set(id, { path: null, expiresAt: nowMs + LOOKUP_TTL_MS });
+  }
+  throwIfAborted(abortSignal, "harmonic retrieval aborted");
+  // A concurrent mutation must not publish absence for a different corpus epoch.
+  if (storage.getCorpusScanVersion() === corpusVersion) {
+    try {
+      await mkdir(stateDir, { recursive: true });
+      if (isPathInsideStorageRoot(rootReal, await realpath(stateDir))) {
+        throwIfAborted(abortSignal, "harmonic retrieval aborted");
+        // ponytail: atomic last-writer-wins cache; contention can repeat a walk, not lose sources.
+        await writeJsonFileAtomic(cachePath, {
+          rebuiltAt, corpusVersion,
+          entries: [...cache].slice(-MAX_CACHE_ENTRIES).map(([id, entry]) => [id, entry.path, entry.expiresAt]),
+        });
+      }
+    } catch {
+      throwIfAborted(abortSignal, "harmonic retrieval aborted");
+      // Cache publication is optional; resolved memories remain usable on a read-only store.
+    }
+  }
+  return found;
+}

package/src/http-retry.ts

@@ -13,6 +13,8 @@
  * `@remnic/core/abort-error`) — the root index stays untouched.
  */
 
+import { sleep as defaultSleep } from "./utils/sleep.js";
+
 export interface RetryingFetchOptions {
   /** Request init for every attempt. Ignored when `buildInit` is set. */
   init?: RequestInit;
@@ -70,12 +72,6 @@ const DEFAULT_MAX_RETRIES = 3;
 const DEFAULT_MAX_RETRY_DELAY_MS = 30_000;
 const DEFAULT_BACKOFF_BASE_MS = 1_000;
 
-function defaultSleep(ms: number): Promise<void> {
-  const { promise, resolve } = Promise.withResolvers<void>();
-  setTimeout(resolve, ms);
-  return promise;
-}
-
 /**
  * Network/timeout failures wrap Node error text that can carry loader
  * paths or stack fragments; sync errors reach MCP clients verbatim, so

package/src/memory-projection-batch.test.ts

@@ -0,0 +1,55 @@
+import assert from "node:assert/strict";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import test from "node:test";
+import { readProjectedMemoryStates } from "./memory-projection-batch.js";
+import { getMemoryProjectionPath, readProjectedMemoryState } from "./memory-projection-store.js";
+import { rebuildMemoryProjection } from "./maintenance/rebuild-memory-projection.js";
+import { openBetterSqlite3 } from "./runtime/better-sqlite.js";
+import { StorageManager } from "./storage.js";
+
+test("projection batches return only requested rows and preserve completeness metadata", async () => {
+  const memoryDir = await mkdtemp(path.join(tmpdir(), "remnic-projection-batch-"));
+  try {
+    assert.equal(readProjectedMemoryStates(memoryDir, ["missing"]), null);
+    const storage = new StorageManager(memoryDir);
+    await storage.ensureDirectories();
+    const source = await storage.writeMemory("fact", "batch source payload", {
+      source: "test", tags: ["zeta", "alpha"], entityRef: "person:narwhal", confidence: 0.9,
+    });
+    const excluded = await storage.writeMemory("fact", "unrequested source payload", { source: "test" });
+    await rebuildMemoryProjection({ memoryDir, storage, dryRun: false });
+
+    const batch = readProjectedMemoryStates(memoryDir, new Set([source.id, "missing"]));
+    assert.ok(batch);
+    assert.deepEqual([...batch.states.keys()], [source.id]);
+    assert.equal(batch.states.has(excluded.id), false);
+    assert.deepEqual(batch.states.get(source.id), readProjectedMemoryState(memoryDir, source.id));
+    assert.deepEqual(batch.states.get(source.id)?.tags, ["alpha", "zeta"]);
+    assert.equal(batch.states.get(source.id)?.preview, "batch source payload");
+    assert.equal(batch.complete, true);
+    assert.ok(Number.isFinite(Date.parse(batch.rebuiltAt ?? "")));
+    assert.deepEqual(readProjectedMemoryStates(memoryDir, []), { ...batch, states: new Map() });
+
+    const db = openBetterSqlite3(getMemoryProjectionPath(memoryDir));
+    try {
+      const stamp = db.prepare("SELECT value FROM meta WHERE key = 'rebuiltAt'").get() as { value: string };
+      assert.equal(batch.rebuiltAt, stamp.value);
+      db.prepare("UPDATE meta SET value = 'false' WHERE key = 'complete'").run();
+      assert.equal(readProjectedMemoryStates(memoryDir, [source.id])?.complete, false);
+      db.prepare("DELETE FROM meta WHERE key IN ('complete', 'rebuiltAt')").run();
+      const legacy = readProjectedMemoryStates(memoryDir, [source.id]);
+      assert.ok(legacy);
+      assert.equal(legacy.complete, false);
+      assert.equal(legacy.rebuiltAt, null);
+      assert.deepEqual(legacy.states, batch.states);
+      db.exec("DROP TABLE memory_current");
+      assert.equal(readProjectedMemoryStates(memoryDir, [source.id]), null, "unreadable schema must fall back");
+    } finally {
+      db.close();
+    }
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});

package/src/memory-projection-batch.ts

@@ -1,4 +1,6 @@
+import path from "node:path";
 import type { MemoryProjectionCurrentState } from "./types.js";
+import { isRecord } from "./store-contract.js";
 import {
   memoryCurrentSelectExpressions,
   openProjectionReadonly,
@@ -80,3 +82,21 @@ export function readProjectedMemoryStates(
     db.close();
   }
 }
+
+/** Path-only census used to avoid parsing already-projected files during a fallback lookup. */
+export function readProjectedMemoryPaths(memoryDir: string): Set<string> | null {
+  const db = openProjectionReadonly(memoryDir);
+  if (!db) return null;
+  try {
+    const paths = new Set<string>();
+    for (const row of db.prepare("SELECT path_rel FROM memory_current").iterate()) {
+      const relative = isRecord(row) ? row.path_rel : undefined;
+      if (typeof relative === "string") paths.add(path.resolve(memoryDir, relative));
+    }
+    return paths;
+  } catch {
+    return null;
+  } finally {
+    db.close();
+  }
+}

package/src/offline-sync-digest-cache.test.ts

@@ -0,0 +1,45 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { parseOfflineSyncDigestCache } from "./offline-sync-digest-cache.js";
+
+const digest = {
+  statBytes: 120, mtimeMs: 1_760_000_000_000, ctimeMs: 1_760_000_000_500,
+  encrypted: false, sha256: "a".repeat(64), bytes: 118,
+};
+const entry = { path: "facts/alpha.md", ...digest };
+const cacheFile = (entries: unknown[]) => JSON.stringify({ version: 1, entries });
+
+test("digest cache parses valid plaintext and encrypted entries without losing fields", () => {
+  const encrypted = { ...digest, encrypted: true, bytes: 0, sha256: "ABCDEF0123456789".repeat(4) };
+  assert.deepEqual([...parseOfflineSyncDigestCache(cacheFile([
+    entry, { path: "facts/beta.md", ...encrypted },
+  ]))], [[entry.path, digest], ["facts/beta.md", encrypted]]);
+});
+
+test("digest cache skips legacy records missing stat data but does not expire old timestamps", () => {
+  const old = { ...digest, mtimeMs: 1, ctimeMs: 1 };
+  const legacy = ["statBytes", "mtimeMs", "ctimeMs", "bytes", "sha256"].map((field) => ({
+    ...entry, [field]: undefined,
+  }));
+  // Freshness is checked against the file's stat by the caller, not by an age-based TTL here.
+  assert.deepEqual([...parseOfflineSyncDigestCache(cacheFile([
+    ...legacy, { path: "facts/old.md", ...old },
+  ]))], [["facts/old.md", old]]);
+});
+
+test("digest cache rejects garbage shapes and unsafe records while retaining valid siblings", () => {
+  for (const raw of ["null", "[]", "42", "{}", '{"entries":{}}']) {
+    assert.equal(parseOfflineSyncDigestCache(raw).size, 0);
+  }
+  const invalid = [
+    null, [], "garbage",
+    ...["", "..", "../escape.md", "/absolute.md"].map((path) => ({ ...entry, path })),
+    { ...entry, sha256: "z".repeat(64) }, { ...entry, sha256: "abc" },
+    { ...entry, statBytes: "120" }, { ...entry, mtimeMs: 1e400 },
+    { ...entry, ctimeMs: null }, { ...entry, bytes: "118" },
+  ];
+  assert.deepEqual([...parseOfflineSyncDigestCache(cacheFile([...invalid, entry]))], [[entry.path, digest]]);
+  for (const raw of ["", "{", '{"entries":[}']) {
+    assert.throws(() => parseOfflineSyncDigestCache(raw), SyntaxError);
+  }
+});

package/src/orchestration/recall-internal.ts

@@ -3648,18 +3648,6 @@ export class RecallInternalCoordinator {
       this.deps.appendRecallSection(sectionBuckets, "trust-zones", trustZoneSection);
     }
 
-    const harmonicRetrievalSection = await awaitEnrichmentSection(
-      "harmonic",
-      harmonicRetrievalPromise,
-    );
-    if (harmonicRetrievalSection) {
-      this.deps.appendRecallSection(
-        sectionBuckets,
-        "harmonic-retrieval",
-        harmonicRetrievalSection,
-      );
-    }
-
     if (verifiedRecallSection) {
       this.deps.appendRecallSection(
         sectionBuckets,
@@ -4780,6 +4768,18 @@ export class RecallInternalCoordinator {
       }
     }
     const phase2AfterQmdMs = Date.now() - recallStart;
+
+    const harmonicRetrievalSection = await awaitEnrichmentSection(
+      "harmonic",
+      harmonicRetrievalPromise,
+    );
+    if (harmonicRetrievalSection) {
+      this.deps.appendRecallSection(
+        sectionBuckets,
+        "harmonic-retrieval",
+        harmonicRetrievalSection,
+      );
+    }
     if (shouldPersistGraphSnapshot) {
       if (!graphSnapshotStatus) {
         graphSnapshotStatus = "skipped";

package/src/procedural/procedure-miner.ts

@@ -14,6 +14,7 @@ import { clusterByKey } from "./reinforcement-core.js";
 import { log } from "../logger.js";
 import { composeMemoryEnvelope, STRUCTURED_ATTRIBUTE_LIMITS } from "../write-envelope.js";
 import { isErrnoCode } from "../utils/errno.js";
+import { sleep } from "../utils/sleep.js";
 
 /** Must match truncation on `procedure_cluster` structured attribute (dedupe + storage). */
 const PROCEDURE_CLUSTER_ATTR_MAX = 500;
@@ -51,10 +52,6 @@ function clusterKeyHash(cluster: string): string {
   return createHash("sha256").update(cluster).digest("hex");
 }
 
-function sleep(ms: number): Promise<void> {
-  return new Promise((resolve) => setTimeout(resolve, ms));
-}
-
 async function acquireClusterWriteLock(memoryDir: string, clusterHash: string): Promise<() => Promise<void>> {
   const lockRoot = path.join(path.resolve(memoryDir), "state", "procedure-miner-locks");
   const lockDir = path.join(lockRoot, `${clusterHash}.lock`);

package/src/qmd-daemon-session.ts

@@ -2,6 +2,7 @@ import { abortError, throwIfAborted } from "./abort-error.js";
 import { log } from "./logger.js";
 import { type CommandChildProcess, launchProcess } from "./runtime/child-process.js";
 import { mergeEnv } from "./runtime/env.js";
+import { sleep } from "./utils/sleep.js";
 
 export type QmdRuntimeEnv = Record<string, string | undefined>;
 
@@ -20,10 +21,6 @@ type PendingRequest = {
 
 let nextJsonRpcId = 1;
 
-function sleep(ms: number): Promise<void> {
-  return new Promise((resolve) => setTimeout(resolve, ms));
-}
-
 /**
  * Drop C0 controls and one ANSI CSI sequence. Char scan on purpose:
  * the regex-safety ratchet flags new regular expressions on changed lines.

package/src/qmd.ts

@@ -218,10 +218,6 @@ function getGlobalQmdState(): QmdGlobalState {
   return state;
 }
 
-function sleep(ms: number): Promise<void> {
-  return new Promise((r) => setTimeout(r, ms));
-}
-
 function errorMessage(err: unknown): string {
   if (typeof err === "string") return err;
   if (err instanceof Error) return err.message;