@remnic/core 9.7.15 → 9.8.0

package/src/graph-recall.test.ts

@@ -6,8 +6,39 @@ import {
   type GraphRecallOptions,
   runGraphRecall,
 } from "./graph-recall.js";
+import { mergeGraphExpandedResults } from "./orchestration/graph-recall-coordinator.js";
 import type { MemoryEdgeSource } from "./graph-retrieval.js";
 
+test("mergeGraphExpandedResults preserves same paths from different namespaces", () => {
+  const merged = mergeGraphExpandedResults(
+    [
+      {
+        docid: "default-memory",
+        path: "facts/same-id.md",
+        namespace: "default",
+        snippet: "default",
+        score: 0.8,
+      },
+      {
+        docid: "project-memory",
+        path: "facts/same-id.md",
+        namespace: "project-x",
+        snippet: "project",
+        score: 0.7,
+      },
+    ],
+    [],
+  );
+
+  assert.deepEqual(
+    merged.map((result) => [result.namespace, result.path]),
+    [
+      ["default", "facts/same-id.md"],
+      ["project-x", "facts/same-id.md"],
+    ],
+  );
+});
+
 const DEFAULT_CONFIG: GraphRecallConfig = {
   recallGraphEnabled: true,
   recallGraphDamping: 0.85,

package/src/index.ts

@@ -111,6 +111,26 @@ export {
   serializeEntityFile,
   type MemoryWriteResult,
 } from "./storage.js";
+export {
+  composeMemoryEnvelope,
+  buildAccessWriteRequestFingerprint,
+  buildObserveRequestFingerprint,
+  buildWriteIdempotencyPayload,
+  isSealedMemoryEnvelope,
+  sealedWriteToLegacyArgs,
+  TAG_LIMITS,
+  STRUCTURED_ATTRIBUTE_LIMITS,
+  WRITE_FINGERPRINT_FIELDS,
+  FINGERPRINT_EXEMPT_FIELDS,
+  FINGERPRINT_SCOPE_FIELDS,
+  type SealedMemoryEnvelope,
+  type MemoryWriteInput,
+  type WriteContext,
+  type FingerprintScope,
+  type AccessWriteFingerprintParts,
+  type ObserveFingerprintParts,
+  type ComposeEnvelopeOptions,
+} from "./write-envelope.js";
 export {
   getHostEmbeddingProvider,
   normalizeHostEmbeddingVector,
@@ -329,7 +349,7 @@ export { MeilisearchBackend } from "./search/meilisearch-backend.js";
 // ---------------------------------------------------------------------------
 
 export { buildEntityRecallSection } from "./entity-retrieval.js";
-export { resolvePrincipal } from "./namespaces/principal.js";
+export { isNamespacePolicyCovered, resolvePrincipal } from "./namespaces/principal.js";
 export {
   NamespaceCatalog,
   type NamespaceRecord,
@@ -810,7 +830,7 @@ export {
 // ---------------------------------------------------------------------------
 // Offline Sync
 // ---------------------------------------------------------------------------
-
+export { drainPendingImpressionsForOfflineSync, drainPendingLifecycleForOfflineSync } from "./offline-sync-impression-drain.js";
 export {
   OFFLINE_SYNC_APPLY_MAX_BODY_BYTES,
   OFFLINE_SYNC_CHANGESET_FORMAT,

package/src/local-llm.ts

@@ -858,11 +858,9 @@ export class LocalLlmClient {
     });
   }
 
-  private hasQueuedRequests(): boolean {
-    return (
-      this.requestQueues["recall-critical"].length > 0 ||
-      this.requestQueues.background.length > 0
-    );
+  /** True when a background-priority request occupies or awaits this client's local lane (issue #2011). */
+  isBackgroundLaneContended(): boolean {
+    return this.queueProcessing.has("background") || this.requestQueues.background.length > 0;
   }
 
   private dequeueQueuedRequest(priority: LocalLlmRequestPriority): LocalLlmQueuedRequest | null {
@@ -924,7 +922,7 @@ export class LocalLlmClient {
       next.resolve(result);
     } finally {
       this.queueProcessing.delete(next.priority);
-      if (this.hasQueuedRequests()) {
+      if (this.requestQueues["recall-critical"].length > 0 || this.requestQueues.background.length > 0) {
         this.scheduleQueueDrain();
       }
     }

package/src/logger.ts

@@ -60,6 +60,22 @@ export function initLogger(backend?: LoggerBackend, debug?: boolean, options?: L
   _clock = options?.clock ?? (() => new Date().toISOString());
 }
 
+// Whether a real backend has been installed via initLogger. Lets diagnostics
+// that must not be swallowed (config misconfiguration warnings) fall back to
+// console when a host-agnostic consumer uses core without calling initLogger.
+export function isLoggerInitialized(): boolean {
+  return _backend !== NOOP_LOGGER;
+}
+
+// Restore the pristine no-op backend (as at module load, before initLogger).
+// Primarily a test seam for exercising the standalone/no-host path.
+export function resetLogger(): void {
+  _backend = NOOP_LOGGER;
+  _debug = false;
+  _timestamps = true;
+  _clock = () => new Date().toISOString();
+}
+
 // Format one log line: "<iso> remnic: <msg>" when timestamps are enabled,
 // otherwise the legacy "remnic: <msg>". Timestamp is per-call.
 function fmt(msg: string, label = "remnic"): string {

package/src/maintenance/dreams-ledger.ts

@@ -13,7 +13,9 @@
  */
 
 import path from "node:path";
+import { createReadStream, type ReadStream } from "node:fs";
 import { appendFile, lstat, mkdir, readdir, readFile } from "node:fs/promises";
+import { createInterface } from "node:readline";
 
 // ── Types ─────────────────────────────────────────────────────────────────────
 
@@ -108,50 +110,100 @@ export async function appendDreamsLedgerEntry(
 // ── Reader ────────────────────────────────────────────────────────────────────
 
 /**
- * Parse all valid `DreamsLedgerEntry` rows from the ledger file.
- * Malformed lines are silently skipped (resilience over strictness for telemetry).
+ * Parse and validate one ledger line into a `DreamsLedgerEntry`, or `null` when
+ * blank/malformed/incomplete. Shared by the streaming reader and the windowed
+ * status aggregator so both apply the identical fail-open guard.
  */
-export async function readDreamsLedgerEntries(memoryDir: string): Promise<DreamsLedgerEntry[]> {
+function parseDreamsLedgerLine(line: string): DreamsLedgerEntry | null {
+  const trimmed = line.trim();
+  if (!trimmed) return null;
+  let parsed: Record<string, unknown>;
+  try {
+    const raw = JSON.parse(trimmed);
+    if (typeof raw !== "object" || raw === null) return null;
+    parsed = raw as Record<string, unknown>;
+  } catch {
+    return null; // Malformed JSON — skip (fail-open).
+  }
+
+  // Required, strongly-typed fields. A row missing or mistyping any of these is
+  // not a usable telemetry entry.
+  const phase = parsed.phase;
+  if (phase !== "lightSleep" && phase !== "rem" && phase !== "deepSleep") return null;
+  if (
+    typeof parsed.startedAt !== "string" ||
+    typeof parsed.completedAt !== "string" ||
+    typeof parsed.durationMs !== "number" ||
+    typeof parsed.itemsProcessed !== "number"
+  ) {
+    return null;
+  }
+
+  // Optional fields: absence is tolerated (older entries predate them, no
+  // backfill), but a PRESENT value with the wrong type/value marks the row
+  // malformed and skips it — never silently coerced to a default (issue #1910).
+  if ("schemaVersion" in parsed && parsed.schemaVersion !== 1) return null;
+  if ("dryRun" in parsed && typeof parsed.dryRun !== "boolean") return null;
+  if ("trigger" in parsed && parsed.trigger !== "manual" && parsed.trigger !== "scheduled") {
+    return null;
+  }
+
+  return {
+    schemaVersion: 1,
+    startedAt: parsed.startedAt,
+    completedAt: parsed.completedAt,
+    durationMs: parsed.durationMs,
+    phase,
+    itemsProcessed: parsed.itemsProcessed,
+    dryRun: parsed.dryRun === true,
+    trigger: parsed.trigger === "manual" ? "manual" : "scheduled",
+    notes: typeof parsed.notes === "string" ? parsed.notes : undefined,
+  };
+}
+
+/**
+ * Stream the dreams ledger line-by-line. The ledger is plaintext `appendFile`
+ * (never encrypted), so a raw line stream is safe at any size — heap stays
+ * O(1) in the file, unlike the previous whole-file `readFile` + `split`.
+ * Yields nothing when the ledger does not exist yet.
+ */
+export async function* streamDreamsLedgerLines(memoryDir: string): AsyncGenerator<string> {
   const ledgerPath = dreamsLedgerPath(memoryDir);
-  let raw: string;
+  let input: ReadStream;
+  try {
+    input = createReadStream(ledgerPath, { encoding: "utf8" });
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException).code === "ENOENT") return;
+    throw err;
+  }
+  const rl = createInterface({ input, crlfDelay: Infinity });
   try {
-    raw = await readFile(ledgerPath, "utf-8");
+    for await (const line of rl) yield line;
   } catch (err) {
-    const code = (err as NodeJS.ErrnoException).code;
-    if (code === "ENOENT") return [];
+    if ((err as NodeJS.ErrnoException).code === "ENOENT") return;
     throw err;
+  } finally {
+    // Close the readline interface AND destroy the underlying stream. When a
+    // consumer stops early (e.g. `getDreamsStatus` breaks out), the generator's
+    // `.return()` runs this finally; `rl.close()` alone leaves the file
+    // descriptor open, so destroy the stream to release it (issue #1910).
+    rl.close();
+    input.destroy();
   }
+}
 
+/**
+ * Parse all valid `DreamsLedgerEntry` rows from the ledger file. Thin wrapper
+ * over the line stream so any caller that genuinely needs every row still
+ * works; `getDreamsStatus` streams directly and filters in-window instead.
+ * Malformed lines are silently skipped (resilience over strictness for
+ * telemetry).
+ */
+export async function readDreamsLedgerEntries(memoryDir: string): Promise<DreamsLedgerEntry[]> {
   const entries: DreamsLedgerEntry[] = [];
-  for (const line of raw.split("\n")) {
-    const trimmed = line.trim();
-    if (!trimmed) continue;
-    try {
-      const parsed = JSON.parse(trimmed) as Partial<DreamsLedgerEntry>;
-      if (typeof parsed !== "object" || parsed === null) continue;
-      if (
-        typeof parsed.phase === "string" &&
-        (parsed.phase === "lightSleep" || parsed.phase === "rem" || parsed.phase === "deepSleep") &&
-        typeof parsed.startedAt === "string" &&
-        typeof parsed.completedAt === "string" &&
-        typeof parsed.durationMs === "number" &&
-        typeof parsed.itemsProcessed === "number"
-      ) {
-        entries.push({
-          schemaVersion: 1,
-          startedAt: parsed.startedAt,
-          completedAt: parsed.completedAt,
-          durationMs: parsed.durationMs,
-          phase: parsed.phase,
-          itemsProcessed: parsed.itemsProcessed,
-          dryRun: parsed.dryRun === true,
-          trigger: parsed.trigger === "manual" ? "manual" : "scheduled",
-          notes: typeof parsed.notes === "string" ? parsed.notes : undefined,
-        });
-      }
-    } catch {
-      // Malformed line — skip.
-    }
+  for await (const line of streamDreamsLedgerLines(memoryDir)) {
+    const entry = parseDreamsLedgerLine(line);
+    if (entry) entries.push(entry);
   }
   return entries;
 }
@@ -199,13 +251,8 @@ export async function getDreamsStatus(
     throw new RangeError("windowHours produces an invalid status window");
   }
 
-  const entries = await readDreamsLedgerEntries(memoryDir);
-
-  // Filter to entries within the window based on completedAt.
-  const windowEntries = entries.filter((e) => {
-    const ts = Date.parse(e.completedAt);
-    return Number.isFinite(ts) && ts >= windowStart.getTime() && ts < windowEnd.getTime();
-  });
+  const windowStartMs = windowStart.getTime();
+  const windowEndMs = windowEnd.getTime();
 
   const statusMap = new Map<DreamsPhase, DreamsPhaseStatus>();
   for (const phase of ALL_PHASES) {
@@ -219,18 +266,22 @@ export async function getDreamsStatus(
     });
   }
 
-  for (const entry of windowEntries) {
+  // Stream the ledger and aggregate every in-window row inline, so heap use is
+  // O(number of phases) regardless of window or file size (issue #1910) — no
+  // per-row array and no row cap, so a large in-window window is still counted
+  // in full rather than truncated.
+  for await (const line of streamDreamsLedgerLines(memoryDir)) {
+    const entry = parseDreamsLedgerLine(line);
+    if (!entry) continue;
+    const ts = Date.parse(entry.completedAt);
+    if (!Number.isFinite(ts) || ts < windowStartMs || ts >= windowEndMs) continue;
     const status = statusMap.get(entry.phase);
     if (!status) continue;
     status.runCount += 1;
     status.totalDurationMs += entry.durationMs;
     status.totalItemsProcessed += entry.itemsProcessed;
-
     // Track the most recent run (by completedAt).
-    if (
-      status.lastRunAt === null ||
-      Date.parse(entry.completedAt) > Date.parse(status.lastRunAt)
-    ) {
+    if (status.lastRunAt === null || ts > Date.parse(status.lastRunAt)) {
       status.lastRunAt = entry.completedAt;
       status.lastDurationMs = entry.durationMs;
     }

package/src/maintenance/rebuild-memory-lifecycle-ledger-cli.ts

@@ -0,0 +1,77 @@
+import path from "node:path";
+
+import { probeEncryptedRegularFileHeader, SECURE_STORE_ENVELOPE_OVERHEAD_BYTES } from "../secure-store/secure-fs.js";
+import { STATE_FILE_MAX_DECRYPT_BYTES } from "../storage/secure-line-reader.js";
+import type { StorageManager } from "../index.js";
+import {
+  rebuildMemoryLifecycleLedger,
+  type RebuildMemoryLifecycleLedgerResult,
+} from "./rebuild-memory-lifecycle-ledger.js";
+
+export interface RebuildMemoryLifecycleLedgerCliCommandOptions {
+  memoryDir: string;
+  write?: boolean;
+  now?: Date;
+  /**
+   * Live storage for secure-store deployments (#2033). When the store is
+   * encrypted at rest, the rebuild must read encrypted memories and re-encrypt
+   * the rewritten ledger through this unlocked StorageManager; a keyless manager
+   * would read with no key and rewrite the ledger as plaintext. The command
+   * refuses when the on-disk ledger is encrypted and no unlocked key is present.
+   */
+  storage?: StorageManager;
+  /**
+   * Read/decrypt cap the rewritten ledger must land strictly below. Defaults to
+   * {@link STATE_FILE_MAX_DECRYPT_BYTES} (the reader's whole-file refusal cap).
+   * Exposed as an override so recovery tooling and tests can bound the rewrite
+   * deterministically, mirroring the scheduler's `lifecycleLedgerMaxBytes` seam.
+   */
+  maxLedgerBytesCap?: number;
+}
+
+export async function runRebuildMemoryLifecycleLedgerCliCommand(
+  options: RebuildMemoryLifecycleLedgerCliCommandOptions,
+): Promise<RebuildMemoryLifecycleLedgerResult> {
+  const storage = options.storage;
+  const ledgerPath = path.join(options.memoryDir, "state", "memory-lifecycle-ledger.jsonl");
+  const encrypted = await probeEncryptedRegularFileHeader(ledgerPath);
+  // Refuse a keyless plaintext rewrite of an encrypted-at-rest ledger (#2033):
+  // without an unlocked secure StorageManager the rebuild would read encrypted
+  // memories with no key and rewrite the ledger as plaintext, defeating
+  // encryption-at-rest. Fail safely and point the operator at the unlock step.
+  if (encrypted && !(storage?.isSecureStoreUnlocked() ?? false)) {
+    throw new Error(
+      "rebuild-memory-lifecycle-ledger: secure store is locked; refusing to rebuild the "
+      + "lifecycle ledger, which would read encrypted memories with no key and rewrite the "
+      + "ledger as plaintext. Run `remnic secure-store unlock` first, then re-run with --write.",
+    );
+  }
+  // This command is the advertised recovery for an over-cap ledger, so it must
+  // give the SAME guarantees as background compaction (#2033): (1) preserve
+  // append-only history frontmatter cannot reconstruct (explicit_capture_accepted,
+  // imported, promoted) instead of silently dropping it, and (2) bound the
+  // rewritten ledger under the read/decrypt cap so the repaired ledger is
+  // readable. Bound the PLAINTEXT payload, reserving the secure-store envelope
+  // (+1 byte) whenever the REPLACEMENT will be encrypted — NOT only when the
+  // current header is encrypted. A plaintext ledger rewritten through an unlocked
+  // encrypt-on-write store becomes encrypted, so it needs the reserve even though
+  // its current header is plaintext; base the decision on
+  // `willEncryptStateWrites()` exactly as auto-compaction does (#2033 write-mode
+  // finding), so the on-disk file lands STRICTLY under the cap.
+  const cap = options.maxLedgerBytesCap ?? STATE_FILE_MAX_DECRYPT_BYTES;
+  const replacementEncrypted = encrypted || (storage?.willEncryptStateWrites() ?? false);
+  const maxLedgerBytes = replacementEncrypted
+    ? cap - SECURE_STORE_ENVELOPE_OVERHEAD_BYTES - 1
+    : cap - 1;
+  return rebuildMemoryLifecycleLedger({
+    memoryDir: options.memoryDir,
+    dryRun: options.write !== true,
+    now: options.now,
+    // Carry the live (unlocked) secure StorageManager so encrypted memories
+    // decrypt on read and the rewritten ledger stays encrypted at rest; a
+    // plaintext store passes a plaintext manager here, which is correct.
+    storage,
+    preserveExistingEvents: true,
+    maxLedgerBytes,
+  });
+}