@remnic/core 9.38.2 → 9.39.0

package/src/orchestration/maintenance.test.ts

@@ -21,6 +21,7 @@ import path from "node:path";
 import { readProjectionRebuiltAt } from "../maintenance/projection-support.js";
 import { lstat, mkdir, mkdtemp, readdir, readFile, rm, stat, symlink, writeFile } from "node:fs/promises";
 import { execFileSync } from "node:child_process";
+import { setTimeout as delay } from "node:timers/promises";
 
 import { MaintenanceScheduler } from "./maintenance.js";
 import {
@@ -2952,3 +2953,32 @@ test("scheduled projection rebuild skips when the projection was rebuilt within
     await rm(memoryDir, { recursive: true, force: true });
   }
 });
+
+test("maintenance tool request invokes the scheduled projection rebuild wiring (#2119)", async () => {
+  const memoryDir = await seedMemoryDirWithOneFact();
+  const scheduler = new MaintenanceScheduler({
+    config: fixtureConfig({
+      memoryDir,
+      projectionRebuildEnabled: true,
+      projectionRebuildIntervalMs: 6 * 60 * 60 * 1000,
+    }),
+    getQmd: () => ({ isAvailable: () => false }) as unknown as SearchBackend,
+    namespaceSearchRouter: {} as unknown as NamespaceSearchRouter,
+    namespaceCatalog: {} as unknown as NamespaceCatalog,
+  });
+  try {
+    scheduler.requestQmdMaintenanceForTool("test");
+    for (let attempt = 0; attempt < 100 && probeProjectionHealth(memoryDir).state === "absent"; attempt += 1) {
+      await delay(10);
+    }
+    assert.equal(
+      probeProjectionHealth(memoryDir).state,
+      "openable",
+      "the public maintenance request entrypoint must reach the projection rebuild",
+    );
+    assert.equal(readProjectedMemoryBrowse(memoryDir, { limit: 5, offset: 0 })?.total, 1);
+  } finally {
+    await scheduler.dispose();
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});

package/src/orchestrator-flush.test.ts

@@ -1781,6 +1781,57 @@ test("runExtraction preserves empty-result buffers when fingerprint persistence
   assert.equal(clearCalls, 0);
 });
 
+test("runExtraction keeps the empty live path fail-open when liveness metadata write fails (#2223)", async () => {
+  const config = parseConfig({});
+  config.extractionMinChars = 0;
+  config.extractionMinUserTurns = 1;
+
+  let clearCalls = 0;
+  let saveMetaCalls = 0;
+  const meta = {
+    extractionCount: 0,
+    lastExtractionAt: null,
+    lastConsolidationAt: null,
+    totalMemories: 0,
+    totalEntities: 0,
+    processedExtractionFingerprints: [] as Array<{ fingerprint: string; observedAt: string }>,
+  };
+
+  const orchestrator = Object.create(Orchestrator.prototype) as any;
+  orchestrator.config = config;
+  orchestrator.buffer = {
+    clearAfterExtraction: async () => {
+      clearCalls += 1;
+    },
+  };
+  orchestrator.storageRouter = {
+    storageFor: async () => ({
+      listEntityNames: async () => [],
+      loadMeta: async () => meta,
+      saveMeta: async () => {
+        saveMetaCalls += 1;
+        throw new Error("meta save failed");
+      },
+    }),
+  };
+  orchestrator.extraction = {
+    extract: async () => ({ facts: [], entities: [], questions: [], profileUpdates: [] }),
+  };
+
+  // Normal live turn (no persistProcessedFingerprint): the liveness stamp is
+  // diagnostic only, so a metadata write failure must neither reject the task
+  // nor retain the buffer — extraction succeeded and produced nothing durable.
+  const result = await orchestrator.runExtraction(
+    [makeTurn("session-live-empty", "a transient note not worth remembering")],
+    { bufferKey: "session-live-empty" },
+  );
+
+  assert.equal(result.status, "skipped");
+  assert.equal(result.reason, "empty_extraction_result");
+  assert.equal(saveMetaCalls, 1, "the liveness stamp was attempted");
+  assert.equal(clearCalls, 1, "fail-open: the live buffer clears despite the metadata write failure");
+});
+
 test("runExtraction preserves deduped buffers when the caller aborts during meta load", async () => {
   const config = parseConfig({});
   config.extractionMinChars = 0;

package/src/storage/memory-lifecycle-ledger-access.ts

@@ -1,7 +1,7 @@
 import { SecureStoreLockedError } from "../secure-store/secure-fs.js";
 import { isErrnoCode } from "../utils/errno.js";
 import { displayErrorDetail } from "../runtime/better-sqlite.js";
-import { rename, stat, unlink } from "node:fs/promises";
+import { lstat, rename, stat, unlink } from "node:fs/promises";
 import { randomUUID } from "node:crypto";
 import path from "node:path";
 import type { MemoryLifecycleEvent } from "../types.js";
@@ -19,6 +19,15 @@ import {
   readMemoryLifecycleEventsFromLines,
   STATE_FILE_MAX_DECRYPT_BYTES,
 } from "./secure-line-reader.js";
+import { getMemoryProjectionPath } from "../memory-projection-store.js";
+import {
+  openProjectionLifecycleLedgerLagTracker,
+  type ProjectionLifecycleLedgerLagTracker,
+} from "../maintenance/projection-support.js";
+import {
+  type ProjectionLedgerLagTelemetry,
+  warnProjectionFallback,
+} from "../storage-guards.js";
 
 /** Reads a lifecycle-ledger file through the secure whole-file/plaintext line source. */
 export type LedgerSecureReader = (filePath: string) => Promise<string>;
@@ -190,6 +199,194 @@ export async function readBoundedLifecycleEventsFromLedger(
   }
 }
 
+export interface LifecycleLedgerProjectionLagRead {
+  events: MemoryLifecycleEvent[];
+  currentEventCount: number;
+  deltaEvents: number;
+  telemetryAvailable: boolean;
+}
+
+export async function readBoundedLifecycleEventsWithProjectionLag(
+  ledgerPath: string,
+  readSecureFile: LedgerSecureReader,
+  limit: number,
+  memoryId: string,
+  tracker: ProjectionLifecycleLedgerLagTracker,
+): Promise<LifecycleLedgerProjectionLagRead> {
+  let currentEventCount = 0;
+  let deltaEvents = 0;
+  let telemetryAvailable = true;
+  try {
+    const events = await readMemoryLifecycleEventsFromLines(
+      readMaybeEncryptedLines(
+        ledgerPath,
+        () => readSecureFile(ledgerPath),
+        STATE_FILE_MAX_DECRYPT_BYTES,
+      ),
+      Math.max(0, Math.floor(limit)),
+      memoryId,
+      compareMemoryLifecycleEvents,
+      (event) => {
+        if (!telemetryAvailable) return;
+        try {
+          const tracked = tracker.record(event);
+          if (!tracked.unique) return;
+          currentEventCount += 1;
+          if (!tracked.projected) deltaEvents += 1;
+        } catch {
+          telemetryAvailable = false;
+          currentEventCount = 0;
+          deltaEvents = 0;
+        }
+      },
+    );
+    return { events, currentEventCount, deltaEvents, telemetryAvailable };
+  } catch (err) {
+    if (err instanceof SecureStoreLockedError) throw err;
+    if (!isErrnoCode(err, "ENOENT")) throw err;
+    return { events: [], currentEventCount: 0, deltaEvents: 0, telemetryAvailable: false };
+  }
+}
+/**
+ * Probe filesystem identity stamps (dev:ino:size:mtimeMs:ctimeMs) for the
+ * projection database and lifecycle-ledger file. Returns `null` when either
+ * file is missing or unstatable so caller can disable the lag cache without
+ * throwing on transient I/O or permission errors (#2119 / review).
+ */
+export async function probeProjectionLedgerIdentities(
+  projectionPath: string,
+  ledgerPath: string,
+): Promise<{ projectionIdentity: string; ledgerIdentity: string } | null> {
+  try {
+    const [projectionIdentity, ledgerIdentity] = await Promise.all(
+      [projectionPath, ledgerPath].map(async (filePath) => {
+        try {
+          const file = await lstat(filePath);
+          return `${file.dev}:${file.ino}:${file.size}:${file.mtimeMs}:${file.ctimeMs}`;
+        } catch (err) {
+          if (isErrnoCode(err, "ENOENT")) return "absent";
+          // Probe failure: disable lag cache, do not block the primary timeline read (#2119 review).
+          return null;
+        }
+      }),
+    );
+    if (!projectionIdentity || !ledgerIdentity) return null;
+    return { projectionIdentity, ledgerIdentity };
+  } catch {
+    return null;
+  }
+}
+
+export interface ProjectionLedgerLagScan {
+  telemetry: ProjectionLedgerLagTelemetry | null;
+  events: MemoryLifecycleEvent[];
+  memoryId: string;
+  limit: number;
+}
+
+export class ProjectionLedgerLagManager {
+  private lagCache: {
+    generation: string;
+    telemetry: ProjectionLedgerLagTelemetry;
+  } | null = null;
+  private lagInFlight: {
+    generation: string;
+    scan: Promise<ProjectionLedgerLagScan>;
+  } | null = null;
+
+  async resolveTimelineWithLag(options: {
+    baseDir: string;
+    ledgerPath: string;
+    memoryId: string;
+    cappedLimit: number;
+    readSecureFile: LedgerSecureReader;
+    projectionAge: () => string;
+  }): Promise<MemoryLifecycleEvent[]> {
+    const { baseDir, ledgerPath, memoryId, cappedLimit, readSecureFile, projectionAge } = options;
+    const projectionPath = getMemoryProjectionPath(baseDir);
+    const identities = await probeProjectionLedgerIdentities(projectionPath, ledgerPath);
+
+    if (!identities) {
+      warnProjectionFallback(baseDir, "getMemoryTimeline", projectionAge);
+      return readBoundedLifecycleEventsFromLedger(ledgerPath, readSecureFile, cappedLimit, memoryId);
+    }
+
+    const { projectionIdentity, ledgerIdentity } = identities;
+    const generation = `${projectionIdentity}\0${ledgerIdentity}`;
+
+    const cachedLag = this.lagCache?.generation === generation
+      ? this.lagCache.telemetry
+      : null;
+    if (cachedLag) {
+      warnProjectionFallback(baseDir, "getMemoryTimeline", projectionAge, cachedLag);
+      return readBoundedLifecycleEventsFromLedger(ledgerPath, readSecureFile, cappedLimit, memoryId);
+    }
+
+    const inFlight = this.lagInFlight?.generation === generation
+      ? this.lagInFlight.scan
+      : null;
+    if (inFlight) {
+      const result = await inFlight;
+      if (result.telemetry) {
+        this.lagCache = { generation, telemetry: result.telemetry };
+      }
+      warnProjectionFallback(baseDir, "getMemoryTimeline", projectionAge, result.telemetry ?? undefined);
+      if (result.memoryId === memoryId && result.limit === cappedLimit) return result.events;
+      return readBoundedLifecycleEventsFromLedger(ledgerPath, readSecureFile, cappedLimit, memoryId);
+    }
+
+    const tracker = openProjectionLifecycleLedgerLagTracker(baseDir);
+    if (tracker) {
+      const highWater = tracker.highWater;
+      const scan = (async (): Promise<ProjectionLedgerLagScan> => {
+        try {
+          const fallback = await readBoundedLifecycleEventsWithProjectionLag(
+            ledgerPath,
+            readSecureFile,
+            cappedLimit,
+            memoryId,
+            tracker,
+          );
+          return {
+            telemetry: fallback.telemetryAvailable
+              ? {
+                  projectedEvents: highWater.eventCount,
+                  currentLedgerEvents: fallback.currentEventCount,
+                  deltaEvents: fallback.deltaEvents,
+                }
+              : null,
+            events: fallback.events,
+            memoryId,
+            limit: cappedLimit,
+          };
+        } finally {
+          try {
+            tracker.close();
+          } catch {
+            // Lag telemetry cleanup must not replace a successful ledger fallback.
+          }
+        }
+      })();
+      this.lagInFlight = { generation, scan };
+      try {
+        const result = await scan;
+        if (result.telemetry) {
+          this.lagCache = { generation, telemetry: result.telemetry };
+        }
+        warnProjectionFallback(baseDir, "getMemoryTimeline", projectionAge, result.telemetry ?? undefined);
+        return result.events;
+      } finally {
+        if (this.lagInFlight?.scan === scan) {
+          this.lagInFlight = null;
+        }
+      }
+    }
+
+    warnProjectionFallback(baseDir, "getMemoryTimeline", projectionAge);
+    return readBoundedLifecycleEventsFromLedger(ledgerPath, readSecureFile, cappedLimit, memoryId);
+  }
+}
+
 /**
  * Directory holding the durable pending-append spill for a lifecycle ledger
  * (#2033). When an append cannot acquire the ledger lock within its budget —

package/src/storage/secure-line-reader.ts

@@ -227,6 +227,7 @@ export async function readMemoryLifecycleEventsFromLines(
   limit: number,
   memoryId?: string,
   compare?: (a: MemoryLifecycleEvent, b: MemoryLifecycleEvent) => number,
+  onEvent?: (event: MemoryLifecycleEvent) => void,
 ): Promise<MemoryLifecycleEvent[]> {
   if (limit <= 0) return [];
   const top = compare ? new BoundedLifecycleTopN(limit, compare) : null;
@@ -238,8 +239,9 @@ export async function readMemoryLifecycleEventsFromLines(
     if (!line) continue;
     try {
       const parsed = JSON.parse(line) as Partial<MemoryLifecycleEvent>;
-      if (memoryId !== undefined && parsed.memoryId !== memoryId) continue;
       if (!isValidLifecycleEventRow(parsed)) continue;
+      onEvent?.(parsed);
+      if (memoryId !== undefined && parsed.memoryId !== memoryId) continue;
       if (top) {
         top.add(parsed, seq++);
       } else if (ring.length < limit) {

package/src/storage-guards.ts

@@ -1,6 +1,7 @@
 import { log } from "./logger.js";
 
 const PROJECTION_FALLBACK_WARN_INTERVAL_MS = 5 * 60_000;
+export const PROJECTION_LEDGER_LAG_WARN_THRESHOLD_EVENTS = 100;
 const projectionFallbackWarnedAt = new Map<string, number>();
 
 export function assertMemoryFrontmatterId(
@@ -18,23 +19,43 @@ export function assertMemoryFrontmatterId(
   }
 }
 
+export interface ProjectionLedgerLagTelemetry {
+  projectedEvents: number;
+  currentLedgerEvents: number;
+  deltaEvents: number;
+  warnThresholdEvents?: number;
+}
+
 export function warnProjectionFallback(
   memoryDir: string,
   consumer: string,
-  // Lazy so the (potentially I/O-bound) lag computation runs only when this
-  // call actually logs, not on every rate-limited-suppressed fallback (#2119).
   detail?: () => string | undefined,
+  ledgerLag?: ProjectionLedgerLagTelemetry,
 ): null {
+  const fallback = `storage.${consumer}: memory projection absent or empty; falling back to full corpus`;
+  const thresholdEvents = Math.max(
+    0,
+    Math.floor(ledgerLag?.warnThresholdEvents ?? PROJECTION_LEDGER_LAG_WARN_THRESHOLD_EVENTS),
+  );
+  const lag = ledgerLag
+    ? `projected_events=${ledgerLag.projectedEvents} current_ledger_events=${ledgerLag.currentLedgerEvents} `
+      + `delta_events=${ledgerLag.deltaEvents} threshold_events=${thresholdEvents} `
+      + "fallback_action=full-ledger-scan"
+    : undefined;
+
+  if (ledgerLag && ledgerLag.deltaEvents <= thresholdEvents) {
+    const suffix = [detail?.(), lag].filter(Boolean).join("; ");
+    log.debug(`${fallback}${suffix ? ` (${suffix})` : ""}`);
+    return null;
+  }
+
   const key = `${memoryDir}\0${consumer}`;
   const now = Date.now();
   const warnedAt = projectionFallbackWarnedAt.get(key) ?? 0;
   if (now - warnedAt >= PROJECTION_FALLBACK_WARN_INTERVAL_MS) {
     projectionFallbackWarnedAt.set(key, now);
-    const suffix = detail?.();
-    log.warn(
-      `storage.${consumer}: memory projection absent or empty; falling back to full corpus`
-      + (suffix ? ` (${suffix})` : ""),
-    );
+    const suffix = [detail?.(), lag].filter(Boolean).join("; ");
+    log.warn(`${fallback}${suffix ? ` (${suffix})` : ""}`);
   }
   return null;
 }