@remnic/core 9.23.4 → 9.24.0

package/src/access-service-health.test.ts

@@ -449,3 +449,215 @@ test("health keeps namespace QMD failures scoped to the namespace", async () =>
     await rm(rootDir, { recursive: true, force: true });
   }
 });
+
+test("health surfaces extraction liveness as degraded when the buffer is non-empty and the watermark is stale", async () => {
+  const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-health-liveness-"));
+  try {
+    const oldTs = new Date(Date.now() - 10_000).toISOString();
+    const config = parseConfig({ memoryDir, qmdEnabled: false, extractionLiveness: { staleWindowMs: 1000 } });
+    const service = new EngramAccessService({
+      config,
+      qmd: makeQmd({}),
+      buffer: {
+        getBufferSnapshot: async () => ({ bufferedSessionCount: 3, pendingTurnCount: 12, oldestTurnTimestamp: oldTs }),
+      },
+      // The extraction watermark is read from the daemon-global root store,
+      // not the per-namespace store (issue #2151).
+      storage: {
+        loadMeta: async () => ({
+          lastExtractionAt: oldTs,
+          extractionCount: 4,
+          lastConsolidationAt: null,
+          totalMemories: 0,
+          totalEntities: 0,
+        }),
+      },
+      async getStorage() {
+        return { dir: memoryDir };
+      },
+    } as unknown as Orchestrator);
+
+    const health = await service.health();
+    assert.equal(health.extraction.degraded, true);
+    assert.equal(health.extraction.bufferedSessionCount, 3);
+    assert.equal(health.extraction.pendingTurnCount, 12);
+    assert.equal(health.extraction.lastExtractionAt, oldTs);
+    const oldestAge = health.extraction.oldestBufferedTurnAgeMs ?? -1;
+    assert.ok(
+      oldestAge >= 10_000 && oldestAge < 60_000,
+      `oldest buffered turn age reflects the ~10s offset, not merely non-negative (got ${oldestAge}ms)`,
+    );
+    assert.ok(
+      health.extraction.degradedReason !== null && health.extraction.degradedReason.length > 0,
+      "degraded reason is populated",
+    );
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});
+
+test("health reports extraction liveness ok when the buffer is empty (nothing to extract)", async () => {
+  const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-health-liveness-ok-"));
+  try {
+    const oldTs = new Date(Date.now() - 10_000).toISOString();
+    const config = parseConfig({ memoryDir, qmdEnabled: false, extractionLiveness: { staleWindowMs: 1000 } });
+    const service = new EngramAccessService({
+      config,
+      qmd: makeQmd({}),
+      buffer: {
+        getBufferSnapshot: async () => ({ bufferedSessionCount: 0, pendingTurnCount: 0, oldestTurnTimestamp: null }),
+      },
+      storage: {
+        loadMeta: async () => ({
+          lastExtractionAt: oldTs,
+          extractionCount: 4,
+          lastConsolidationAt: null,
+          totalMemories: 0,
+          totalEntities: 0,
+        }),
+      },
+      async getStorage() {
+        return { dir: memoryDir };
+      },
+    } as unknown as Orchestrator);
+
+    const health = await service.health();
+    assert.equal(health.extraction.degraded, false);
+    assert.equal(health.extraction.bufferedSessionCount, 0);
+    assert.equal(health.extraction.lastExtractionAt, oldTs);
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});
+
+test("health returns the same daemon-global extraction verdict for every namespace (issue #2151)", async () => {
+  const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-health-liveness-ns-"));
+  try {
+    const rootTs = new Date().toISOString(); // fresh daemon-global watermark
+    const staleTs = new Date(Date.now() - 7_200_000).toISOString(); // 2h old
+    const config = parseConfig({
+      memoryDir,
+      qmdEnabled: false,
+      namespacesEnabled: true,
+      defaultNamespace: "default",
+      extractionLiveness: { staleWindowMs: 3_600_000 },
+    });
+    // Per-namespace stores return DIVERGING watermarks (fresh vs 2h stale vs
+    // never), but the global buffer is shared and the watermark must be sourced
+    // from the root store. Pre-fix, the extraction verdict tracked the
+    // namespace argument; post-fix it is identical for every namespace.
+    const perNamespaceMeta: Record<string, string | null> = { fresh: rootTs, stale: staleTs };
+    const service = new EngramAccessService({
+      config,
+      qmd: makeQmd({}),
+      buffer: {
+        getBufferSnapshot: async () => ({ bufferedSessionCount: 3, pendingTurnCount: 12, oldestTurnTimestamp: staleTs }),
+      },
+      storage: {
+        loadMeta: async () => ({
+          lastExtractionAt: rootTs,
+          extractionCount: 4,
+          lastConsolidationAt: null,
+          totalMemories: 0,
+          totalEntities: 0,
+        }),
+      },
+      async getStorage(namespace?: string) {
+        return {
+          dir: memoryDir,
+          loadMeta: async () => ({
+            lastExtractionAt: namespace ? perNamespaceMeta[namespace] ?? null : staleTs,
+            extractionCount: 4,
+            lastConsolidationAt: null,
+            totalMemories: 0,
+            totalEntities: 0,
+          }),
+        };
+      },
+    } as unknown as Orchestrator);
+
+    const fresh = (await service.health("fresh")).extraction;
+    const stale = (await service.health("stale")).extraction;
+    const def = (await service.health()).extraction;
+    // The namespace-governed fields must not vary with the namespace argument
+    // (oldestBufferedTurnAgeMs is time-derived and intentionally excluded).
+    assert.equal(stale.degraded, fresh.degraded, "degraded must not vary by namespace");
+    assert.equal(def.degraded, fresh.degraded, "default health matches namespaced health");
+    assert.equal(stale.lastExtractionAt, fresh.lastExtractionAt, "watermark must not vary by namespace");
+    assert.equal(def.lastExtractionAt, fresh.lastExtractionAt);
+    assert.equal(stale.degradedReason, fresh.degradedReason);
+    // Sourced from the fresh root watermark → healthy despite stale per-namespace stores.
+    assert.equal(fresh.degraded, false);
+    assert.equal(fresh.lastExtractionAt, rootTs);
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});
+
+test("health reports extraction liveness degraded when the buffer read fails (§22)", async () => {
+  const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-health-liveness-readfail-"));
+  try {
+    const freshTs = new Date().toISOString();
+    const config = parseConfig({ memoryDir, qmdEnabled: false, extractionLiveness: { staleWindowMs: 3_600_000 } });
+    const service = new EngramAccessService({
+      config,
+      qmd: makeQmd({}),
+      buffer: {
+        getBufferSnapshot: async () => {
+          throw new Error("buffer file corrupt");
+        },
+      },
+      // Fresh watermark → the ONLY fault is the unreadable buffer, proving the
+      // read failure (not staleness) drives the degradation.
+      storage: {
+        loadMeta: async () => ({
+          lastExtractionAt: freshTs,
+          extractionCount: 4,
+          lastConsolidationAt: null,
+          totalMemories: 0,
+          totalEntities: 0,
+        }),
+      },
+      async getStorage() {
+        return { dir: memoryDir };
+      },
+    } as unknown as Orchestrator);
+
+    const health = await service.health();
+    assert.equal(health.extraction.degraded, true, "an unreadable buffer must not report a healthy pipeline");
+    assert.match(health.extraction.degradedReason ?? "", /unreadable/);
+    assert.match(health.extraction.degradedReason ?? "", /buffer file corrupt/);
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});
+
+test("health reports extraction liveness degraded when the watermark (meta) read fails (§22)", async () => {
+  const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-health-liveness-metafail-"));
+  try {
+    const config = parseConfig({ memoryDir, qmdEnabled: false, extractionLiveness: { staleWindowMs: 3_600_000 } });
+    const service = new EngramAccessService({
+      config,
+      qmd: makeQmd({}),
+      // Empty buffer: proves the meta-read failure alone drives the degradation.
+      buffer: {
+        getBufferSnapshot: async () => ({ bufferedSessionCount: 0, pendingTurnCount: 0, oldestTurnTimestamp: null }),
+      },
+      storage: {
+        loadMeta: async () => {
+          throw new Error("meta.json unreadable");
+        },
+      },
+      async getStorage() {
+        return { dir: memoryDir };
+      },
+    } as unknown as Orchestrator);
+
+    const health = await service.health();
+    assert.equal(health.extraction.degraded, true, "an unreadable watermark must not report a healthy pipeline");
+    assert.match(health.extraction.degradedReason ?? "", /watermark unreadable/);
+    assert.match(health.extraction.degradedReason ?? "", /meta\.json unreadable/);
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});

package/src/access-service.ts

@@ -6,6 +6,7 @@ import { readdirSync, unlinkSync } from "node:fs";
 import { createHash } from "node:crypto";
 import { ZodError } from "zod";
 import { AccessIdempotencyStore, hashAccessIdempotencyPayload } from "./access-idempotency.js";
+import { computeExtractionLivenessStatus, ExtractionLivenessWarnThrottle } from "./extraction-liveness.js";
 import { enforceNamespaceAllowList, tokenCapabilityStore } from "./access-token-capabilities.js";
 import {
   recordCitationUsage as recordCitationUsageForAccess,
@@ -18,7 +19,13 @@ import { resolveNamespaceCapabilities,
   resolveMemoryLifecycleCapabilities,
   resolveQmdCapabilities,
   resolveSecurityCapabilities, resolveObjectiveStateCapabilities, resolveCompressionCapabilities, resolveRecallAuxiliaryCapabilities } from "./capabilities.js";
-import { CorpusWatermarkCache, computeServiceCorpusWatermarks, type CorpusWatermark } from "./corpus-watermark.js";
+import { CorpusWatermarkCache, computeServiceCorpusWatermarks } from "./corpus-watermark.js";
+import type {
+  EngramAccessHealthResponse,
+  EngramAccessQmdCollectionState,
+  EngramAccessQmdHealthResponse,
+} from "./access-health-types.js";
+export type { EngramAccessHealthResponse, EngramAccessQmdCollectionState, EngramAccessQmdHealthResponse };
 import { AccessAuditAdapter, type AccessAuditConfig, type AccessAuditResult } from "./access-audit.js";
 import type { AnomalyDetectorResult } from "./recall-audit-anomaly.js";
 import { resolveGitContext } from "./coding/git-context.js";
@@ -342,44 +349,6 @@ function normalizeTrustZoneInputError(error: unknown): EngramAccessInputError |
 
 export const ENGRAM_ACCESS_WRITE_SCHEMA_VERSION = 1;
 
-export interface EngramAccessHealthResponse {
-  ok: true;
-  memoryDir: string;
-  namespacesEnabled: boolean;
-  defaultNamespace: string;
-  searchBackend: string;
-  qmdEnabled: boolean;
-  qmd: EngramAccessQmdHealthResponse;
-  nativeKnowledgeEnabled: boolean;
-  projectionAvailable: boolean;
-  corpus: CorpusWatermark[];
-}
-
-export type EngramAccessQmdCollectionState =
-  | "present"
-  | "missing"
-  | "unknown"
-  | "skipped";
-
-export interface EngramAccessQmdHealthResponse {
-  enabled: boolean;
-  active: boolean;
-  degraded: boolean;
-  mode: "cli" | "daemon" | "fallback" | "disabled" | "not-selected";
-  collection: string;
-  collectionState: EngramAccessQmdCollectionState;
-  installedVersion: string | null;
-  supportedVersion: string | null;
-  supported: boolean | null;
-  upgradeAvailable: boolean | null;
-  doctorAvailable: boolean | null;
-  debugStatus: string;
-  pendingEmbeddings: number | null;
-  oldestPendingAgeMs: number | null;
-  embeddingBacklogThreshold: number;
-  degradedReason?: string;
-}
-
 export interface EngramAccessRecallRequest {
   query: string;
   sessionKey?: string;
@@ -1365,6 +1334,7 @@ export class EngramAccessService {
     }
     return this._accessIdentityContinuitySurface;
   }
+  private readonly extractionLivenessWarn = new ExtractionLivenessWarnThrottle();
 
   constructor(private readonly orchestrator: Orchestrator) {
     this.idempotency = new AccessIdempotencyStore(orchestrator.config.memoryDir);
@@ -2458,6 +2428,7 @@ export class EngramAccessService {
     const storage = await this.orchestrator.getStorage(resolvedNamespace);
     const searchBackend = this.orchestrator.config.searchBackend ?? "qmd";
     const qmdEnabled = resolveQmdCapabilities(this.orchestrator.config).qmd === true;
+    const extraction = await computeExtractionLivenessStatus(this.orchestrator, this.extractionLivenessWarn);
     let projectionAvailable = false;
     try {
       await stat(getMemoryProjectionPath(storage.dir));
@@ -2481,6 +2452,7 @@ export class EngramAccessService {
       ),
       nativeKnowledgeEnabled: this.orchestrator.config.nativeKnowledge?.enabled === true,
       projectionAvailable,
+      extraction,
       corpus: await computeServiceCorpusWatermarks(this.orchestrator, {
         cache: this.corpusWatermarkCache, caps: tokenCapabilityStore.getStore() }),
     };

package/src/buffer.ts

@@ -10,6 +10,7 @@ import type {
   SignalLevel,
 } from "./types.js";
 import { resolvePresentationCapabilities } from "./capabilities.js";
+import type { ExtractionBufferSnapshot } from "./extraction-liveness.js";
 
 export type TriggerDecision = "extract_now" | "extract_batch" | "keep_buffering";
 
@@ -306,6 +307,35 @@ export class SmartBuffer {
     await this.enqueueMutation(async () => this.loadUnlocked());
   }
 
+  /** Snapshot of buffered, not-yet-extracted work across sessions (#2151): session count,
+   * pending-turn total, oldest buffered turn. Counts retained (#562) + active turns per entry. */
+  async getBufferSnapshot(): Promise<ExtractionBufferSnapshot> {
+    await this.load();
+    let bufferedSessionCount = 0;
+    let pendingTurnCount = 0;
+    let oldestMs = Number.POSITIVE_INFINITY;
+    let oldestTurnTimestamp: string | null = null;
+    const consider = (turns: BufferTurn[] | undefined): void => {
+      if (!Array.isArray(turns) || turns.length === 0) return;
+      bufferedSessionCount += 1;
+      pendingTurnCount += turns.length;
+      const first = turns[0]?.timestamp;
+      const parsed = typeof first === "string" ? Date.parse(first) : Number.NaN;
+      if (Number.isFinite(parsed) && parsed < oldestMs) {
+        oldestMs = parsed;
+        oldestTurnTimestamp = first ?? null;
+      }
+    };
+    const entries = this.state.entries;
+    if (entries) {
+      // Retained turns (deferred #562) are pending too - count with active (oldest-first), like getTurns().
+      for (const entry of Object.values(entries)) consider([...(entry?.retainedTurns ?? []), ...(entry?.turns ?? [])]);
+    } else {
+      consider(this.state.turns);
+    }
+    return { bufferedSessionCount, pendingTurnCount, oldestTurnTimestamp };
+  }
+
   /**
    * Reset the buffer to an empty, usable state.
    * Called when the persisted buffer file is corrupt and load() fails,

package/src/cli.ts

@@ -30,6 +30,7 @@ import { isRecallDisclosure, RECALL_DISCLOSURE_LEVELS } from "./types.js";
 import { chunkContent } from "./chunking.js";
 import { rescoreMemoryImportance } from "./importance.js";
 import { renderQmdBacklogStatus } from "./qmd-status.js";
+import { renderExtractionLivenessStats } from "./extraction-liveness.js";
 import { exportJsonBundle } from "./transfer/export-json.js";
 import { exportMarkdownBundle } from "./transfer/export-md.js";
 import { backupMemoryDir } from "./transfer/backup.js";
@@ -3653,7 +3654,6 @@ export function registerCli(
           // Ensure QMD is probed before checking availability
           await orchestrator.qmd.probe();
 
-          const meta = await orchestrator.storage.loadMeta();
           const memories = await orchestrator.storage.readAllMemories();
           const entities = await orchestrator.storage.readEntities();
           const profile = await orchestrator.storage.readProfile();
@@ -3662,9 +3662,7 @@ export function registerCli(
           console.log(`Total memories: ${memories.length}`);
           console.log(`Total entities: ${entities.length}`);
           console.log(`Profile size: ${profile.length} chars`);
-          console.log(`Extractions: ${meta.extractionCount}`);
-          console.log(`Last extraction: ${meta.lastExtractionAt ?? "never"}`);
-          console.log(`Last consolidation: ${meta.lastConsolidationAt ?? "never"}`);
+          for (const line of await renderExtractionLivenessStats(orchestrator)) console.log(line);
           console.log(`QMD: ${orchestrator.qmd.isAvailable() ? "available" : "not available"}`);
           for (const line of await renderQmdBacklogStatus(orchestrator.qmd, orchestrator.config.qmdEmbeddingBacklogThreshold)) console.log(line);
 

package/src/config.ts

@@ -37,6 +37,7 @@ import { expandTildePath } from "./utils/path.js";
 // config.ts → connectors/index.ts nor the reverse circular import arises.
 import { coerceBool, coerceBooleanLike, coerceInstallExtension, coerceNumber } from "./connectors/coerce.js";
 import { parseRecallConcurrencyConfig } from "./recall-concurrency-config.js";
+import { parseExtractionLivenessConfig } from "./extraction-liveness.js";
 import { hasLegacyConnectorEntries } from "./connectors/paths.js";
 import {
   parseQmdChunkStrategy,
@@ -2326,15 +2327,14 @@ export function parseConfig(
       typeof cfg.activeRecallEntityGraphDepth === "number"
         ? Math.min(3, Math.max(0, Math.floor(cfg.activeRecallEntityGraphDepth)))
         : 1,
-    activeRecallIncludeCausalTrajectories:
-      cfg.activeRecallIncludeCausalTrajectories === true,
+    activeRecallIncludeCausalTrajectories: cfg.activeRecallIncludeCausalTrajectories === true,
     activeRecallIncludeDaySummary: cfg.activeRecallIncludeDaySummary === true,
     activeRecallAttachRecallExplain: cfg.activeRecallAttachRecallExplain === true,
-    activeRecallAllowChainedActiveMemory:
-      cfg.activeRecallAllowChainedActiveMemory === true,
+    activeRecallAllowChainedActiveMemory: cfg.activeRecallAllowChainedActiveMemory === true,
     dreaming,
     dreamsPhases,
     procedural,
+    extractionLiveness: parseExtractionLivenessConfig(cfg),
     wearables,
     meetings,
     activity,