@remnic/core 10.0.69 → 10.0.70

package/src/access-observe-stage-timing.test.ts

@@ -0,0 +1,246 @@
+/**
+ * Noncontent stage timing for slow/failed observe requests (25s incident).
+ * A failed request must record which stage blocked it (phase attribution) in
+ * the `access-observe-failed` log line without leaking message content,
+ * session keys, namespaces or principals, and must still throw unchanged.
+ */
+import assert from "node:assert/strict";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import test from "node:test";
+
+import { EngramAccessService, NamespaceNotWritableError } from "./access-service.js";
+import { initLogger, resetLogger, type LoggerBackend } from "./logger.js";
+import { Orchestrator } from "./orchestrator.js";
+import { parseConfig } from "./config.js";
+import type { EngramAccessObserveRequest } from "./access-service.js";
+import type { CodingContext, PluginConfig } from "./types.js";
+
+interface OrchestratorInternals {
+  config: PluginConfig;
+  _codingContextBySession: Map<string, CodingContext>;
+}
+
+function makeService(memoryDir: string): EngramAccessService {
+  // SAFETY: the probe drives only the observe scope path on this orchestrator.
+  const orch = Object.create(Orchestrator.prototype) as Orchestrator;
+  // SAFETY: the probe sets exactly these two internal fields.
+  const internals: OrchestratorInternals = {
+    config: parseConfig({
+      namespacesEnabled: true,
+      defaultNamespace: "default",
+      sharedNamespace: "shared",
+      namespacePolicies: [],
+      codingMode: { projectScope: true },
+      transcriptEnabled: false,
+      memoryDir,
+    }),
+    _codingContextBySession: new Map(),
+  };
+  Object.assign(orch, internals);
+  return new EngramAccessService(orch);
+}
+
+test("a rejected observe logs phase attribution with stage durations and still throws", async () => {
+  const dir = await mkdtemp(path.join(tmpdir(), "remnic-observe-timing-"));
+  const lines: string[] = [];
+  const backend: LoggerBackend = {
+    info(msg) {
+      lines.push(msg);
+    },
+    warn(msg) {
+      lines.push(msg);
+    },
+    error(msg) {
+      lines.push(msg);
+    },
+  };
+  initLogger(backend, false, { timestamps: false });
+  try {
+    const service = makeService(dir);
+    // SAFETY: the literal sets the request fields the scope path reads.
+    const rejectedRequest = {
+      sessionKey: "sess-timing-1",
+      authenticatedPrincipal: "alice",
+      namespace: "victim-secret",
+      messages: [
+        { role: "user", content: "first secret message body" },
+        { role: "assistant", content: "second secret message body" },
+      ],
+    } as EngramAccessObserveRequest;
+    await assert.rejects(
+      service.observe(rejectedRequest),
+      NamespaceNotWritableError
+    );
+
+    const failed = lines.filter((l) => l.includes("access-observe-failed"));
+    assert.equal(failed.length, 1, `expected exactly one access-observe-failed line, got: ${failed}`);
+    const line = failed[0] ?? "";
+    // Phase attribution: the blocked stage (scope) plus total and message count.
+    assert.match(line, /totalMs=\d+/);
+    assert.match(line, /messages=2/);
+    assert.match(line, /scopeMs=\d+/);
+    // Noncontent: no message bodies, session key, namespace or principal.
+    assert.ok(!line.includes("secret"), `leaked content in: ${line}`);
+    assert.ok(!line.includes("sess-timing-1"), `leaked session key in: ${line}`);
+    assert.ok(!line.includes("victim-secret"), `leaked namespace in: ${line}`);
+    assert.ok(!line.includes("alice"), `leaked principal in: ${line}`);
+  } finally {
+    resetLogger();
+    await rm(dir, { recursive: true, force: true });
+  }
+});
+
+// Plain-object orchestrator for full observe success/failure paths (mirrors
+// the access-service-observe-lcm-parity harness).
+function makeProbe(options: {
+  enqueueObserveMessages: (sessionKey: string, messages: unknown[]) => Promise<void> | void;
+}) {
+  const config = parseConfig({
+    namespacesEnabled: true,
+    defaultNamespace: "default",
+    sharedNamespace: "shared",
+    namespacePolicies: [],
+    defaultRecallNamespaces: ["self", "shared"],
+    codingMode: { projectScope: true },
+    transcriptEnabled: false,
+    memoryDir: "/synthetic/remnic-observe-timing",
+    objectiveStateMemoryEnabled: false,
+    objectiveStateSnapshotWritesEnabled: false,
+    principalFromSessionKeyMode: "prefix",
+    principalFromSessionKeyRules: [],
+    recallCrossNamespaceBudgetEnabled: false,
+    recallCrossNamespaceBudgetWindowMs: 60_000,
+    recallCrossNamespaceBudgetSoftLimit: 10,
+    recallCrossNamespaceBudgetHardLimit: 30,
+  });
+  const contexts = new Map<string, CodingContext>();
+  const extractionCalls: string[][] = [];
+  // SAFETY: the probe exercises the observe surface; the rest of Orchestrator
+  // is unused on this path.
+  const orch = {
+    config,
+    getCodingContextForSession: (sk: string | undefined) => (sk ? contexts.get(sk) : null) ?? null,
+    setCodingContextForSession: (sk: string, ctx: CodingContext | null) => {
+      if (ctx === null) contexts.delete(sk);
+      else contexts.set(sk, ctx);
+    },
+    applyCodingNamespaceOverlay: (sk: string | undefined, base: string) =>
+      Orchestrator.prototype.applyCodingNamespaceOverlay.call(orch, sk, base),
+    resolvePrincipal: (sk?: string) => Orchestrator.prototype.resolvePrincipal.call(orch, sk),
+    resolveSelfNamespace: (sk?: string) => Orchestrator.prototype.resolveSelfNamespace.call(orch, sk),
+    lcmEngine: {
+      enabled: true,
+      enqueueObserveMessages: options.enqueueObserveMessages,
+    },
+    ingestReplayBatch: async (turns: Array<{ sessionKey: string }>) => {
+      extractionCalls.push(turns.map((t) => t.sessionKey));
+    },
+  } as any;
+  return { orch, extractionCalls };
+}
+
+test("a slow successful observe logs access-observe-slow with per-stage durations", async () => {
+  const lines: string[] = [];
+  const backend: LoggerBackend = {
+    info(msg) {
+      lines.push(msg);
+    },
+    warn(msg) {
+      lines.push(msg);
+    },
+    error(msg) {
+      lines.push(msg);
+    },
+  };
+  initLogger(backend, false, { timestamps: false });
+  // Deterministic monotonic clock: every read advances 6s, so each timed
+  // stage records exactly 6000ms and the total exceeds the slow threshold.
+  let now = 0;
+  const realPerformanceNow = performance.now;
+  performance.now = () => (now += 6000);
+  try {
+    const { orch } = makeProbe({
+      enqueueObserveMessages: (_sessionKey) => {},
+    });
+    const service = new EngramAccessService(orch);
+    // SAFETY: the literal sets the request fields the observe path reads.
+    const slowRequest = {
+      sessionKey: "alice:sess-slow",
+      authenticatedPrincipal: "alice",
+      messages: [
+        { role: "user", content: "slow path message body" },
+        { role: "assistant", content: "reply body" },
+      ],
+    } as EngramAccessObserveRequest;
+    const res = await service.observe(slowRequest);
+    assert.equal(res.accepted, 2);
+
+    const slow = lines.filter((l) => l.includes("access-observe-slow"));
+    assert.equal(slow.length, 1, `expected one access-observe-slow line, got: ${lines}`);
+    const line = slow[0] ?? "";
+    assert.match(line, /totalMs=(\d+)/);
+    const totalMs = Number(/totalMs=(\d+)/.exec(line)?.[1]);
+    assert.ok(totalMs >= 10_000, `slow line must exceed threshold, got totalMs=${totalMs}`);
+    assert.match(line, /messages=2/);
+    // Real stages are attributed, including the durable-buffer terminal stage.
+    assert.match(line, /scopeMs=6000/);
+    assert.match(line, /durable-bufferMs=6000/);
+    assert.ok(!line.includes("sess-slow") && !line.includes("alice") && !line.includes("body"));
+  } finally {
+    performance.now = realPerformanceNow;
+    resetLogger();
+  }
+});
+
+test("a failed awaited LCM archival records the lcm-accept phase and still throws", async () => {
+  const lines: string[] = [];
+  const backend: LoggerBackend = {
+    info(msg) {
+      lines.push(msg);
+    },
+    warn(msg) {
+      lines.push(msg);
+    },
+    error(msg) {
+      lines.push(msg);
+    },
+  };
+  initLogger(backend, false, { timestamps: false });
+  try {
+    const { orch } = makeProbe({
+      enqueueObserveMessages: async () => {
+        throw new Error("lcm archival append failed");
+      },
+    });
+    const service = new EngramAccessService(orch);
+    // SAFETY: the literal sets the request fields the observe path reads.
+    const failedLcmRequest = {
+      sessionKey: "alice:sess-lcmfail",
+      authenticatedPrincipal: "alice",
+      messages: [
+        { role: "user", content: "lcm failure message body" },
+        { role: "assistant", content: "reply body" },
+      ],
+    } as EngramAccessObserveRequest;
+    await assert.rejects(
+      service.observe(failedLcmRequest),
+      /lcm archival append failed/
+    );
+
+    const failed = lines.filter((l) => l.includes("access-observe-failed"));
+    assert.equal(failed.length, 1, `expected exactly one access-observe-failed line, got: ${failed}`);
+    const line = failed[0] ?? "";
+    assert.match(line, /totalMs=\d+/);
+    assert.match(line, /messages=2/);
+    // The rejected LCM accept is attributed with its duration, alongside earlier stages.
+    assert.match(line, /scopeMs=\d+/);
+    assert.match(line, /lcm-acceptMs=\d+/);
+    assert.ok(!line.includes("sess-lcmfail"), `leaked session key in: ${line}`);
+    assert.ok(!line.includes("alice"), `leaked principal in: ${line}`);
+    assert.ok(!line.includes("body"), `leaked content in: ${line}`);
+  } finally {
+    resetLogger();
+  }
+});

package/src/access-observe-stage-timing.ts

@@ -0,0 +1,58 @@
+/**
+ * Noncontent stage timing for the observe surface.
+ *
+ * Durations only — never message content, session keys, namespaces or
+ * principals. A monotonic clock (`performance.now`) keeps phase durations
+ * sane under wall-clock corrections.
+ *
+ * A slow observe logs `access-observe-slow`; a failed observe always logs
+ * `access-observe-failed`, so a 25s-class stall attributes to a phase
+ * instead of a guess.
+ */
+import { log } from "./logger.js";
+
+/** Observes slower than this log `access-observe-slow` with per-stage durations. */
+const OBSERVE_SLOW_LOG_MS = 10_000;
+
+export interface ObserveStageTimer {
+  /** Run one stage, recording its duration even when the stage rejects. */
+  timed<T>(stage: string, fn: () => Promise<T>): Promise<T>;
+  /** Log a slow successful observe with per-stage durations. */
+  logSlow(messages: number): void;
+  /** Log a failed observe with per-stage durations. */
+  logFailed(messages: number): void;
+}
+
+/** Timer for one observe request. */
+export function createObserveStageTimer(): ObserveStageTimer {
+  const startedAtMs = performance.now();
+  const stageMs: Record<string, number> = {};
+
+  const formatStages = (): string =>
+    Object.entries(stageMs)
+      .map(([stage, ms]) => `${stage}Ms=${Math.round(ms)}`)
+      .join(" ");
+
+  return {
+    async timed<T>(stage: string, fn: () => Promise<T>): Promise<T> {
+      const stageStartMs = performance.now();
+      try {
+        return await fn();
+      } finally {
+        stageMs[stage] = (stageMs[stage] ?? 0) + (performance.now() - stageStartMs);
+      }
+    },
+    logSlow(messages: number): void {
+      const totalMs = Math.round(performance.now() - startedAtMs);
+      if (totalMs < OBSERVE_SLOW_LOG_MS) return;
+      log.warn(
+        `access-observe-slow totalMs=${totalMs} messages=${messages} ${formatStages()}`,
+      );
+    },
+    logFailed(messages: number): void {
+      log.warn(
+        `access-observe-failed totalMs=${Math.round(performance.now() - startedAtMs)} messages=${messages} ${formatStages()}`,
+      );
+    },
+  };
+}

package/src/access-observe-write-surface.ts

@@ -34,6 +34,9 @@ import { abortError } from "./abort-error.js";
 import { withBriefingDeadline } from "./briefing-deadline.js";
 import { extractionForceFlush } from "./access-extraction-force-flush.js";
 import { ObserveTranscriptPersister, observeTranscriptSessionKey } from "./access-observe-transcript.js";
+import { createObserveStageTimer } from "./access-observe-stage-timing.js";
+import { archiveObservedTurns } from "./access-observe-lcm-archive.js";
+import { recordObserveObjectiveState } from "./access-observe-objective-state.js";
 import { FileCalendarSource, buildBriefing, parseBriefingFocus, parseBriefingWindow } from "./briefing.js";
 import {
   resolveCompressionCapabilities,
@@ -708,10 +711,16 @@ export class AccessObserveWriteSurface {
       request.sessionKey,
       pendingObserveScopeHint(request)
     );
+    // Noncontent stage timing for slow/failed observes: durations only, logged
+    // in the `access-observe-failed` line (always) and `access-observe-slow`
+    // (successes at or above the slow threshold). Never message content,
+    // session keys, namespaces or principals.
+    const stageTimer = createObserveStageTimer();
+    const timed = stageTimer.timed.bind(stageTimer);
     try {
       let scope: MemoryScopePlan;
       try {
-        scope = await this.deps.resolveMemoryScopePlan(request);
+        scope = await timed("scope", () => this.deps.resolveMemoryScopePlan(request));
       } catch (err) {
         // A replay re-submit sets suppressQuarantine so a still-unwritable target
         // propagates instead of re-parking (#1888); observe has no dryRun path.
@@ -749,10 +758,12 @@ export class AccessObserveWriteSurface {
       //    on the same session is project-scoped (rule 42: same namespace layer as
       //    recall). Done AFTER the scope plan authorized the write, so a rejected
       //    request never leaves orphaned context on the session.
-      await this.deps.maybeAttachCodingContext(request.sessionKey, {
-        cwd: request.cwd,
-        projectTag: request.projectTag,
-      });
+      await timed("context", () =>
+        this.deps.maybeAttachCodingContext(request.sessionKey, {
+          cwd: request.cwd,
+          projectTag: request.projectTag,
+        })
+      );
 
       // Prefix sessionKey with the EFFECTIVE write namespace for LCM archival so
       // observed turns are scoped to the same namespace project-scoped recall
@@ -771,51 +782,35 @@ export class AccessObserveWriteSurface {
       //    (#928 contract, already auth-checked inside the scope plan), not the
       //    general default-store write namespace.
       if (shouldWriteObjectiveState) {
-        try {
-          const objectiveStateLocation = await this.deps.objectiveStateStoreLocationForNamespace(
-            scope.objectiveStateNamespace
-          );
-          await recordObjectiveStateSnapshotsFromObservedMessages({
-            memoryDir: objectiveStateLocation.memoryDir,
-            objectiveStateStoreDir: objectiveStateLocation.objectiveStateStoreDir,
-            objectiveStateMemoryEnabled: resolveObjectiveStateCapabilities(this.deps.orchestrator.config)
-              .objectiveStateMemory,
-            objectiveStateSnapshotWritesEnabled: resolveObjectiveStateCapabilities(this.deps.orchestrator.config)
-              .objectiveStateSnapshotWrites,
-            sessionKey: request.sessionKey,
-            recordedAt: new Date().toISOString(),
-            messages: request.messages,
-          });
-        } catch (err) {
-          log.error(`access-observe objective-state snapshot write failed: ${err}`);
-        }
+        await timed("objective", () =>
+          recordObserveObjectiveState(
+            this.deps.orchestrator,
+            this.deps.objectiveStateStoreLocationForNamespace,
+            scope,
+            request,
+          ),
+        );
       }
 
       // 5. LCM archival → effective write namespace.
-      // lcmArchived in the response means "LCM archival was queued" (not
-      // "completed"), matching extractionQueued semantics.  Both run async.
-      let lcmArchived = false;
-      if (this.deps.orchestrator.lcmEngine && this.deps.orchestrator.lcmEngine.enabled) {
-        // Fire-and-forget: LCM archival writes to SQLite and builds summary
-        // DAGs, which can take tens of seconds for large sessions.  Don't
-        // block the HTTP response — the caller only needs acknowledgment.
-        try {
-          this.deps.orchestrator.lcmEngine.enqueueObserveMessages(lcmSessionKey, request.messages);
-          lcmArchived = true;
-        } catch (err) {
-          log.error(`access-observe LCM enqueue failed: ${err}`);
-        }
-      }
+      // The archival append is durable before this returns: the queue holds
+      // only a session reference, and summary DAGs are built in the
+      // background, so the caller is never blocked on inference.
+      const lcmArchived = await timed("lcm-accept", () =>
+        archiveObservedTurns(this.deps.orchestrator.lcmEngine, lcmSessionKey, request.messages),
+      );
 
       // 5b. Transcript persistence → the per-session transcript store the
-      // hourly summarizer reads (issue #2783). Best-effort like the LCM
-      // enqueue above: a transcript failure must not fail the observe.
+      // hourly summarizer reads (issue #2783). Best-effort: a transcript
+      // failure must not fail the observe.
       const transcriptPersisted = resolvePresentationCapabilities(this.deps.orchestrator.config).transcript
-        ? await this.observeTranscriptPersister.persist(
-            this.deps.orchestrator,
-            observeTranscriptSessionKey(request.sessionKey, writeNamespace, this.deps.orchestrator.config),
-            request.messages,
-            request.sessionKey
+        ? await timed("transcript", () =>
+            this.observeTranscriptPersister.persist(
+              this.deps.orchestrator,
+              observeTranscriptSessionKey(request.sessionKey, writeNamespace, this.deps.orchestrator.config),
+              request.messages,
+              request.sessionKey
+            )
           )
         : false;
 
@@ -895,10 +890,11 @@ export class AccessObserveWriteSurface {
             extractionQueued = true;
           };
         }
-        await this.deps.orchestrator.ingestReplayBatch(turns, ingestOptions);
+        await timed("durable-buffer", () => this.deps.orchestrator.ingestReplayBatch(turns, ingestOptions));
         // A native trigger may leave the durable capture pending rather than queued.
       }
 
+      stageTimer.logSlow(request.messages.length);
       log.info(
         `access-observe namespace=${namespace} effectiveNamespace=${writeNamespace} sessionKey=${request.sessionKey} messages=${request.messages.length} lcm=${lcmArchived} extraction=${extractionQueued} transcript=${transcriptPersisted}`
       );
@@ -924,6 +920,10 @@ export class AccessObserveWriteSurface {
         extractionQueued,
         transcriptPersisted,
       };
+    } catch (err) {
+      // Failed requests always get phase attribution (noncontent), regardless of duration.
+      stageTimer.logFailed(request.messages.length);
+      throw err;
     } finally {
       observePreparation.release();
     }

package/src/access-service.ts

@@ -1079,6 +1079,7 @@ export interface EngramAccessObserveResponse {
   effectiveNamespace: string;
   /** Additive diagnostic view of the resolved scope plan (#1495). */
   scopeDebug?: EngramAccessScopeDebug;
+  /** True when the observed turns are durably archived in LCM — the archive append completed before the response was produced. */
   lcmArchived: boolean;
   extractionQueued: boolean;
   /** True when at least one observe-derived turn was appended to the transcript store (issue #2783). False when the transcript capability is off, every message was filtered/deduped, or every append failed. */

package/src/lcm/archive.ts

@@ -188,20 +188,72 @@ export class LcmArchive {
       .all(sessionId, fromTurn, toTurn) as LcmMessage[];
   }
 
-  /** Retrieve unsummarized messages (after last leaf summary). */
-  getUnsummarizedMessages(sessionId: string): LcmMessage[] {
-    const lastLeafEnd = this.db
+  /** Highest turn index covered by a leaf summary (-1 when none exist). */
+  private lastLeafSummaryEnd(sessionId: string): number {
+    // SAFETY: the query returns exactly one aggregate column, `last_end`.
+    const row = this.db
       .prepare(
         "SELECT MAX(msg_end) as last_end FROM lcm_summary_nodes WHERE session_id = ? AND depth = 0",
       )
       .get(sessionId) as { last_end: number | null } | undefined;
+    return row?.last_end ?? -1;
+  }
 
-    const lastSummarized = lastLeafEnd?.last_end ?? -1;
-    return this.db
+  /** Retrieve unsummarized messages (after last leaf summary), optionally bounded. */
+  getUnsummarizedMessages(sessionId: string, limit?: number): LcmMessage[] {
+    const lastSummarized = this.lastLeafSummaryEnd(sessionId);
+    if (limit !== undefined && limit <= 0) return [];
+
+    const stmt = this.db.prepare(
+      limit === undefined
+        ? "SELECT * FROM lcm_messages WHERE session_id = ? AND turn_index > ? ORDER BY turn_index"
+        : "SELECT * FROM lcm_messages WHERE session_id = ? AND turn_index > ? ORDER BY turn_index LIMIT ?",
+    );
+    // SAFETY: the statement selects the lcm_messages row shape.
+    const rows = (
+      limit === undefined ? stmt.all(sessionId, lastSummarized) : stmt.all(sessionId, lastSummarized, limit)
+    ) as LcmMessage[];
+    return rows;
+  }
+
+  /** Count archived turns that no leaf summary covers yet. */
+  countUnsummarizedMessages(sessionId: string): number {
+    const lastSummarized = this.lastLeafSummaryEnd(sessionId);
+    // SAFETY: the query returns exactly one aggregate column, `n`.
+    const row = this.db
+      .prepare(
+        "SELECT COUNT(*) as n FROM lcm_messages WHERE session_id = ? AND turn_index > ?",
+      )
+      .get(sessionId, lastSummarized) as { n: number };
+    return row.n;
+  }
+
+  /**
+   * Sessions whose durable archive still holds at least `minTurns`
+   * unsummarized turns. Metadata only — used to resume outstanding summary
+   * work after a restart.
+   */
+  listSessionsWithUnsummarizedTurns(minTurns: number): string[] {
+    if (minTurns <= 0) return [];
+    // SAFETY: the query returns one `session_id` column per session.
+    const rows = this.db
       .prepare(
-        "SELECT * FROM lcm_messages WHERE session_id = ? AND turn_index > ? ORDER BY turn_index",
+        `
+        SELECT m.session_id AS session_id
+        FROM lcm_messages m
+        LEFT JOIN (
+          SELECT session_id, MAX(msg_end) AS last_end
+          FROM lcm_summary_nodes
+          WHERE depth = 0
+          GROUP BY session_id
+        ) leaves ON leaves.session_id = m.session_id
+        WHERE m.turn_index > COALESCE(leaves.last_end, -1)
+        GROUP BY m.session_id
+        HAVING COUNT(*) >= ?
+      `,
       )
-      .all(sessionId, lastSummarized) as LcmMessage[];
+      .all(minTurns) as Array<{ session_id: string }>;
+    return rows.map((row) => row.session_id);
   }
 
   /** Full-text search across all messages. */