@remnic/core 9.69.16 → 9.69.18

package/src/access-observe-transcript-persistence.test.ts

@@ -0,0 +1,252 @@
+/**
+ * Issue #2783: the daemon's observe path must persist observe-derived turns
+ * into the transcript store, and `memory_summarize_hourly` must report an
+ * empty/stale transcript store distinctly instead of a bare unconditional
+ * ok. Delegate-mode gateways (and any observe-only client) starve the
+ * hourly summarizer otherwise — 11 days of silent empty runs in the field.
+ */
+import assert from "node:assert/strict";
+import { mkdtemp, readdir, readFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import test from "node:test";
+
+import { EngramAccessService } from "./access-service.js";
+import { HourlySummarizer } from "./summarizer.js";
+import { Orchestrator } from "./orchestrator.js";
+import { TranscriptManager } from "./transcript.js";
+import type { CodingContext, PluginConfig } from "./types.js";
+
+interface StubOrchestrator {
+  config: PluginConfig;
+  transcript: TranscriptManager;
+  summarizer: HourlySummarizer;
+  lcmEngine?: undefined;
+  ingestReplayBatch: () => Promise<unknown[]>;
+  _codingContextBySession: Map<string, CodingContext>;
+}
+
+function makeService(
+  memoryDir: string,
+  transcriptEnabled: boolean,
+  transcriptSkipChannelTypes: string[] = []
+): { service: EngramAccessService; orchestrator: StubOrchestrator } {
+  const config = {
+    memoryDir,
+    transcriptEnabled,
+    transcriptSkipChannelTypes,
+    namespacesEnabled: false,
+    defaultNamespace: "default",
+  } as unknown as PluginConfig;
+  const orch = Object.create(Orchestrator.prototype) as Orchestrator;
+  const stub: StubOrchestrator = {
+    config,
+    transcript: new TranscriptManager(config),
+    summarizer: new HourlySummarizer(config),
+    lcmEngine: undefined,
+    ingestReplayBatch: async () => [],
+    _codingContextBySession: new Map<string, CodingContext>(),
+  };
+  Object.assign(orch as unknown as Record<string, unknown>, stub);
+  return { service: new EngramAccessService(orch), orchestrator: stub };
+}
+
+async function withTempDir(fn: (dir: string) => Promise<void>): Promise<void> {
+  const dir = await mkdtemp(path.join(tmpdir(), "remnic-2783-"));
+  try {
+    await fn(dir);
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+}
+
+async function readAllTranscriptLines(dir: string): Promise<Array<Record<string, unknown>>> {
+  const root = path.join(dir, "transcripts");
+  const entries: Array<Record<string, unknown>> = [];
+  const typeDirs = await readdir(root, { withFileTypes: true }).catch(() => []);
+  for (const typeEnt of typeDirs) {
+    if (!typeEnt.isDirectory()) continue;
+    const idDirs = await readdir(path.join(root, typeEnt.name), { withFileTypes: true });
+    for (const idEnt of idDirs) {
+      if (!idEnt.isDirectory()) continue;
+      const files = await readdir(path.join(root, typeEnt.name, idEnt.name));
+      for (const file of files) {
+        if (!file.endsWith(".jsonl")) continue;
+        const raw = await readFile(path.join(root, typeEnt.name, idEnt.name, file), "utf-8");
+        for (const line of raw.split("\n")) {
+          if (line.trim().length === 0) continue;
+          entries.push(JSON.parse(line) as Record<string, unknown>);
+        }
+      }
+    }
+  }
+  return entries;
+}
+
+const USER_TURN = "please summarize the replication topology for the fleet";
+const ASSISTANT_TURN = "the fleet uses one primary daemon with a replicated failover copy";
+
+test("observe persists user and assistant turns into the transcript store (#2783)", async () => {
+  await withTempDir(async (dir) => {
+    const { service } = makeService(dir, true);
+    const response = await service.observe({
+      sessionKey: "agent:delegate-test:session-1",
+      messages: [
+        { role: "user", content: USER_TURN },
+        { role: "assistant", content: ASSISTANT_TURN },
+      ],
+    });
+    assert.equal(response.transcriptPersisted, true);
+    const entries = await readAllTranscriptLines(dir);
+    assert.equal(entries.length, 2);
+    assert.equal(entries[0]?.role, "user");
+    assert.equal(entries[0]?.content, USER_TURN);
+    assert.equal(entries[0]?.sessionKey, "agent:delegate-test:session-1");
+    assert.equal(entries[1]?.role, "assistant");
+    assert.equal(entries[1]?.content, ASSISTANT_TURN);
+    assert.equal(typeof entries[0]?.turnId, "string");
+    assert.equal(typeof entries[0]?.timestamp, "string");
+  });
+});
+
+test("observe filters short messages below the embedded parity noise floor (#2783)", async () => {
+  await withTempDir(async (dir) => {
+    const { service } = makeService(dir, true);
+    const response = await service.observe({
+      sessionKey: "agent:delegate-test:session-2",
+      messages: [
+        { role: "user", content: "ok" },
+        { role: "assistant", content: ASSISTANT_TURN },
+      ],
+    });
+    assert.equal(response.transcriptPersisted, true);
+    const entries = await readAllTranscriptLines(dir);
+    assert.equal(entries.length, 1);
+    assert.equal(entries[0]?.role, "assistant");
+  });
+});
+
+test("a re-POST of the same user turn is deduped; new turns still append (#2783)", async () => {
+  await withTempDir(async (dir) => {
+    const { service } = makeService(dir, true);
+    await service.observe({
+      sessionKey: "agent:delegate-test:session-3",
+      messages: [
+        { role: "user", content: USER_TURN },
+        { role: "assistant", content: ASSISTANT_TURN },
+      ],
+    });
+    // Un-keyed client retry of the SAME turn (no idempotencyKey).
+    await service.observe({
+      sessionKey: "agent:delegate-test:session-3",
+      messages: [
+        { role: "user", content: USER_TURN },
+        { role: "assistant", content: ASSISTANT_TURN },
+      ],
+    });
+    const afterRetry = await readAllTranscriptLines(dir);
+    assert.equal(afterRetry.length, 2);
+
+    // A genuinely new turn still appends.
+    await service.observe({
+      sessionKey: "agent:delegate-test:session-3",
+      messages: [{ role: "user", content: "now check the failover read path end to end please" }],
+    });
+    const afterNew = await readAllTranscriptLines(dir);
+    assert.equal(afterNew.length, 3);
+  });
+});
+
+test("transcript persistence is gated on the transcriptEnabled capability (#2783)", async () => {
+  await withTempDir(async (dir) => {
+    const { service } = makeService(dir, false);
+    const response = await service.observe({
+      sessionKey: "agent:delegate-test:session-4",
+      messages: [{ role: "user", content: USER_TURN }],
+    });
+    assert.equal(response.transcriptPersisted, false);
+    const entries = await readAllTranscriptLines(dir);
+    assert.equal(entries.length, 0);
+  });
+});
+
+test("memory_summarize_hourly warns distinctly when the transcript store is empty (#2783)", async () => {
+  await withTempDir(async (dir) => {
+    const { service } = makeService(dir, true);
+    const result = await service.memorySummarizeHourly();
+    assert.equal(result.ok, true);
+    assert.equal(result.sessionsConsidered, 0);
+    assert.equal(result.warning, "transcript store is empty");
+    assert.match(result.message, /transcript store is empty/);
+  });
+});
+
+test("memory_summarize_hourly warns distinctly when the store is stale and the hour was idle (#2783)", async () => {
+  await withTempDir(async (dir) => {
+    const { service, orchestrator } = makeService(dir, true);
+    // Seed one transcript entry far in the past: the store is non-empty but
+    // stale, and the target hour has no entries.
+    await orchestrator.transcript.append({
+      timestamp: new Date(Date.now() - 6 * 60 * 60 * 1000).toISOString(),
+      role: "user",
+      content: "an old turn from a much earlier hour",
+      sessionKey: "agent:delegate-test:session-5",
+      turnId: "seed-turn",
+    });
+    const result = await service.memorySummarizeHourly();
+    assert.equal(result.ok, true);
+    assert.ok(result.sessionsConsidered > 0);
+    assert.equal(result.sessionsWithEntries, 0);
+    assert.equal(result.staleStore, true);
+    assert.equal(result.warning, "no transcript entries for the target hour and no new entries recently");
+  });
+});
+
+test("observeTranscriptSessionKey scopes transcripts to the effective write namespace (#2783 review)", async () => {
+  const { observeTranscriptSessionKey } = await import("./access-observe-transcript.js");
+  const config = { defaultNamespace: "generalist" } as { defaultNamespace: string };
+  // Same client-controlled sessionKey from two principals in different
+  // namespaces must not share one transcript identity.
+  const a = observeTranscriptSessionKey("agent:shared-key:s1", "project-origin-aaa", config);
+  const b = observeTranscriptSessionKey("agent:shared-key:s1", "project-origin-bbb", config);
+  assert.notEqual(a, b);
+  assert.ok(a.includes("project-origin-aaa"));
+  assert.ok(b.includes("project-origin-bbb"));
+  // A default-store write keeps the raw key (single-store deployments
+  // unchanged, mirroring the LCM archive key rules).
+  assert.equal(observeTranscriptSessionKey("agent:shared-key:s1", "generalist", config), "agent:shared-key:s1");
+});
+
+test("skip-listed channel types report transcriptPersisted=false, not a silent success (#2783 review)", async () => {
+  await withTempDir(async (dir) => {
+    // cron is in the default skip list; a session whose channel type
+    // resolves to it must not report persisted=true for a no-op append.
+    const { service } = makeService(dir, true, ["cron"]);
+    const response = await service.observe({
+      sessionKey: "agent:nightly:cron:job-1",
+      messages: [{ role: "user", content: "this nightly cron turn must not claim transcript persistence" }],
+    });
+    assert.equal(response.transcriptPersisted, false);
+    const entries = await readAllTranscriptLines(dir);
+    assert.equal(entries.length, 0);
+  });
+});
+
+test("overlapping identical observes do not double-append the same turn (#2783 review)", async () => {
+  await withTempDir(async (dir) => {
+    const { service } = makeService(dir, true);
+    const [a, b] = await Promise.all([
+      service.observe({
+        sessionKey: "agent:overlap:s1",
+        messages: [{ role: "user", content: "the same turn submitted concurrently by a retrying client" }],
+      }),
+      service.observe({
+        sessionKey: "agent:overlap:s1",
+        messages: [{ role: "user", content: "the same turn submitted concurrently by a retrying client" }],
+      }),
+    ]);
+    const entries = await readAllTranscriptLines(dir);
+    assert.equal(entries.length, 1);
+    assert.equal(a.transcriptPersisted !== b.transcriptPersisted, true);
+  });
+});

package/src/access-observe-transcript.ts

@@ -0,0 +1,153 @@
+/**
+ * Observe-derived transcript persistence (issue #2783).
+ *
+ * Delegate-mode gateways POST every turn to `/observe`; without a transcript
+ * append in the daemon's observe path the transcript store never grows and
+ * the hourly summarizer starves silently while every run reports ok. This
+ * module is that append: it mirrors the embedded runtime's `agent_end`
+ * handler (src/index.ts) — same `TranscriptManager`, same entry shape, same
+ * `transcriptEnabled` presentation gate, same 10-char noise floor.
+ *
+ * Turns are deduped against a bounded process-lifetime (session, role,
+ * content) fingerprint — the daemon-side equivalent of the embedded
+ * runtime's `observedInboundContentFingerprints`, extended to both roles
+ * because an un-keyed client re-POST always carries the whole turn — so a
+ * retry cannot double-write it.
+ */
+
+import { createHash, randomUUID } from "node:crypto";
+import { lcmSessionKeyForNamespace } from "./coding/coding-namespace.js";
+import type { PluginConfig } from "./types.js";
+import { log } from "./logger.js";
+import type { Orchestrator } from "./orchestrator.js";
+import type { TranscriptEntry } from "./types.js";
+
+const OBSERVE_TRANSCRIPT_DEDUPE_MAX_ENTRIES = 8192;
+const OBSERVE_TRANSCRIPT_MIN_CONTENT_CHARS = 10;
+/**
+ * A fingerprint only suppresses re-POSTs inside this window. A client retry
+ * lands seconds later; a session LEGITIMATELY repeating the same >10-char
+ * turn usually does so minutes or hours later, and that repeat is a real
+ * conversation event the transcripts must keep (review: preserve legitimate
+ * repeated turns).
+ */
+const OBSERVE_TRANSCRIPT_DEDUPE_TTL_MS = 10 * 60 * 1000;
+
+/**
+ * The transcript-side session identity for an observe payload (issue #2783
+ * review: namespace scoping). Two authenticated principals can submit the
+ * same client-controlled sessionKey while resolving to different effective
+ * write namespaces; persisting through the raw key would let one
+ * principal's transcript-derived summaries surface under the other's
+ * session. Mirror the LCM archive's answer (rule 42 parity): prefix the
+ * key with the effective write namespace whenever it diverges from the
+ * default store. The summarizer lists sessions from the store itself, so
+ * reads self-consistently follow the same prefixed keys.
+ */
+export function observeTranscriptSessionKey(
+  sessionKey: string,
+  writeNamespace: string,
+  config: Pick<PluginConfig, "defaultNamespace">
+): string {
+  return lcmSessionKeyForNamespace(writeNamespace, sessionKey, config.defaultNamespace) ?? sessionKey;
+}
+
+/** Bounded FIFO of seen fingerprints; insertion order is the eviction order. */
+export class ObserveTranscriptPersister {
+  private readonly seenFingerprintsAt = new Map<string, number>();
+  /** In-flight appends by fingerprint: an overlapping identical observe waits for the first append instead of double-writing. */
+  private readonly inflightAppends = new Map<string, Promise<void>>();
+
+  /**
+   * Append every eligible message of an observe payload to the per-session
+   * transcript store. Best-effort: a failing append logs and does not fail
+   * the observe. Returns true when at least one turn was appended.
+   */
+  async persist(
+    orchestrator: Orchestrator,
+    sessionKey: string,
+    messages: ReadonlyArray<{ role: string; content: string }>,
+    rawSessionKey?: string
+  ): Promise<boolean> {
+    let persisted = false;
+    // TranscriptManager.append silently no-ops for skip-listed channel types
+    // (cron by default) — check BEFORE appending so the response's
+    // transcriptPersisted stays truthful (review: report skipped appends).
+    // The channel identity must derive from the RAW session key: the legacy
+    // parser requires the key to start with "agent:", so a namespace-prefixed
+    // transcript identity would always classify as "session" and the skip
+    // list would silently not apply to namespaced cron sessions (review r4).
+    const { channelType } = orchestrator.transcript.getTranscriptPath(rawSessionKey ?? sessionKey);
+    if (orchestrator.config.transcriptSkipChannelTypes.includes(channelType)) {
+      return false;
+    }
+    for (const message of messages) {
+      if (message.content.length < OBSERVE_TRANSCRIPT_MIN_CONTENT_CHARS) continue;
+      // Fixed-size digest, never the raw content: 8192 retained entries of
+      // full turn text is an avoidable heap-exhaustion surface on a daemon
+      // that accepts arbitrary observe payloads (review round 2).
+      const fingerprint = createHash("sha256")
+        .update(`${sessionKey}\0${message.role}\0${message.content}`)
+        .digest("hex");
+      // Two identical observes can overlap before either records its
+      // fingerprint; both would then append and the transcript would carry
+      // the turn twice. The attempt loop serializes per fingerprint: a caller
+      // that finds an in-flight append awaits it and RE-RUNS BOTH checks — on
+      // success the completed-set check skips the message; on failure (no
+      // fingerprint recorded) exactly ONE waiter proceeds to retry, because
+      // the check-then-register sequence has no await between its steps and
+      // is therefore atomic on the event loop (review r4).
+      for (let attempt = 0; attempt < 2; attempt++) {
+        const seenAt = this.seenFingerprintsAt.get(fingerprint);
+        if (seenAt !== undefined && Date.now() - seenAt < OBSERVE_TRANSCRIPT_DEDUPE_TTL_MS) {
+          break; // duplicate within the retry window — skip the message
+        }
+        const inflight = this.inflightAppends.get(fingerprint);
+        if (inflight) {
+          await inflight;
+          continue; // re-run both checks with the append's outcome visible
+        }
+        const entry: TranscriptEntry = {
+          timestamp: new Date().toISOString(),
+          role: message.role as TranscriptEntry["role"],
+          content: message.content,
+          sessionKey,
+          turnId: randomUUID(),
+        };
+        const appendPromise = orchestrator.transcript
+          .append(entry)
+          .then(() => {
+            // Remember only after a successful append: a transient failure
+            // must not permanently swallow the turn — the client's retry has
+            // to be able to re-append it (review round 2).
+            this.remember(fingerprint);
+            persisted = true;
+          })
+          .catch((err) => {
+            // Same policy as the LCM enqueue in the observe path: transcript
+            // persistence must never fail the observe itself.
+            log.error(`access-observe transcript append failed: ${err}`);
+          })
+          .finally(() => {
+            this.inflightAppends.delete(fingerprint);
+          });
+        this.inflightAppends.set(fingerprint, appendPromise);
+        await appendPromise;
+        break;
+      }
+    }
+    return persisted;
+  }
+
+  private remember(fingerprint: string): void {
+    // Map.set keeps an existing key at its ORIGINAL insertion position, so a
+    // TTL-refreshed fingerprint could still be the FIFO-oldest and get
+    // evicted immediately — delete first to move it to the end (review r4).
+    this.seenFingerprintsAt.delete(fingerprint);
+    this.seenFingerprintsAt.set(fingerprint, Date.now());
+    if (this.seenFingerprintsAt.size > OBSERVE_TRANSCRIPT_DEDUPE_MAX_ENTRIES) {
+      const oldest = this.seenFingerprintsAt.keys().next().value;
+      if (typeof oldest === "string") this.seenFingerprintsAt.delete(oldest);
+    }
+  }
+}