@rubytech/create-maxy 1.0.830 → 1.0.832

package/payload/platform/plugins/memory/bin/conversation-archive-ingest.mjs

@@ -1,7 +1,9 @@
 #!/usr/bin/env node
 // =============================================================================
 // conversation-archive-ingest.mjs — in-process orchestrator for
-// conversation-archive-ingest.sh (Task 894 — supersedes whatsapp-import bin).
+// conversation-archive-ingest.sh (Task 894 — supersedes whatsapp-import bin;
+// Task 900 — env-only identity, fsync-per-line progress sink, per-session
+// checkpointing).
 //
 // Source-agnostic. The same pipeline runs for every conversation source;
 // `--source <enum>` selects the normaliser at the top of the pipeline:
@@ -9,26 +11,44 @@
 //   normalise  → bind canonical senders → derive conversationIdentity
 //   → look up prior :ConversationArchive (delta cursor)
 //   → sessionize delta at gap-hours boundaries
-//   → for each session: classify (mode='chat') → collect chunks
-//   → memoryIngest(parentLabel='ConversationArchive', source=<enum>)
+//   → for each session: classify (mode='chat') → memoryIngest immediately
+//                       → advance lastIngestedMessageHash + lastIngestedMessageAt
+//
+// Identity (Task 900): `process.env.ACCOUNT_ID` and `process.env.USER_ID` are
+// the only inputs. Both are plumbed by `spawn-env.ts` into every Bash
+// subprocess; missing/malformed → LOUD-FAIL at phase=argv. `--account-id` and
+// `--owner-element-id` flags are GONE; the writer derives the AdminUser
+// elementId from (accountId, userId) one Cypher hop away.
 //
 // Argv (positional): <archive-path>
-// Argv (flags):       --source <whatsapp|telegram|signal|linkedin-messages|zoom-transcript|meeting-minutes|imessage|slack|other>
-//                     --owner-element-id <id>
+// Argv (flags):       --source <enum>
 //                     --participant-person-ids <csv>
 //                     --scope <admin|public>
 //                     [--session-gap-hours <N>]   (default 12)
-//                     [--account-id <accountId>]
 //                     [--timezone <iana>]
 //                     [--date-format <DD/MM/YY|MM/DD/YY|DD/MM/YYYY|MM/DD/YYYY>]
 //                     [--session-id <id>]
 //
 // Stdout (success): one JSON line — counters the skill needs to formulate
 // the three operator-facing messages. See SKILL.md for the shape.
-// Stderr: one [conversation-archive] FAIL line on failure, exit non-zero.
+// Stderr + progress file: `[conversation-archive] ...` lines, fsync per write.
+// Progress path: data/accounts/<accountId>/logs/conversation-archive-<sessionId>.log
+// Wrapper sh prints `[conversation-archive] progress-file=<absolute-path>` on
+// stdout before exec node, so the agent computes nothing.
 // =============================================================================
 
-import { existsSync, mkdtempSync, readdirSync, rmSync, statSync } from "node:fs";
+import {
+  closeSync,
+  existsSync,
+  fsyncSync,
+  mkdirSync,
+  mkdtempSync,
+  openSync,
+  readdirSync,
+  rmSync,
+  statSync,
+  writeSync,
+} from "node:fs";
 import { join, resolve, dirname } from "node:path";
 import { tmpdir } from "node:os";
 import { spawnSync } from "node:child_process";
@@ -89,13 +109,60 @@ const MEMORY_INGEST_PATH = resolve(
   "tools",
   "memory-ingest.js",
 );
+const UUID_LIB_PATH = resolve(
+  platformRoot,
+  "plugins",
+  "memory",
+  "mcp",
+  "dist",
+  "lib",
+  "uuid.js",
+);
 
 // ---------------------------------------------------------------------------
-// 2. Logger
+// 2. Progress sink + logger fan-out (Task 900 sub-scope A).
+//    `log()` and `fail()` write to stderr AND the progress file with fsync per
+//    write. Parent Bash returns nothing until child exits; the progress file
+//    is the only surface the agent's heartbeat poll can read mid-run.
 // ---------------------------------------------------------------------------
+let progressFd = -1;
+const startedMs = Date.now();
+
+function openProgressSink(absolutePath) {
+  mkdirSync(dirname(absolutePath), { recursive: true });
+  progressFd = openSync(absolutePath, "a");
+}
+
+function writeProgressLine(line) {
+  // stderr first — preserves existing observability for parents that read
+  // stderr after the child exits, AND makes the line visible via the
+  // mcp-spawn-tee'd device server.log without a separate sink.
+  process.stderr.write(line);
+  if (progressFd >= 0) {
+    try {
+      writeSync(progressFd, line);
+      fsyncSync(progressFd);
+    } catch {
+      // Disk pressure / EBADF: don't kill the run — stderr already carries
+      // the line.
+    }
+  }
+}
+
 function log(line) {
-  process.stderr.write(`[conversation-archive] ${line}\n`);
+  writeProgressLine(`[conversation-archive] ${line}\n`);
 }
+
+function endRun(exitCode) {
+  const totalMs = Date.now() - startedMs;
+  writeProgressLine(`[conversation-archive] end exit-code=${exitCode} ms=${totalMs}\n`);
+  if (progressFd >= 0) {
+    try { closeSync(progressFd); } catch { /* fd already closed */ }
+    progressFd = -1;
+  }
+  process.exit(exitCode);
+}
+
 function fail(phase, fields) {
   const fieldStr = Object.entries(fields)
     .map(([k, v]) =>
@@ -104,8 +171,8 @@ function fail(phase, fields) {
         : `${k}=${v ?? "-"}`,
     )
     .join(" ");
-  process.stderr.write(`[conversation-archive] FAIL phase=${phase} ${fieldStr}\n`);
-  process.exit(1);
+  writeProgressLine(`[conversation-archive] FAIL phase=${phase} ${fieldStr}\n`);
+  endRun(1);
 }
 
 // ---------------------------------------------------------------------------
@@ -144,7 +211,6 @@ function parseArgv(argv) {
   if (!VALID_SOURCES.has(flags.source)) {
     fail("argv", { reason: `invalid --source "${flags.source}" (whatsapp|telegram|signal|linkedin-messages|zoom-transcript|meeting-minutes|imessage|slack|other)` });
   }
-  if (!flags.ownerElementId) fail("argv", { reason: "missing --owner-element-id" });
   if (!flags.participantPersonIds) {
     fail("argv", {
       reason: "missing --participant-person-ids (csv of operator-confirmed :Person/:AdminUser elementIds, owner excluded)",
@@ -216,35 +282,41 @@ function findChatTxt(dir) {
 }
 
 // ---------------------------------------------------------------------------
-// 5. Account resolution (Phase 0 = single account)
-// ---------------------------------------------------------------------------
-function resolveAccountId(flags) {
-  if (flags.accountId && flags.accountId.trim()) return flags.accountId.trim();
-  const installDir = resolve(platformRoot, "..");
-  const accountsDir = join(installDir, "data", "accounts");
-  if (!existsSync(accountsDir)) {
-    fail("argv", { reason: `accounts dir not found: ${accountsDir}; pass --account-id explicitly` });
-  }
-  const dirs = readdirSync(accountsDir, { withFileTypes: true })
-    .filter((d) => d.isDirectory() && !d.name.startsWith("."))
-    .map((d) => d.name);
-  if (dirs.length === 0) fail("argv", { reason: `no accounts found under ${accountsDir}` });
-  if (dirs.length > 1) {
-    fail("argv", {
-      reason: `multiple accounts under ${accountsDir} (${dirs.join(",")}); pass --account-id explicitly`,
-    });
-  }
-  return dirs[0];
-}
-
-// ---------------------------------------------------------------------------
-// 6. Main
+// 5. Main
 // ---------------------------------------------------------------------------
 async function main() {
-  const startedMs = Date.now();
   const { archive, flags } = parseArgv(process.argv);
   const source = flags.source;
-  const ownerElementId = flags.ownerElementId;
+
+  // Identity is env-only (Task 900 sub-scope C). spawn-env.ts plumbs both
+  // ACCOUNT_ID and USER_ID into every Bash subprocess; the bin LOUD-FAILs if
+  // either is absent or malformed. The disk-scan resolver and the
+  // --account-id / --owner-element-id flags are gone.
+  let UUID_REGEX;
+  try {
+    ({ UUID_REGEX } = await import(UUID_LIB_PATH));
+  } catch (err) {
+    fail("import", {
+      reason: "failed to import compiled uuid lib",
+      detail: err instanceof Error ? err.message : String(err),
+    });
+  }
+
+  const accountId = process.env.ACCOUNT_ID?.trim();
+  if (!accountId) {
+    fail("argv", { reason: "ACCOUNT_ID env missing — bin must be invoked under an authenticated agent context" });
+  }
+  if (!UUID_REGEX.test(accountId)) {
+    fail("argv", { reason: `ACCOUNT_ID env malformed (not a UUID): "${accountId}"` });
+  }
+  const userId = process.env.USER_ID?.trim();
+  if (!userId) {
+    fail("argv", { reason: "USER_ID env missing — bin must be invoked under an authenticated user session" });
+  }
+  if (!UUID_REGEX.test(userId)) {
+    fail("argv", { reason: `USER_ID env malformed (not a UUID): "${userId}"` });
+  }
+
   const participantElementIds = flags.participantPersonIds
     .split(",")
     .map((s) => s.trim())
@@ -253,7 +325,6 @@ async function main() {
     fail("argv", { reason: "--participant-person-ids must list at least one elementId" });
   }
   const scope = flags.scope;
-  const accountId = resolveAccountId(flags);
   const timezone = flags.timezone || "Europe/London";
   const dateFormat = flags.dateFormat;
   const sessionGapHours = flags.sessionGapHours
@@ -266,20 +337,34 @@ async function main() {
     flags.sessionId ||
     `conversation-archive:${source}:${Date.now()}:${Math.random().toString(36).slice(2, 10)}`;
 
+  // ---------------------------------------------------------------------------
+  // Open progress sink BEFORE any await. Path is deterministic from
+  // (accountId, sessionId); the wrapper sh prints the same path on stdout so
+  // the agent's heartbeat polls the right file.
+  // ---------------------------------------------------------------------------
+  const accountsDir = resolve(platformRoot, "..", "data", "accounts");
+  const logsDir = resolve(accountsDir, accountId, "logs");
+  const progressFilePath = resolve(logsDir, `conversation-archive-${sessionId}.log`);
+  openProgressSink(progressFilePath);
+
+  log(
+    `start session=${sessionId} source=${source} archive=${archive} accountId=${accountId} userId=${userId} participants=${participantElementIds.length} scope=${scope} pid=${process.pid}`,
+  );
+
   // Imports — fail loudly if any compiled dist missing
   let getNormaliser;
   let sessionize, toTurnText, findDeltaCursor;
-  let normaliseSenderName, deriveConversationIdentity, deriveMessageContentHash;
-  let bindCanonicalSenders;
+  let deriveConversationIdentity, deriveMessageContentHash;
+  let verifyParticipantElementIds;
   let getSession, classifyDocument, memoryIngest;
   try {
     ({ getNormaliser } = await import(NORMALISERS_PATH));
     ({ sessionize } = await import(join(PIPELINE_PATH, "sessionize.js")));
     ({ toTurnText } = await import(join(PIPELINE_PATH, "to-turn-text.js")));
     ({ findDeltaCursor } = await import(join(PIPELINE_PATH, "delta-cursor.js")));
-    ({ normaliseSenderName, deriveConversationIdentity, deriveMessageContentHash } =
+    ({ deriveConversationIdentity, deriveMessageContentHash } =
       await import(join(PIPELINE_PATH, "derive-keys.js")));
-    ({ bindCanonicalSenders } = await import(join(PIPELINE_PATH, "sender-bind.js")));
+    ({ verifyParticipantElementIds } = await import(join(PIPELINE_PATH, "sender-bind.js")));
     ({ getSession } = await import(NEO4J_LIB_PATH));
     ({ classifyDocument } = await import(LLM_CLASSIFIER_PATH));
     ({ memoryIngest } = await import(MEMORY_INGEST_PATH));
@@ -290,6 +375,36 @@ async function main() {
     });
   }
 
+  // Resolve owner AdminUser elementId from (accountId, userId). One Cypher
+  // hop replaces the deleted --owner-element-id flag — owner identity is env
+  // input, never an out-of-band flag.
+  let ownerElementId;
+  {
+    const lookupSession = getSession();
+    try {
+      const r = await lookupSession.run(
+        `MATCH (au:AdminUser { accountId: $accountId, userId: $userId })
+         RETURN elementId(au) AS ownerElementId LIMIT 1`,
+        { accountId, userId },
+      );
+      const row = r.records[0]?.get("ownerElementId");
+      if (typeof row !== "string" || !row) {
+        await lookupSession.close().catch(() => {});
+        fail("argv", {
+          reason: `USER_ID does not resolve to an AdminUser under ACCOUNT_ID (accountId=${accountId.slice(0, 8)}, userId=${userId.slice(0, 8)})`,
+        });
+      }
+      ownerElementId = row;
+    } catch (err) {
+      await lookupSession.close().catch(() => {});
+      fail("argv", {
+        reason: `AdminUser lookup failed: ${err instanceof Error ? err.message : String(err)}`,
+      });
+    }
+    await lookupSession.close().catch(() => {});
+  }
+  log(`owner-resolved ownerElementId=${ownerElementId.slice(0, 12)} userId=${userId.slice(0, 8)}`);
+
   // 6a. Resolve source file
   const { sourceFile, cleanup } = resolveSourceFile(archive, source);
 
@@ -316,29 +431,34 @@ async function main() {
   log(
     `parsed lines=${normaliserResult.counters.parsed} media-skipped=${normaliserResult.counters.mediaSkipped} system-skipped=${normaliserResult.counters.systemSkipped}`,
   );
+  // Task 897: zero matched timestamps means this isn't a chat archive — the
+  // file should route through document-ingest instead. The agent's
+  // database-operator skill picks the right entry; this LOUD-FAIL surfaces
+  // misroute as an actionable signal rather than a silent empty ingest.
   if (allLines.length === 0) {
     cleanup();
-    fail("parse", { reason: "zero parsed lines after walking archive" });
+    fail("parse", {
+      reason: `no timestamp prefixes detected — this archive contains no chat messages with the expected source=${source} grammar; route through document-ingest with mode='document', not conversation-archive-ingest`,
+    });
   }
 
-  // 6c. Bind canonical senders against the confirmed set
-  const distinctSenderNames = Array.from(new Set(allLines.map((l) => l.senderName)));
-  const senderHistogram = computeSenderHistogram(allLines);
+  // 6c. Verify participant elementIds resolve to graph nodes with the right
+  // accountId and labels. Per-line senderName binding is gone (Task 897);
+  // operator-confirmed participants attach to the parent :ConversationArchive.
   let session = getSession();
   try {
-    await bindCanonicalSenders({
+    await verifyParticipantElementIds({
       session,
       accountId,
       ownerElementId,
       participantElementIds,
-      senderNames: distinctSenderNames,
     });
   } catch (err) {
     await session.close().catch(() => {});
     cleanup();
     if (err && err.userFacing) {
-      process.stderr.write(`[conversation-archive] FAIL ${err.message}\n`);
-      process.exit(1);
+      writeProgressLine(`[conversation-archive] FAIL ${err.message}\n`);
+      endRun(1);
     }
     fail("argv", { reason: err instanceof Error ? err.message : String(err) });
   }
@@ -403,12 +523,13 @@ async function main() {
         nextEdgesCreated: 0,
         participantsLinked: 0,
         dateRange: { first: allLines[0].dateSent, last: allLines[allLines.length - 1].dateSent },
-        senderHistogram,
+        parsedLineCount: allLines.length,
+        sessionsUnenriched: 0,
         topicKeywords: [],
         ms: totalMs,
         priorLastIngestedMessageAt: priorArchive.lastAt,
       }) + "\n");
-      process.exit(0);
+      endRun(0);
     }
     deltaStart = cursor.deltaStart;
     deltaKind = "delta";
@@ -424,10 +545,24 @@ async function main() {
     `sessionize source=${source} archiveSha256=${archiveSha256.slice(0, 12)} messages=${deltaLines.length} sessions=${sessions.length} gap-hours=${sessionGapHours}`,
   );
 
-  // 6g. Classify each session via Haiku (mode='chat')
-  const allChunks = [];
+  // 6g. Per-session classify + immediate memoryIngest (Task 900 sub-scope E).
+  // Each session commits atomically: chunks + cursor advance happen together
+  // via memoryIngest. A kill mid-loop leaves a partial archive whose cursor
+  // is at session N-1's last message; the next invocation slices from there
+  // and re-classifies only session N onward. degrade-on-error from Task 897
+  // is preserved: a Haiku error on session N still emits an unenriched chunk
+  // and advances the cursor, so resumption never re-attempts the same Haiku
+  // error.
   const allKeywords = new Set();
-  for (const s of sessions) {
+  let sessionsUnenriched = 0;
+  let totalChunksWritten = 0;
+  let totalNextEdges = 0;
+  let participantsLinked = 0;
+  let archiveElementIdLatest = priorArchive ? priorArchive.elemId : "";
+
+  for (let sIdx = 0; sIdx < sessions.length; sIdx++) {
+    const s = sessions[sIdx];
+    const isFirstSessionInRun = sIdx === 0;
     const sessionStart = Date.now();
     const text = toTurnText(s);
     const result = await classifyDocument({
@@ -438,74 +573,114 @@ async function main() {
       naturalEdgeMap: "",
       documentText: text,
     });
-    if (result.kind === "fallback") {
-      cleanup();
-      fail("classify", { reason: `Haiku fallback on session ${s.index}: ${result.reason}` });
+
+    let sessionChunks;
+    let sessionKeywords = [];
+    if (result.kind === "error") {
+      sessionsUnenriched += 1;
+      log(`session-unenriched session=${s.index + 1}/${sessions.length} reason="${result.reason.replace(/"/g, '\\"')}"`);
+      // Emit one raw :Section:Conversation chunk covering the whole session.
+      // No summary, no topicKeywords — a follow-up pass can re-enrich.
+      sessionChunks = [{
+        kind: "Conversation",
+        title: `Unenriched session ${s.index + 1}`,
+        body: text,
+        summary: "",
+        sourceStart: 0,
+        sourceEnd: text.length,
+        properties: {
+          unenriched: true,
+          unenrichedReason: result.reason,
+          firstMessageAt: s.firstMessageAt,
+          lastMessageAt: s.lastMessageAt,
+          messageCount: s.messages.length,
+        },
+        anchorEdge: null,
+      }];
+    } else {
+      const chunkCount = result.output.sections.length;
+      log(
+        `classify-session sessionIndex=${s.index + 1}/${sessions.length} messages=${s.messages.length} chars=${text.length} chunks=${chunkCount} ms=${Date.now() - sessionStart}`,
+      );
+      if (chunkCount === 0 && s.messages.length > 0) {
+        cleanup();
+        fail("classify", {
+          reason: `session ${s.index} of ${s.messages.length} messages produced zero chunks (classifier-prompt regression)`,
+        });
+      }
+      sessionChunks = result.output.sections;
+      sessionKeywords = result.output.documentKeywords ?? [];
+      for (const kw of sessionKeywords) allKeywords.add(kw);
     }
-    const chunkCount = result.output.sections.length;
-    log(
-      `classify-session sessionIndex=${s.index + 1}/${sessions.length} messages=${s.messages.length} chars=${text.length} chunks=${chunkCount} ms=${Date.now() - sessionStart}`,
-    );
-    if (chunkCount === 0 && s.messages.length > 0) {
+
+    // Per-session cursor: advance to THIS session's last message. Atomic with
+    // chunk writes inside memoryIngest's MERGE/CREATE Cypher transaction.
+    const lastSessionLine = s.messages[s.messages.length - 1];
+    const sessionLastHash = deriveMessageContentHash({
+      dateSent: lastSessionLine.dateSent,
+      body: lastSessionLine.body,
+    });
+    const sessionLastAt = lastSessionLine.dateSent;
+    const sessionDocumentSummary = `Session ${s.index + 1}/${sessions.length}: ${s.messages.length} messages, ${sessionChunks.length} chunks.`;
+
+    let ingestResult;
+    const ingestStart = Date.now();
+    try {
+      ingestResult = await memoryIngest({
+        accountId,
+        attachmentId: conversationIdentity,
+        parentLabel: "ConversationArchive",
+        source,
+        documentSummary: sessionDocumentSummary,
+        anchorNodeId: ownerElementId,
+        anchorLabel: "AdminUser",
+        sections: sessionChunks,
+        scope,
+        sessionId,
+        documentKeywords: Array.from(allKeywords),
+        archiveSha256,
+        archiveSourceFile,
+        lastIngestedMessageHash: sessionLastHash,
+        lastIngestedMessageAt: sessionLastAt,
+        participantElementIds: [ownerElementId, ...participantElementIds],
+        // First session of THIS run cleans prior chunks for matching
+        // archiveSha256 (re-run-with-same-bytes idempotency); subsequent
+        // sessions in the same run skip cleanup or they would delete the
+        // chunks just written by sessions above them.
+        cleanupPriorChunks: isFirstSessionInRun,
+      });
+    } catch (err) {
       cleanup();
-      fail("classify", {
-        reason: `session ${s.index} of ${s.messages.length} messages produced zero chunks (classifier-prompt regression)`,
+      fail("memory-ingest", {
+        reason: err instanceof Error ? err.message : String(err),
+        sessionIndex: s.index + 1,
+        sessionsTotal: sessions.length,
       });
     }
-    for (const sec of result.output.sections) allChunks.push(sec);
-    for (const kw of result.output.documentKeywords) allKeywords.add(kw);
-  }
 
-  // 6h. Compute lastIngestedMessageHash from the last delta line
-  const lastLine = deltaLines[deltaLines.length - 1];
-  const lastIngestedMessageHash = deriveMessageContentHash({
-    dateSent: lastLine.dateSent,
-    senderName: lastLine.senderName,
-    body: lastLine.body,
-  });
-  const lastIngestedMessageAt = lastLine.dateSent;
-
-  // 6i. Aggregate document-level summary across sessions
-  const documentSummary = sessions.length === 1
-    ? `${deltaLines.length} messages in 1 session, ${allChunks.length} chunks.`
-    : `${deltaLines.length} messages in ${sessions.length} sessions, ${allChunks.length} chunks.`;
+    archiveElementIdLatest = ingestResult.documentNodeId;
+    totalChunksWritten += ingestResult.sectionCount;
+    totalNextEdges += ingestResult.edgeBreakdown.NEXT ?? 0;
+    if (isFirstSessionInRun) {
+      participantsLinked = ingestResult.edgeBreakdown.PARTICIPANT_IN ?? 0;
+    }
+    log(
+      `session-committed sessionIndex=${s.index + 1}/${sessions.length} chunks-written=${ingestResult.sectionCount} cursor-at="${sessionLastAt}" ms=${Date.now() - ingestStart}`,
+    );
+  }
 
-  // 6j. Call memoryIngest with parentLabel='ConversationArchive'
-  let ingestResult;
-  const ingestStart = Date.now();
-  try {
-    ingestResult = await memoryIngest({
-      accountId,
-      attachmentId: conversationIdentity,
-      parentLabel: "ConversationArchive",
-      source,
-      documentSummary,
-      anchorNodeId: ownerElementId,
-      anchorLabel: "AdminUser",
-      sections: allChunks,
-      scope,
-      sessionId,
-      documentKeywords: Array.from(allKeywords),
-      archiveSha256,
-      archiveSourceFile,
-      lastIngestedMessageHash,
-      lastIngestedMessageAt,
-      participantElementIds: [ownerElementId, ...participantElementIds],
-    });
-  } catch (err) {
-    cleanup();
-    fail("memory-ingest", { reason: err instanceof Error ? err.message : String(err) });
+  if (sessionsUnenriched > 0) {
+    log(`classify-summary sessions=${sessions.length} unenriched=${sessionsUnenriched} enriched=${sessions.length - sessionsUnenriched}`);
   }
-  log(
-    `source=${source} file=${archiveSourceFile} conversationIdentity=${conversationIdentity.slice(0, 12)} archiveElementId=${ingestResult.documentNodeId} chunks-written=${ingestResult.sectionCount} next-edges=${ingestResult.edgeBreakdown.NEXT ?? 0} participants=${ingestResult.edgeBreakdown.PARTICIPANT_IN ?? 0} ms=${Date.now() - ingestStart}`,
-  );
 
   cleanup();
   const totalMs = Date.now() - startedMs;
-  log(`done source=${source} conversationIdentity=${conversationIdentity.slice(0, 12)} total-ms=${totalMs} exit=0`);
+  log(
+    `done source=${source} conversationIdentity=${conversationIdentity.slice(0, 12)} archiveElementId=${archiveElementIdLatest} chunks-written=${totalChunksWritten} sessions=${sessions.length} total-ms=${totalMs}`,
+  );
 
   process.stdout.write(JSON.stringify({
-    archiveElementId: ingestResult.documentNodeId,
+    archiveElementId: archiveElementIdLatest,
     conversationIdentity,
     archiveSha256,
     archiveSourceFile,
@@ -515,25 +690,16 @@ async function main() {
     systemSkipped: normaliserResult.counters.systemSkipped,
     delta: { kind: deltaKind, deltaStart, deltaMessages: deltaLines.length },
     sessions: sessions.length,
-    chunks: ingestResult.sectionCount,
-    nextEdgesCreated: ingestResult.edgeBreakdown.NEXT ?? 0,
-    participantsLinked: ingestResult.edgeBreakdown.PARTICIPANT_IN ?? 0,
+    chunks: totalChunksWritten,
+    nextEdgesCreated: totalNextEdges,
+    participantsLinked,
     dateRange: { first: allLines[0].dateSent, last: allLines[allLines.length - 1].dateSent },
-    senderHistogram,
+    parsedLineCount: allLines.length,
+    sessionsUnenriched,
     topicKeywords: Array.from(allKeywords),
     ms: totalMs,
   }) + "\n");
-  process.exit(0);
-}
-
-function computeSenderHistogram(lines) {
-  const counts = new Map();
-  for (const l of lines) {
-    counts.set(l.senderName, (counts.get(l.senderName) ?? 0) + 1);
-  }
-  return Array.from(counts.entries())
-    .map(([name, count]) => ({ name, count }))
-    .sort((a, b) => b.count - a.count);
+  endRun(0);
 }
 
 main().catch((err) => {