@rubytech/create-maxy 1.0.799 → 1.0.801

package/payload/platform/plugins/whatsapp-import/bin/ingest.mjs

@@ -0,0 +1,732 @@
+#!/usr/bin/env node
+// =============================================================================
+// ingest.mjs — in-process orchestrator for whatsapp-ingest.sh (Task 855).
+//
+// Collapses parse → archive-write → insight into one Node process so the
+// 357K-char MCP-envelope ceiling no longer exists between steps. The
+// database-operator subagent's only handle on this pipeline is the wrapper
+// shell script; the gate at platform/plugins/admin/hooks/
+// archive-ingest-surface-gate.sh blocks the legacy MCP tools mechanically.
+//
+// Argv (positional): <archive-path>
+// Argv (flags):       --owner-element-id <id> --scope <admin|public>
+//                     [--account-id <accountId>] [--timezone <iana>]
+//                     [--date-format <DD/MM/YY|MM/DD/YY|DD/MM/YYYY|MM/DD/YYYY>]
+//                     [--no-insight]
+//
+// Stdout (success): one JSON line
+//   {conversationId, parsed, mediaSkipped, systemSkipped,
+//    createdMessages, insightCounters, ms}
+//
+// Stderr (failure): one [whatsapp-ingest] FAIL line, exit non-zero.
+// =============================================================================
+
+import {
+  createReadStream,
+  existsSync,
+  mkdtempSync,
+  readdirSync,
+  readFileSync,
+  rmSync,
+  statSync,
+  writeFileSync,
+} from "node:fs";
+import { join, resolve, dirname, basename } from "node:path";
+import { tmpdir } from "node:os";
+import { spawnSync } from "node:child_process";
+import { fileURLToPath } from "node:url";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+// ---------------------------------------------------------------------------
+// 1. Resolve dist paths from the platform install root.
+// ---------------------------------------------------------------------------
+// MAXY_PLATFORM_ROOT is set by the wrapper script (and by claude-agent.ts in
+// production). Fall back to walking up from this file's location: the bin/
+// directory sits at platform/plugins/whatsapp-import/bin/, so platform/ is
+// three levels up.
+
+const platformRoot =
+  process.env.MAXY_PLATFORM_ROOT?.trim() ||
+  resolve(__dirname, "..", "..", "..");
+
+const PARSE_EXPORT_PATH = resolve(
+  platformRoot,
+  "plugins",
+  "whatsapp-import",
+  "lib",
+  "dist",
+  "index.js",
+);
+const ARCHIVE_WRITE_PATH = resolve(
+  platformRoot,
+  "plugins",
+  "memory",
+  "mcp",
+  "dist",
+  "tools",
+  "memory-archive-write.js",
+);
+const NEO4J_LIB_PATH = resolve(
+  platformRoot,
+  "plugins",
+  "memory",
+  "mcp",
+  "dist",
+  "lib",
+  "neo4j.js",
+);
+const OAUTH_LLM_PATH = resolve(
+  platformRoot,
+  "lib",
+  "oauth-llm",
+  "dist",
+  "index.js",
+);
+const MODELS_PATH = resolve(
+  platformRoot,
+  "lib",
+  "models",
+  "dist",
+  "index.js",
+);
+
+// ---------------------------------------------------------------------------
+// 2. Logger — every line `[whatsapp-ingest]`-prefixed to stderr (server.log).
+// ---------------------------------------------------------------------------
+
+function log(line) {
+  process.stderr.write(`[whatsapp-ingest] ${line}\n`);
+}
+
+function fail(phase, fields) {
+  // Single failure line. Used as the only stderr line on non-zero exit.
+  const fieldStr = Object.entries(fields)
+    .map(([k, v]) =>
+      typeof v === "string" && (v.includes(" ") || v.includes("="))
+        ? `${k}="${v.replace(/"/g, '\\"')}"`
+        : `${k}=${v ?? "-"}`,
+    )
+    .join(" ");
+  process.stderr.write(`[whatsapp-ingest] FAIL phase=${phase} ${fieldStr}\n`);
+  process.exit(1);
+}
+
+// ---------------------------------------------------------------------------
+// 3. Argv parsing.
+// ---------------------------------------------------------------------------
+
+function parseArgv(argv) {
+  const args = argv.slice(2);
+  let archive = null;
+  const flags = {};
+  for (let i = 0; i < args.length; i++) {
+    const a = args[i];
+    if (!a.startsWith("--")) {
+      if (archive == null) archive = a;
+      else fail("argv", { reason: `unexpected positional argument "${a}"` });
+      continue;
+    }
+    const key = a.slice(2);
+    if (key === "no-insight") {
+      flags.noInsight = true;
+      continue;
+    }
+    const v = args[++i];
+    if (v == null) fail("argv", { reason: `flag --${key} requires a value` });
+    flags[camelCase(key)] = v;
+  }
+  if (!archive) fail("argv", { reason: "missing positional <archive>" });
+  if (!flags.ownerElementId) fail("argv", { reason: "missing --owner-element-id" });
+  if (!flags.scope) fail("argv", { reason: "missing --scope" });
+  if (flags.scope !== "admin" && flags.scope !== "public") {
+    fail("argv", { reason: `invalid --scope "${flags.scope}" (admin|public)` });
+  }
+  return { archive, flags };
+}
+
+function camelCase(s) {
+  return s.replace(/-([a-z])/g, (_m, c) => c.toUpperCase());
+}
+
+// ---------------------------------------------------------------------------
+// 4. Archive resolution — accept .zip / dir / _chat.txt; return absolute
+//    path to _chat.txt and a cleanup callback for any tmp dir we created.
+// ---------------------------------------------------------------------------
+
+function resolveChatTxt(archivePath) {
+  const abs = resolve(archivePath);
+  if (!existsSync(abs)) {
+    fail("argv", { reason: `archive path not found: ${abs}` });
+  }
+  const st = statSync(abs);
+
+  if (st.isFile() && abs.endsWith(".zip")) {
+    const tmp = mkdtempSync(join(tmpdir(), "whatsapp-ingest-"));
+    const unzip = spawnSync("unzip", ["-q", "-o", abs, "-d", tmp], {
+      encoding: "utf8",
+    });
+    if (unzip.status !== 0) {
+      rmSync(tmp, { recursive: true, force: true });
+      fail("argv", {
+        reason: "unzip failed",
+        archive: abs,
+        stderr: (unzip.stderr || "").slice(0, 200),
+      });
+    }
+    const chat = findChatTxt(tmp);
+    if (!chat) {
+      rmSync(tmp, { recursive: true, force: true });
+      fail("argv", { reason: "_chat.txt not found in zip", archive: abs });
+    }
+    return { chatTxt: chat, cleanup: () => rmSync(tmp, { recursive: true, force: true }) };
+  }
+
+  if (st.isDirectory()) {
+    const chat = findChatTxt(abs);
+    if (!chat) fail("argv", { reason: "_chat.txt not found in directory", archive: abs });
+    return { chatTxt: chat, cleanup: () => {} };
+  }
+
+  if (st.isFile()) {
+    return { chatTxt: abs, cleanup: () => {} };
+  }
+
+  fail("argv", { reason: `unsupported archive shape: ${abs}` });
+  // unreachable, but keeps tools happy
+  return { chatTxt: abs, cleanup: () => {} };
+}
+
+function findChatTxt(dir) {
+  const entries = readdirSync(dir, { withFileTypes: true });
+  for (const e of entries) {
+    if (e.isFile() && e.name === "_chat.txt") return join(dir, e.name);
+  }
+  for (const e of entries) {
+    if (e.isDirectory()) {
+      const nested = findChatTxt(join(dir, e.name));
+      if (nested) return nested;
+    }
+  }
+  return null;
+}
+
+// ---------------------------------------------------------------------------
+// 5. Account resolution — Phase 0 has one account dir. The wrapper passes
+//    --account-id when it is known; otherwise we pick the single dir under
+//    {install}/data/accounts/.
+// ---------------------------------------------------------------------------
+
+function resolveAccountId(flags) {
+  if (flags.accountId && flags.accountId.trim()) return flags.accountId.trim();
+  const installDir = resolve(platformRoot, "..");
+  const accountsDir = join(installDir, "data", "accounts");
+  if (!existsSync(accountsDir)) {
+    fail("argv", {
+      reason: `accounts dir not found: ${accountsDir}; pass --account-id explicitly`,
+    });
+  }
+  const dirs = readdirSync(accountsDir, { withFileTypes: true })
+    .filter((d) => d.isDirectory() && !d.name.startsWith("."))
+    .map((d) => d.name);
+  if (dirs.length === 0) {
+    fail("argv", { reason: `no accounts found under ${accountsDir}` });
+  }
+  if (dirs.length > 1) {
+    fail("argv", {
+      reason: `multiple accounts under ${accountsDir} (${dirs.join(",")}); pass --account-id explicitly`,
+    });
+  }
+  return dirs[0];
+}
+
+// ---------------------------------------------------------------------------
+// 6. Auto-create participants — one :Person node per distinct senderName
+//    minus the owner. Provenance + participantStatus='auto-created' so a
+//    later semantic-enrichment pass (Task 856) can promote/disambiguate.
+// ---------------------------------------------------------------------------
+
+const PARTICIPANT_UPSERT_CYPHER = `
+UNWIND $names AS senderName
+MERGE (p:Person {accountId: $accountId, source: 'whatsapp', name: senderName})
+  ON CREATE SET
+    p.createdByAgent     = 'whatsapp-import',
+    p.createdBySource    = 'whatsapp-import',
+    p.createdBySession   = $sessionId,
+    p.createdAt          = datetime(),
+    p.scope              = $scope,
+    p.participantStatus  = 'auto-created'
+RETURN elementId(p) AS elemId, senderName AS name
+`;
+
+async function upsertAutoParticipants({
+  session,
+  accountId,
+  scope,
+  sessionId,
+  senderNames,
+}) {
+  if (senderNames.length === 0) return new Map();
+  // Owner-as-sender reconciliation is deferred to Task 856 — when the
+  // owner's display name appears as a sender, the messages SENT-edge from
+  // the auto-created :Person, not from the operator's :AdminUser. The
+  // post-load enrichment skill rewrites those edges on operator confirm.
+  return await session.executeWrite(async (tx) => {
+    const res = await tx.run(PARTICIPANT_UPSERT_CYPHER, {
+      names: senderNames,
+      accountId,
+      scope,
+      sessionId,
+    });
+    const map = new Map();
+    for (const r of res.records) {
+      map.set(r.get("name"), r.get("elemId"));
+    }
+    return map;
+  });
+}
+
+// ---------------------------------------------------------------------------
+// 7. Insight pass — chunked Haiku call, in-process Cypher writes.
+//    One :Observation node per extracted item, OBSERVED_IN edge to the
+//    Conversation. Edge wiring to specific :Person/:Task/etc. is Task 856.
+// ---------------------------------------------------------------------------
+
+const INSIGHT_CHUNK_SIZE = 1500; // messages per Haiku call (≈75K tokens budget)
+
+const INSIGHT_SYSTEM_PROMPT = `You extract structured insights from a chunk of a WhatsApp conversation.
+
+Return STRICT JSON via the provided tool. No prose, no commentary. Only items with concrete, verbatim evidence in the chunk. Empty arrays are valid; prefer omission to invention.
+
+Definitions:
+- "mention": a person, organisation, place, or named topic referred to by name.
+- "task": something a participant committed to do or asked another to do (imperative or future-tense).
+- "preference": stated like, dislike, opinion, or rule of behaviour.
+- "observedRelationship": an explicit relational claim (works at, is married to, manages, etc.).
+
+Snippets must be ≤80 characters of the original message body, no sender names, no timestamps.`;
+
+const INSIGHT_TOOL = {
+  name: "submit_insights",
+  description: "Submit the structured insights extracted from the chunk.",
+  input_schema: {
+    type: "object",
+    properties: {
+      mentions: {
+        type: "array",
+        items: {
+          type: "object",
+          properties: {
+            name: { type: "string" },
+            snippet: { type: "string" },
+          },
+          required: ["name", "snippet"],
+        },
+      },
+      tasks: {
+        type: "array",
+        items: {
+          type: "object",
+          properties: {
+            task: { type: "string" },
+            snippet: { type: "string" },
+          },
+          required: ["task", "snippet"],
+        },
+      },
+      preferences: {
+        type: "array",
+        items: {
+          type: "object",
+          properties: {
+            subject: { type: "string" },
+            preference: { type: "string" },
+          },
+          required: ["subject", "preference"],
+        },
+      },
+      observedRelationships: {
+        type: "array",
+        items: {
+          type: "object",
+          properties: {
+            from: { type: "string" },
+            to: { type: "string" },
+            relationship: { type: "string" },
+          },
+          required: ["from", "to", "relationship"],
+        },
+      },
+    },
+    required: ["mentions", "tasks", "preferences", "observedRelationships"],
+  },
+};
+
+const INSIGHT_WRITE_CYPHER = `
+MATCH (c:Conversation:WhatsAppConversation {conversationId: $conversationId})
+UNWIND $observations AS obs
+CREATE (o:Observation)
+SET
+  o:WhatsAppObservation,
+  o.accountId        = $accountId,
+  o.kind             = obs.kind,
+  o.summary          = obs.summary,
+  o.snippet          = obs.snippet,
+  o.subject          = obs.subject,
+  o.from             = obs.from,
+  o.to               = obs.to,
+  o.source           = 'whatsapp',
+  o.createdByAgent   = 'whatsapp-import',
+  o.createdBySource  = 'whatsapp-import',
+  o.createdBySession = $sessionId,
+  o.createdAt        = datetime(),
+  o.scope            = $scope,
+  o.insightPass      = true,
+  o.observationStatus = 'auto-extracted'
+MERGE (o)-[r:OBSERVED_IN]->(c)
+  ON CREATE SET r.source = 'whatsapp', r.createdAt = datetime()
+RETURN count(o) AS created
+`;
+
+async function runInsightPass({
+  callOauthLlm,
+  HAIKU_MODEL,
+  session,
+  conversationId,
+  accountId,
+  scope,
+  sessionId,
+  parsedLines,
+}) {
+  const counters = {
+    chunks: 0,
+    mentions: 0,
+    tasks: 0,
+    preferences: 0,
+    observedRelationships: 0,
+  };
+
+  if (parsedLines.length === 0) return counters;
+
+  // Build chunks. Each chunk is rendered as a numbered transcript including
+  // sender-names — the LLM needs them to attribute tasks/preferences/observed-
+  // relationships. The system prompt's "no sender names" rule constrains the
+  // returned snippet field only, not the input transcript.
+  const chunks = [];
+  for (let i = 0; i < parsedLines.length; i += INSIGHT_CHUNK_SIZE) {
+    chunks.push(parsedLines.slice(i, i + INSIGHT_CHUNK_SIZE));
+  }
+  counters.chunks = chunks.length;
+
+  for (let chunkIdx = 0; chunkIdx < chunks.length; chunkIdx++) {
+    const chunk = chunks[chunkIdx];
+    const lines = chunk
+      .map((l, j) => `[${j + 1}] ${l.senderName}: ${l.body}`)
+      .join("\n");
+
+    let llmResult;
+    try {
+      llmResult = await callOauthLlm({
+        model: HAIKU_MODEL,
+        system: INSIGHT_SYSTEM_PROMPT,
+        userMessage: lines,
+        maxTokens: 8192,
+        timeoutMs: 180_000,
+        tools: [INSIGHT_TOOL],
+        toolChoiceName: INSIGHT_TOOL.name,
+      });
+    } catch (err) {
+      log(
+        `insight-pass chunk=${chunkIdx + 1}/${chunks.length} threw=${err instanceof Error ? err.message : String(err)}`,
+      );
+      continue;
+    }
+
+    if (llmResult.kind === "fallback") {
+      log(
+        `insight-pass chunk=${chunkIdx + 1}/${chunks.length} fallback cause=${llmResult.cause} reason="${llmResult.reason}"`,
+      );
+      continue;
+    }
+    if (llmResult.kind !== "ok-tool") {
+      log(
+        `insight-pass chunk=${chunkIdx + 1}/${chunks.length} unexpected-result kind=${llmResult.kind}`,
+      );
+      continue;
+    }
+
+    const input = llmResult.input ?? {};
+    const observations = [];
+    for (const m of asArray(input.mentions)) {
+      observations.push({
+        kind: "mention",
+        summary: String(m.name ?? "").slice(0, 200),
+        snippet: String(m.snippet ?? "").slice(0, 200),
+        subject: null,
+        from: null,
+        to: null,
+      });
+    }
+    for (const t of asArray(input.tasks)) {
+      observations.push({
+        kind: "task",
+        summary: String(t.task ?? "").slice(0, 200),
+        snippet: String(t.snippet ?? "").slice(0, 200),
+        subject: null,
+        from: null,
+        to: null,
+      });
+    }
+    for (const p of asArray(input.preferences)) {
+      observations.push({
+        kind: "preference",
+        summary: String(p.preference ?? "").slice(0, 200),
+        snippet: null,
+        subject: String(p.subject ?? "").slice(0, 200),
+        from: null,
+        to: null,
+      });
+    }
+    for (const r of asArray(input.observedRelationships)) {
+      observations.push({
+        kind: "observed-relationship",
+        summary: String(r.relationship ?? "").slice(0, 200),
+        snippet: null,
+        subject: null,
+        from: String(r.from ?? "").slice(0, 200),
+        to: String(r.to ?? "").slice(0, 200),
+      });
+    }
+
+    counters.mentions += asArray(input.mentions).length;
+    counters.tasks += asArray(input.tasks).length;
+    counters.preferences += asArray(input.preferences).length;
+    counters.observedRelationships += asArray(input.observedRelationships).length;
+
+    if (observations.length === 0) continue;
+
+    try {
+      await session.executeWrite(async (tx) => {
+        await tx.run(INSIGHT_WRITE_CYPHER, {
+          conversationId,
+          accountId,
+          scope,
+          sessionId,
+          observations,
+        });
+      });
+    } catch (err) {
+      log(
+        `insight-pass chunk=${chunkIdx + 1}/${chunks.length} write-failed reason="${err instanceof Error ? err.message : String(err)}"`,
+      );
+    }
+  }
+
+  return counters;
+}
+
+function asArray(v) {
+  return Array.isArray(v) ? v : [];
+}
+
+// ---------------------------------------------------------------------------
+// 8. Main.
+// ---------------------------------------------------------------------------
+
+async function main() {
+  const startedMs = Date.now();
+  const { archive, flags } = parseArgv(process.argv);
+  const ownerElementId = flags.ownerElementId;
+  const scope = flags.scope;
+  const accountId = resolveAccountId(flags);
+  const timezone = flags.timezone || "Europe/London";
+  const dateFormat = flags.dateFormat;
+  const sessionId =
+    flags.sessionId || `whatsapp-ingest:${Date.now()}:${Math.random().toString(36).slice(2, 10)}`;
+
+  let parseExport, memoryArchiveWrite, getSession, callOauthLlm, HAIKU_MODEL;
+  try {
+    ({ parseExport } = await import(PARSE_EXPORT_PATH));
+    ({ memoryArchiveWrite } = await import(ARCHIVE_WRITE_PATH));
+    ({ getSession } = await import(NEO4J_LIB_PATH));
+    ({ callOauthLlm } = await import(OAUTH_LLM_PATH));
+    ({ HAIKU_MODEL } = await import(MODELS_PATH));
+  } catch (err) {
+    fail("import", {
+      reason: `failed to import compiled dist`,
+      detail: err instanceof Error ? err.message : String(err),
+    });
+  }
+
+  // 8a. Resolve _chat.txt
+  const { chatTxt, cleanup } = resolveChatTxt(archive);
+  const archiveBytes = statSync(chatTxt).size;
+  log(
+    `start file=${chatTxt} owner=${ownerElementId} scope=${scope} accountId=${accountId} archive-bytes=${archiveBytes}`,
+  );
+
+  // 8b. Parse
+  let parseResult;
+  const parseStart = Date.now();
+  try {
+    parseResult = parseExport({ filePath: chatTxt, accountId, timezone, dateFormat });
+  } catch (err) {
+    cleanup();
+    fail("parse", { reason: err instanceof Error ? err.message : String(err) });
+  }
+  const parseMs = Date.now() - parseStart;
+  const firstTs = parseResult.parsedLines[0]?.dateSent || "-";
+  const lastTs = parseResult.parsedLines[parseResult.parsedLines.length - 1]?.dateSent || "-";
+  log(
+    `parsed lines=${parseResult.counters.parsed} media-skipped=${parseResult.counters.mediaSkipped} system-skipped=${parseResult.counters.systemSkipped} first-line-ts=${firstTs} last-line-ts=${lastTs} ms=${parseMs}`,
+  );
+
+  // 8c. Auto-create participants and build rows
+  let session = getSession();
+  let participantIds = new Map();
+  const distinctSenderNames = Array.from(
+    new Set(parseResult.parsedLines.map((l) => l.senderName)),
+  );
+
+  try {
+    participantIds = await upsertAutoParticipants({
+      session,
+      accountId,
+      scope,
+      sessionId,
+      senderNames: distinctSenderNames,
+    });
+  } catch (err) {
+    await session.close().catch(() => {});
+    cleanup();
+    fail("archive-write", {
+      phase: "participant-upsert",
+      reason: err instanceof Error ? err.message : String(err),
+    });
+  }
+
+  if (participantIds.size !== distinctSenderNames.length) {
+    log(
+      `participant-upsert mismatch expected=${distinctSenderNames.length} got=${participantIds.size}`,
+    );
+  }
+
+  const conversationId = parseResult.conversationId;
+  const rows = parseResult.parsedLines.map((l, idx) => ({
+    messageId: `whatsapp-export:${conversationId}:${l.sequenceIndex}:${hashLine(l.body, l.dateSent)}`,
+    conversationId,
+    senderNodeId: participantIds.get(l.senderName) || ownerElementId,
+    senderName: l.senderName,
+    dateSent: l.dateSent,
+    body: l.body,
+    sequenceIndex: idx,
+  }));
+
+  const conversationBlock = {
+    conversationId,
+    archiveSourceFile: parseResult.archiveSourceFile,
+    firstMessageAt: firstTs,
+    lastMessageAt: lastTs,
+    participantCount: distinctSenderNames.length,
+    messageCount: rows.length,
+  };
+
+  // Close the session before archive-write — memoryArchiveWrite opens its
+  // own session and closes it in finally.
+  await session.close().catch(() => {});
+
+  // 8d. Archive-write
+  let archiveResult;
+  const writeStart = Date.now();
+  try {
+    archiveResult = await memoryArchiveWrite({
+      archiveType: "whatsapp-export",
+      ownerNodeId: ownerElementId,
+      accountId,
+      conversation: conversationBlock,
+      participantNodeIds: Array.from(participantIds.values()),
+      rows,
+      sessionId,
+      archiveFilePath: chatTxt,
+    });
+  } catch (err) {
+    cleanup();
+    fail("archive-write", {
+      reason: err instanceof Error ? err.message : String(err),
+    });
+  }
+  const writeMs = Date.now() - writeStart;
+  const createdMessages = archiveResult.counters.createdMessages || 0;
+  log(
+    `archive-write created-conversations=${archiveResult.counters.createdConversations || 0} created-messages=${createdMessages} created-sent-edges=${archiveResult.counters.createdSentEdges || 0} created-participant-edges=${archiveResult.counters.createdParticipantEdges || 0} ms=${writeMs}`,
+  );
+
+  // 8e. Insight pass
+  let insightCounters = {
+    chunks: 0,
+    mentions: 0,
+    tasks: 0,
+    preferences: 0,
+    observedRelationships: 0,
+  };
+  if (!flags.noInsight) {
+    const insightStart = Date.now();
+    let insightSession = getSession();
+    try {
+      insightCounters = await runInsightPass({
+        callOauthLlm,
+        HAIKU_MODEL,
+        session: insightSession,
+        conversationId,
+        accountId,
+        scope,
+        sessionId,
+        parsedLines: parseResult.parsedLines,
+      });
+    } catch (err) {
+      log(
+        `insight-pass FAILED reason="${err instanceof Error ? err.message : String(err)}"`,
+      );
+    } finally {
+      await insightSession.close().catch(() => {});
+    }
+    const insightMs = Date.now() - insightStart;
+    log(
+      `insight-pass model=haiku chunks=${insightCounters.chunks} mentions=${insightCounters.mentions} tasks=${insightCounters.tasks} preferences=${insightCounters.preferences} observed-relationships=${insightCounters.observedRelationships} ms=${insightMs}`,
+    );
+  }
+
+  cleanup();
+
+  const totalMs = Date.now() - startedMs;
+  log(`done conversationId=${conversationId} total-ms=${totalMs} exit=0`);
+
+  process.stdout.write(
+    JSON.stringify({
+      conversationId,
+      parsed: parseResult.counters.parsed,
+      mediaSkipped: parseResult.counters.mediaSkipped,
+      systemSkipped: parseResult.counters.systemSkipped,
+      createdMessages,
+      insightCounters,
+      ms: totalMs,
+    }) + "\n",
+  );
+  process.exit(0);
+}
+
+function hashLine(body, dateSent) {
+  // Stable per-line hash for messageId. The parser already hashed the file
+  // bytes for conversationId; per-message we hash (body+dateSent) to get a
+  // collision-resistant tail without crypto module overhead per line.
+  let h = 0;
+  const s = `${dateSent}${body}`;
+  for (let i = 0; i < s.length; i++) {
+    h = (h * 31 + s.charCodeAt(i)) | 0;
+  }
+  return (h >>> 0).toString(16).padStart(8, "0");
+}
+
+main().catch((err) => {
+  fail("uncaught", { reason: err instanceof Error ? err.message : String(err) });
+});