@rubytech/create-maxy 1.0.807 → 1.0.809

package/payload/platform/plugins/whatsapp-import/bin/ingest.mjs

@@ -1,37 +1,43 @@
 #!/usr/bin/env node
 // =============================================================================
-// ingest.mjs — in-process orchestrator for whatsapp-ingest.sh (Task 855).
+// ingest.mjs — in-process orchestrator for whatsapp-ingest.sh.
 //
-// Collapses parse → archive-write → insight into one Node process so the
-// 357K-char MCP-envelope ceiling no longer exists between steps. The
-// database-operator subagent's only handle on this pipeline is the wrapper
-// shell script; the gate at platform/plugins/admin/hooks/
-// archive-ingest-surface-gate.sh blocks the legacy MCP tools mechanically.
+// Phase 1 of the two-phase WhatsApp ingest contract (Task 855 / Task 871).
+// Deterministic only: parse → operator-supplied filter → archive-write
+// (Conversation + Messages + auto-Person participants + NEXT chronology).
+// No LLM in the per-message decision path. The Haiku insight pass moved to
+// Phase 2 (`mcp__memory__whatsapp-export-insight-pass`), invoked consciously
+// by the operator via the `whatsapp-import-enrich` skill.
 //
 // Argv (positional): <archive-path>
 // Argv (flags):       --owner-element-id <id> --scope <admin|public>
+//                     --filter <all|senders=<csv>|date-range=<isoFrom>..<isoTo>>
 //                     [--account-id <accountId>] [--timezone <iana>]
 //                     [--date-format <DD/MM/YY|MM/DD/YY|DD/MM/YYYY|MM/DD/YYYY>]
-//                     [--no-insight]
 //
-// Stdout (success): one JSON line
-//   {conversationId, parsed, mediaSkipped, systemSkipped,
-//    createdMessages, insightCounters, ms}
+// Stdout (success): one JSON line — Honest counters per Task 871.5.
+//   {conversationElementId, conversationId,
+//    parsed, mediaSkipped, systemSkipped,
+//    filtered,
+//    written, messagesAlreadyExisted,
+//    nextEdgesProcessed, nextEdgesCreated,
+//    participantsAlreadyExisted,
+//    ms}
+// The skill (`whatsapp-import` SKILL.md) maps this verbose-diagnostic shape
+// to the agent-return short shape per Task 871.6 (`alreadyExisted` etc.)
+// when surfacing the result to the admin agent.
 //
 // Stderr (failure): one [whatsapp-ingest] FAIL line, exit non-zero.
 // =============================================================================
 
 import {
-  createReadStream,
   existsSync,
   mkdtempSync,
   readdirSync,
-  readFileSync,
   rmSync,
   statSync,
-  writeFileSync,
 } from "node:fs";
-import { join, resolve, dirname, basename } from "node:path";
+import { join, resolve, dirname } from "node:path";
 import { tmpdir } from "node:os";
 import { spawnSync } from "node:child_process";
 import { fileURLToPath } from "node:url";
@@ -76,19 +82,14 @@ const NEO4J_LIB_PATH = resolve(
   "lib",
   "neo4j.js",
 );
-const OAUTH_LLM_PATH = resolve(
-  platformRoot,
-  "lib",
-  "oauth-llm",
-  "dist",
-  "index.js",
-);
-const MODELS_PATH = resolve(
+// Task 870: pure key-derivation functions ship in the whatsapp-import lib.
+const DERIVE_KEYS_PATH = resolve(
   platformRoot,
+  "plugins",
+  "whatsapp-import",
   "lib",
-  "models",
   "dist",
-  "index.js",
+  "derive-keys.js",
 );
 
 // ---------------------------------------------------------------------------
@@ -128,10 +129,6 @@ function parseArgv(argv) {
       continue;
     }
     const key = a.slice(2);
-    if (key === "no-insight") {
-      flags.noInsight = true;
-      continue;
-    }
     const v = args[++i];
     if (v == null) fail("argv", { reason: `flag --${key} requires a value` });
     flags[camelCase(key)] = v;
@@ -142,6 +139,15 @@ function parseArgv(argv) {
   if (flags.scope !== "admin" && flags.scope !== "public") {
     fail("argv", { reason: `invalid --scope "${flags.scope}" (admin|public)` });
   }
+  // Task 871: --filter is mandatory. The deterministic Bash entry refuses
+  // bulk archive writes without an operator-supplied filter — closes the
+  // doctrine gap named in feedback_compress_at_ingest_for_bulk_archives.md.
+  if (!flags.filter || !flags.filter.trim()) {
+    process.stderr.write(
+      `[whatsapp-ingest] FAIL filter-required reason="bulk-archive-gate (Task 871) — operator must specify --filter (one of all, senders=<csv>, date-range=<isoFrom>..<isoTo>)"\n`,
+    );
+    fail("argv", { reason: "--filter is required (one of all, senders=<csv>, date-range=<isoFrom>..<isoTo>)" });
+  }
   return { archive, flags };
 }
 
@@ -241,11 +247,31 @@ function resolveAccountId(flags) {
 }
 
 // ---------------------------------------------------------------------------
-// 6. Auto-create participants — one :Person node per distinct senderName
-//    minus the owner. Provenance + participantStatus='auto-created' so a
-//    later semantic-enrichment pass (Task 856) can promote/disambiguate.
+// 6. Resolve participants — match each sender against canonical
+//    (:Person|:AdminUser {accountId}) by NFKC-trim-lower normalisation
+//    before falling through to auto-Person creation (Task 870). Per-sender
+//    `participant-resolved` log line so re-imports surface canonical reuse.
+//    Auto-Person fallback retains the legacy `{accountId, source:'whatsapp',
+//    name}` shape; switching to givenName/familyName is Task 874.
 // ---------------------------------------------------------------------------
 
+const CANONICAL_FETCH_CYPHER = `
+MATCH (p:Person {accountId: $accountId})
+WHERE coalesce(p.participantStatus, '') <> 'auto-created'
+RETURN elementId(p) AS elemId,
+       coalesce(p.givenName, '') AS givenName,
+       coalesce(p.familyName, '') AS familyName,
+       NULL AS adminName,
+       'Person' AS label
+UNION
+MATCH (u:AdminUser {accountId: $accountId})
+RETURN elementId(u) AS elemId,
+       '' AS givenName,
+       '' AS familyName,
+       coalesce(u.name, '') AS adminName,
+       'AdminUser' AS label
+`;
+
 const PARTICIPANT_UPSERT_CYPHER = `
 UNWIND $names AS senderName
 MERGE (p:Person {accountId: $accountId, source: 'whatsapp', name: senderName})
@@ -259,278 +285,109 @@ MERGE (p:Person {accountId: $accountId, source: 'whatsapp', name: senderName})
 RETURN elementId(p) AS elemId, senderName AS name
 `;
 
-async function upsertAutoParticipants({
-  session,
-  accountId,
-  scope,
-  sessionId,
-  senderNames,
-}) {
-  if (senderNames.length === 0) return new Map();
-  // Owner-as-sender reconciliation is deferred to Task 856 — when the
-  // owner's display name appears as a sender, the messages SENT-edge from
-  // the auto-created :Person, not from the operator's :AdminUser. The
-  // post-load enrichment skill rewrites those edges on operator confirm.
-  return await session.executeWrite(async (tx) => {
-    const res = await tx.run(PARTICIPANT_UPSERT_CYPHER, {
-      names: senderNames,
-      accountId,
-      scope,
-      sessionId,
-    });
-    const map = new Map();
-    for (const r of res.records) {
-      map.set(r.get("name"), r.get("elemId"));
+async function fetchCanonicalIndex({ session, accountId, normaliseSenderName }) {
+  // Returns Map<normalisedName, {elemId, label}> for canonical participants.
+  // Excludes auto-created Persons so we never reuse our own shadow nodes.
+  const res = await session.executeRead(async (tx) =>
+    tx.run(CANONICAL_FETCH_CYPHER, { accountId }),
+  );
+  const index = new Map();
+  for (const r of res.records) {
+    const elemId = r.get("elemId");
+    const label = r.get("label");
+    const candidates = [];
+    if (label === "Person") {
+      const given = r.get("givenName") || "";
+      const family = r.get("familyName") || "";
+      if (given || family) candidates.push(`${given} ${family}`.trim());
+      if (given) candidates.push(given);
+      if (family) candidates.push(family);
+    } else {
+      const adminName = r.get("adminName") || "";
+      if (adminName) candidates.push(adminName);
     }
-    return map;
-  });
+    for (const c of candidates) {
+      const norm = normaliseSenderName(c);
+      if (!norm) continue;
+      // First write wins — Person before AdminUser per UNION order.
+      if (!index.has(norm)) index.set(norm, { elemId, label });
+    }
+  }
+  return index;
 }
 
-// ---------------------------------------------------------------------------
-// 7. Insight pass — chunked Haiku call, in-process Cypher writes.
-//    One :Observation node per extracted item, OBSERVED_IN edge to the
-//    Conversation. Edge wiring to specific :Person/:Task/etc. is Task 856.
-// ---------------------------------------------------------------------------
-
-const INSIGHT_CHUNK_SIZE = 1500; // messages per Haiku call (≈75K tokens budget)
-
-const INSIGHT_SYSTEM_PROMPT = `You extract structured insights from a chunk of a WhatsApp conversation.
-
-Return STRICT JSON via the provided tool. No prose, no commentary. Only items with concrete, verbatim evidence in the chunk. Empty arrays are valid; prefer omission to invention.
-
-Definitions:
-- "mention": a person, organisation, place, or named topic referred to by name.
-- "task": something a participant committed to do or asked another to do (imperative or future-tense).
-- "preference": stated like, dislike, opinion, or rule of behaviour.
-- "observedRelationship": an explicit relational claim (works at, is married to, manages, etc.).
-
-Snippets must be ≤80 characters of the original message body, no sender names, no timestamps.`;
-
-const INSIGHT_TOOL = {
-  name: "submit_insights",
-  description: "Submit the structured insights extracted from the chunk.",
-  input_schema: {
-    type: "object",
-    properties: {
-      mentions: {
-        type: "array",
-        items: {
-          type: "object",
-          properties: {
-            name: { type: "string" },
-            snippet: { type: "string" },
-          },
-          required: ["name", "snippet"],
-        },
-      },
-      tasks: {
-        type: "array",
-        items: {
-          type: "object",
-          properties: {
-            task: { type: "string" },
-            snippet: { type: "string" },
-          },
-          required: ["task", "snippet"],
-        },
-      },
-      preferences: {
-        type: "array",
-        items: {
-          type: "object",
-          properties: {
-            subject: { type: "string" },
-            preference: { type: "string" },
-          },
-          required: ["subject", "preference"],
-        },
-      },
-      observedRelationships: {
-        type: "array",
-        items: {
-          type: "object",
-          properties: {
-            from: { type: "string" },
-            to: { type: "string" },
-            relationship: { type: "string" },
-          },
-          required: ["from", "to", "relationship"],
-        },
-      },
-    },
-    required: ["mentions", "tasks", "preferences", "observedRelationships"],
-  },
-};
-
-const INSIGHT_WRITE_CYPHER = `
-MATCH (c:Conversation:WhatsAppConversation {conversationId: $conversationId})
-UNWIND $observations AS obs
-CREATE (o:Observation)
-SET
-  o:WhatsAppObservation,
-  o.accountId        = $accountId,
-  o.kind             = obs.kind,
-  o.summary          = obs.summary,
-  o.snippet          = obs.snippet,
-  o.subject          = obs.subject,
-  o.from             = obs.from,
-  o.to               = obs.to,
-  o.source           = 'whatsapp',
-  o.createdByAgent   = 'whatsapp-import',
-  o.createdBySource  = 'whatsapp-import',
-  o.createdBySession = $sessionId,
-  o.createdAt        = datetime(),
-  o.scope            = $scope,
-  o.insightPass      = true,
-  o.observationStatus = 'auto-extracted'
-MERGE (o)-[r:OBSERVED_IN]->(c)
-  ON CREATE SET r.source = 'whatsapp', r.createdAt = datetime()
-RETURN count(o) AS created
-`;
-
-async function runInsightPass({
-  callOauthLlm,
-  HAIKU_MODEL,
+async function resolveParticipants({
   session,
-  conversationId,
   accountId,
   scope,
   sessionId,
-  parsedLines,
+  senderNames,
+  normaliseSenderName,
 }) {
-  const counters = {
-    chunks: 0,
-    mentions: 0,
-    tasks: 0,
-    preferences: 0,
-    observedRelationships: 0,
-  };
-
-  if (parsedLines.length === 0) return counters;
-
-  // Build chunks. Each chunk is rendered as a numbered transcript including
-  // sender-names — the LLM needs them to attribute tasks/preferences/observed-
-  // relationships. The system prompt's "no sender names" rule constrains the
-  // returned snippet field only, not the input transcript.
-  const chunks = [];
-  for (let i = 0; i < parsedLines.length; i += INSIGHT_CHUNK_SIZE) {
-    chunks.push(parsedLines.slice(i, i + INSIGHT_CHUNK_SIZE));
+  if (senderNames.length === 0) {
+    return { idsByName: new Map(), participantsAlreadyExisted: 0 };
   }
-  counters.chunks = chunks.length;
-
-  for (let chunkIdx = 0; chunkIdx < chunks.length; chunkIdx++) {
-    const chunk = chunks[chunkIdx];
-    const lines = chunk
-      .map((l, j) => `[${j + 1}] ${l.senderName}: ${l.body}`)
-      .join("\n");
-
-    let llmResult;
-    try {
-      llmResult = await callOauthLlm({
-        model: HAIKU_MODEL,
-        system: INSIGHT_SYSTEM_PROMPT,
-        userMessage: lines,
-        maxTokens: 8192,
-        timeoutMs: 180_000,
-        tools: [INSIGHT_TOOL],
-        toolChoiceName: INSIGHT_TOOL.name,
-      });
-    } catch (err) {
-      log(
-        `insight-pass chunk=${chunkIdx + 1}/${chunks.length} threw=${err instanceof Error ? err.message : String(err)}`,
-      );
-      continue;
-    }
 
-    if (llmResult.kind === "fallback") {
-      log(
-        `insight-pass chunk=${chunkIdx + 1}/${chunks.length} fallback cause=${llmResult.cause} reason="${llmResult.reason}"`,
-      );
-      continue;
-    }
-    if (llmResult.kind !== "ok-tool") {
+  const canonicalIndex = await fetchCanonicalIndex({
+    session,
+    accountId,
+    normaliseSenderName,
+  });
+
+  const idsByName = new Map();
+  const fallbackSenders = [];
+  let canonicalMatches = 0;
+  for (const senderName of senderNames) {
+    const norm = normaliseSenderName(senderName);
+    const hit = canonicalIndex.get(norm);
+    if (hit) {
+      idsByName.set(senderName, hit.elemId);
+      canonicalMatches++;
       log(
-        `insight-pass chunk=${chunkIdx + 1}/${chunks.length} unexpected-result kind=${llmResult.kind}`,
+        `participant-resolved senderName="${senderName}" matched=canonical nodeId=${hit.elemId} label=${hit.label}`,
       );
-      continue;
-    }
-
-    const input = llmResult.input ?? {};
-    const observations = [];
-    for (const m of asArray(input.mentions)) {
-      observations.push({
-        kind: "mention",
-        summary: String(m.name ?? "").slice(0, 200),
-        snippet: String(m.snippet ?? "").slice(0, 200),
-        subject: null,
-        from: null,
-        to: null,
-      });
-    }
-    for (const t of asArray(input.tasks)) {
-      observations.push({
-        kind: "task",
-        summary: String(t.task ?? "").slice(0, 200),
-        snippet: String(t.snippet ?? "").slice(0, 200),
-        subject: null,
-        from: null,
-        to: null,
-      });
-    }
-    for (const p of asArray(input.preferences)) {
-      observations.push({
-        kind: "preference",
-        summary: String(p.preference ?? "").slice(0, 200),
-        snippet: null,
-        subject: String(p.subject ?? "").slice(0, 200),
-        from: null,
-        to: null,
-      });
-    }
-    for (const r of asArray(input.observedRelationships)) {
-      observations.push({
-        kind: "observed-relationship",
-        summary: String(r.relationship ?? "").slice(0, 200),
-        snippet: null,
-        subject: null,
-        from: String(r.from ?? "").slice(0, 200),
-        to: String(r.to ?? "").slice(0, 200),
-      });
+    } else {
+      fallbackSenders.push(senderName);
     }
+  }
 
-    counters.mentions += asArray(input.mentions).length;
-    counters.tasks += asArray(input.tasks).length;
-    counters.preferences += asArray(input.preferences).length;
-    counters.observedRelationships += asArray(input.observedRelationships).length;
-
-    if (observations.length === 0) continue;
-
-    try {
-      await session.executeWrite(async (tx) => {
-        await tx.run(INSIGHT_WRITE_CYPHER, {
-          conversationId,
-          accountId,
-          scope,
-          sessionId,
-          observations,
-        });
+  let autoCreated = 0;
+  if (fallbackSenders.length > 0) {
+    const result = await session.executeWrite(async (tx) => {
+      const res = await tx.run(PARTICIPANT_UPSERT_CYPHER, {
+        names: fallbackSenders,
+        accountId,
+        scope,
+        sessionId,
       });
-    } catch (err) {
+      const m = new Map();
+      for (const r of res.records) {
+        m.set(r.get("name"), r.get("elemId"));
+      }
+      const stats = res.summary.counters.updates();
+      return { m, created: stats.nodesCreated };
+    });
+    autoCreated = result.created;
+    for (const senderName of fallbackSenders) {
+      const elemId = result.m.get(senderName);
+      if (!elemId) continue;
+      idsByName.set(senderName, elemId);
       log(
-        `insight-pass chunk=${chunkIdx + 1}/${chunks.length} write-failed reason="${err instanceof Error ? err.message : String(err)}"`,
+        `participant-resolved senderName="${senderName}" matched=auto nodeId=${elemId} label=Person`,
       );
     }
   }
 
-  return counters;
-}
-
-function asArray(v) {
-  return Array.isArray(v) ? v : [];
+  // participantsAlreadyExisted = canonical hits + auto-Persons that were already in graph.
+  const autoAlreadyExisted = fallbackSenders.length - autoCreated;
+  return {
+    idsByName,
+    participantsAlreadyExisted: canonicalMatches + autoAlreadyExisted,
+  };
 }
 
 // ---------------------------------------------------------------------------
-// 8. Main.
+// 7. Main.
 // ---------------------------------------------------------------------------
 
 async function main() {
@@ -544,13 +401,13 @@ async function main() {
   const sessionId =
     flags.sessionId || `whatsapp-ingest:${Date.now()}:${Math.random().toString(36).slice(2, 10)}`;
 
-  let parseExport, memoryArchiveWrite, getSession, callOauthLlm, HAIKU_MODEL;
+  let parseExport, parseFilterArg, applyFilter, memoryArchiveWrite, getSession;
+  let normaliseSenderName, deriveMessageId;
   try {
-    ({ parseExport } = await import(PARSE_EXPORT_PATH));
+    ({ parseExport, parseFilterArg, applyFilter } = await import(PARSE_EXPORT_PATH));
     ({ memoryArchiveWrite } = await import(ARCHIVE_WRITE_PATH));
     ({ getSession } = await import(NEO4J_LIB_PATH));
-    ({ callOauthLlm } = await import(OAUTH_LLM_PATH));
-    ({ HAIKU_MODEL } = await import(MODELS_PATH));
+    ({ normaliseSenderName, deriveMessageId } = await import(DERIVE_KEYS_PATH));
   } catch (err) {
     fail("import", {
       reason: `failed to import compiled dist`,
@@ -558,14 +415,23 @@ async function main() {
     });
   }
 
-  // 8a. Resolve _chat.txt
+  // 7a. Parse the operator-supplied filter ahead of any IO so a malformed
+  // value LOUD-FAILs before unzip / Neo4j connection.
+  let filter;
+  try {
+    filter = parseFilterArg(flags.filter);
+  } catch (err) {
+    fail("argv", { reason: err instanceof Error ? err.message : String(err) });
+  }
+
+  // 7b. Resolve _chat.txt
   const { chatTxt, cleanup } = resolveChatTxt(archive);
   const archiveBytes = statSync(chatTxt).size;
   log(
-    `start file=${chatTxt} owner=${ownerElementId} scope=${scope} accountId=${accountId} archive-bytes=${archiveBytes}`,
+    `start file=${chatTxt} owner=${ownerElementId} scope=${scope} accountId=${accountId} archive-bytes=${archiveBytes} filter=${flags.filter}`,
   );
 
-  // 8b. Parse
+  // 7c. Parse
   let parseResult;
   const parseStart = Date.now();
   try {
@@ -575,51 +441,84 @@ async function main() {
     fail("parse", { reason: err instanceof Error ? err.message : String(err) });
   }
   const parseMs = Date.now() - parseStart;
-  const firstTs = parseResult.parsedLines[0]?.dateSent || "-";
-  const lastTs = parseResult.parsedLines[parseResult.parsedLines.length - 1]?.dateSent || "-";
   log(
-    `parsed lines=${parseResult.counters.parsed} media-skipped=${parseResult.counters.mediaSkipped} system-skipped=${parseResult.counters.systemSkipped} first-line-ts=${firstTs} last-line-ts=${lastTs} ms=${parseMs}`,
+    `parsed lines=${parseResult.counters.parsed} media-skipped=${parseResult.counters.mediaSkipped} system-skipped=${parseResult.counters.systemSkipped} ms=${parseMs}`,
+  );
+
+  // 7d. Apply filter
+  const filteredLines = applyFilter(parseResult.parsedLines, filter);
+  log(
+    `filter-applied kind=${filter.kind} parsed=${parseResult.parsedLines.length} filtered=${filteredLines.length}`,
   );
+  if (filteredLines.length === 0) {
+    cleanup();
+    fail("filter", {
+      reason: `filter "${flags.filter}" matched zero rows from ${parseResult.parsedLines.length} parsed lines — refusing to write an empty conversation`,
+    });
+  }
+
+  const firstTs = filteredLines[0].dateSent;
+  const lastTs = filteredLines[filteredLines.length - 1].dateSent;
 
-  // 8c. Auto-create participants and build rows
+  // 7e. Auto-create participants and build rows
   let session = getSession();
-  let participantIds = new Map();
+  let participantUpsert = { idsByName: new Map(), participantsAlreadyExisted: 0 };
   const distinctSenderNames = Array.from(
-    new Set(parseResult.parsedLines.map((l) => l.senderName)),
+    new Set(filteredLines.map((l) => l.senderName)),
   );
 
   try {
-    participantIds = await upsertAutoParticipants({
+    participantUpsert = await resolveParticipants({
       session,
       accountId,
       scope,
       sessionId,
       senderNames: distinctSenderNames,
+      normaliseSenderName,
     });
   } catch (err) {
     await session.close().catch(() => {});
     cleanup();
     fail("archive-write", {
-      phase: "participant-upsert",
+      phase: "participant-resolve",
       reason: err instanceof Error ? err.message : String(err),
     });
   }
 
+  const participantIds = participantUpsert.idsByName;
+  const participantsAlreadyExisted = participantUpsert.participantsAlreadyExisted;
+
   if (participantIds.size !== distinctSenderNames.length) {
     log(
-      `participant-upsert mismatch expected=${distinctSenderNames.length} got=${participantIds.size}`,
+      `participant-resolve mismatch expected=${distinctSenderNames.length} got=${participantIds.size}`,
     );
   }
 
   const conversationId = parseResult.conversationId;
-  const rows = parseResult.parsedLines.map((l, idx) => ({
-    messageId: `whatsapp-export:${conversationId}:${l.sequenceIndex}:${hashLine(l.body, l.dateSent)}`,
+  // Task 870: messageId derives from a content-derived natural key. Stable
+  // across re-imports — same archive bytes + same operator timezone produce
+  // identical messageIds, so the existing MERGE in memory-archive-write is
+  // naturally idempotent.
+  const conversationSha256 = parseResult.archiveSourceFile.replace(
+    /^whatsapp-export:/,
+    "",
+  );
+  const rows = filteredLines.map((l) => ({
+    messageId: deriveMessageId({
+      conversationSha256,
+      dateSent: l.dateSent,
+      senderName: l.senderName,
+      body: l.body,
+    }),
     conversationId,
     senderNodeId: participantIds.get(l.senderName) || ownerElementId,
     senderName: l.senderName,
     dateSent: l.dateSent,
     body: l.body,
-    sequenceIndex: idx,
+    // sequenceIndex preserved on the Message node for chain ordering tiebreaker
+    // in WHATSAPP_NEXT_CHAIN_CYPHER. NOT part of messageId — that would couple
+    // identity to array position and re-introduce shadowing.
+    sequenceIndex: l.sequenceIndex,
   }));
 
   const conversationBlock = {
@@ -635,7 +534,7 @@ async function main() {
   // own session and closes it in finally.
   await session.close().catch(() => {});
 
-  // 8d. Archive-write
+  // 7f. Archive-write
   let archiveResult;
   const writeStart = Date.now();
   try {
@@ -657,43 +556,29 @@ async function main() {
   }
   const writeMs = Date.now() - writeStart;
   const createdMessages = archiveResult.counters.createdMessages || 0;
+  const nextEdgesProcessed = archiveResult.counters.nextEdgesProcessed || 0;
+  const nextEdgesCreated = archiveResult.counters.nextEdgesCreated || 0;
+  const messagesAlreadyExisted = rows.length - createdMessages;
   log(
-    `archive-write created-conversations=${archiveResult.counters.createdConversations || 0} created-messages=${createdMessages} created-sent-edges=${archiveResult.counters.createdSentEdges || 0} created-participant-edges=${archiveResult.counters.createdParticipantEdges || 0} ms=${writeMs}`,
+    `archive-write created-messages=${createdMessages} messages-already-existed=${messagesAlreadyExisted} next-edges-processed=${nextEdgesProcessed} next-edges-created=${nextEdgesCreated} participants-already-existed=${participantsAlreadyExisted} ms=${writeMs}`,
   );
 
-  // 8e. Insight pass
-  let insightCounters = {
-    chunks: 0,
-    mentions: 0,
-    tasks: 0,
-    preferences: 0,
-    observedRelationships: 0,
-  };
-  if (!flags.noInsight) {
-    const insightStart = Date.now();
-    let insightSession = getSession();
-    try {
-      insightCounters = await runInsightPass({
-        callOauthLlm,
-        HAIKU_MODEL,
-        session: insightSession,
-        conversationId,
-        accountId,
-        scope,
-        sessionId,
-        parsedLines: parseResult.parsedLines,
-      });
-    } catch (err) {
-      log(
-        `insight-pass FAILED reason="${err instanceof Error ? err.message : String(err)}"`,
-      );
-    } finally {
-      await insightSession.close().catch(() => {});
-    }
-    const insightMs = Date.now() - insightStart;
-    log(
-      `insight-pass model=haiku chunks=${insightCounters.chunks} mentions=${insightCounters.mentions} tasks=${insightCounters.tasks} preferences=${insightCounters.preferences} observed-relationships=${insightCounters.observedRelationships} ms=${insightMs}`,
+  // 7g. Resolve conversationElementId for the agent-return shape (Task 871.6).
+  // One small read-after-write — the operator's database-operator subagent
+  // surfaces this as the canonical handle for downstream Phase 2 calls.
+  let conversationElementId = null;
+  const ridSession = getSession();
+  try {
+    const res = await ridSession.run(
+      `MATCH (c:Conversation:WhatsAppConversation {conversationId: $cid})
+       RETURN elementId(c) AS elemId LIMIT 1`,
+      { cid: conversationId },
     );
+    conversationElementId = res.records[0]?.get("elemId") ?? null;
+  } catch (err) {
+    log(`conversation-elementid-lookup failed reason="${err instanceof Error ? err.message : String(err)}"`);
+  } finally {
+    await ridSession.close().catch(() => {});
   }
 
   cleanup();
@@ -703,29 +588,23 @@ async function main() {
 
   process.stdout.write(
     JSON.stringify({
+      conversationElementId,
       conversationId,
       parsed: parseResult.counters.parsed,
       mediaSkipped: parseResult.counters.mediaSkipped,
       systemSkipped: parseResult.counters.systemSkipped,
-      createdMessages,
-      insightCounters,
+      filtered: filteredLines.length,
+      written: createdMessages,
+      messagesAlreadyExisted,
+      nextEdgesProcessed,
+      nextEdgesCreated,
+      participantsAlreadyExisted,
       ms: totalMs,
     }) + "\n",
   );
   process.exit(0);
 }
 
-function hashLine(body, dateSent) {
-  // Stable per-line hash for messageId. The parser already hashed the file
-  // bytes for conversationId; per-message we hash (body+dateSent) to get a
-  // collision-resistant tail without crypto module overhead per line.
-  let h = 0;
-  const s = `${dateSent}${body}`;
-  for (let i = 0; i < s.length; i++) {
-    h = (h * 31 + s.charCodeAt(i)) | 0;
-  }
-  return (h >>> 0).toString(16).padStart(8, "0");
-}
 
 main().catch((err) => {
   fail("uncaught", { reason: err instanceof Error ? err.message : String(err) });