@rubytech/create-realagent 1.0.825 → 1.0.828

package/payload/platform/plugins/whatsapp-import/bin/ingest.mjs

@@ -1,52 +1,38 @@
 #!/usr/bin/env node
 // =============================================================================
-// ingest.mjs — in-process orchestrator for whatsapp-ingest.sh.
+// ingest.mjs — in-process orchestrator for whatsapp-ingest.sh (Task 891).
 //
-// Phase 1 of the two-phase WhatsApp ingest contract (Task 855 / Task 871).
-// Deterministic only: parse → operator-supplied filter → archive-write
-// (Conversation + Messages + auto-Person participants + NEXT chronology).
-// No LLM in the per-message decision path. The Haiku insight pass moved to
-// Phase 2 (`mcp__memory__whatsapp-export-insight-pass`), invoked consciously
-// by the operator via the `whatsapp-import-enrich` skill.
+// Pipeline (single phase — Phase 2 insight derivation deferred to its own
+// follow-up task):
+//
+//   parse  → bind canonical senders → derive conversationIdentity
+//   → look up prior :ConversationArchive (delta cursor)
+//   → sessionize delta at gapHours boundaries
+//   → for each session: classify (mode='chat') → collect chunks
+//   → memoryIngest(parentLabel='ConversationArchive')
 //
 // Argv (positional): <archive-path>
-// Argv (flags):       --owner-element-id <id> --scope <admin|public>
-//                     --subject-person-id <id>
-//                     --filter <all|senders=<csv>|date-range=<isoFrom>..<isoTo>>
-//                     [--account-id <accountId>] [--timezone <iana>]
+// Argv (flags):       --owner-element-id <id>
+//                     --participant-person-ids <csv>
+//                     --scope <admin|public>
+//                     [--session-gap-hours <N>]   (default 12)
+//                     [--account-id <accountId>]
+//                     [--timezone <iana>]
 //                     [--date-format <DD/MM/YY|MM/DD/YY|DD/MM/YYYY|MM/DD/YYYY>]
 //
-// Task 887 §A0 — `--subject-person-id` is required. DM scope: the operator
-// confirms the third party's :Person elementId from the preview histogram
-// before invoking. The writer accepts EXACTLY the senderNames carried by
-// {owner, subject}; any parsed senderName outside that set LOUD-FAILs with
-// `parser-miss reason="senderName=<verbatim> not in preview histogram
-// (parser failure — re-export or report)"`. Bounds writer cardinality to
-// the deterministic preview output — closes the auto-Person leak structurally.
-//
-// Stdout (success): one JSON line — Honest counters per Task 871.5.
-//   {conversationElementId, conversationId,
-//    parsed, mediaSkipped, systemSkipped,
-//    filtered,
-//    written, messagesAlreadyExisted,
-//    nextEdgesProcessed, nextEdgesCreated,
-//    participantsAlreadyExisted,
-//    ms}
-// The skill (`whatsapp-import` SKILL.md) maps this verbose-diagnostic shape
-// to the agent-return short shape per Task 871.6 (`alreadyExisted` etc.)
-// when surfacing the result to the admin agent.
+// Owner + participants form the closed sender set; any parsed senderName
+// outside that set LOUD-FAILs with `parser-miss` (preserves Task 887 §A0).
+// `--subject-person-id` and `--filter` are gone (the chunked archive shape
+// makes per-message filtering obsolete — chunking bounds operator surface).
 //
-// Stderr (failure): one [whatsapp-ingest] FAIL line, exit non-zero.
+// Stdout (success): one JSON line — all the counters the skill needs to
+// formulate the three operator-facing messages. See SKILL.md for the shape.
+// Stderr: one [whatsapp-import] FAIL line on failure, exit non-zero.
 // =============================================================================
 
-import {
-  existsSync,
-  mkdtempSync,
-  readdirSync,
-  rmSync,
-  statSync,
-} from "node:fs";
-import { join, resolve, dirname } from "node:path";
+import { existsSync, mkdtempSync, readdirSync, rmSync, statSync, createReadStream } from "node:fs";
+import { createHash } from "node:crypto";
+import { join, resolve, dirname, basename } from "node:path";
 import { tmpdir } from "node:os";
 import { spawnSync } from "node:child_process";
 import { fileURLToPath } from "node:url";
@@ -54,13 +40,8 @@ import { fileURLToPath } from "node:url";
 const __dirname = dirname(fileURLToPath(import.meta.url));
 
 // ---------------------------------------------------------------------------
-// 1. Resolve dist paths from the platform install root.
+// 1. Resolve dist paths.
 // ---------------------------------------------------------------------------
-// MAXY_PLATFORM_ROOT is set by the wrapper script (and by claude-agent.ts in
-// production). Fall back to walking up from this file's location: the bin/
-// directory sits at platform/plugins/whatsapp-import/bin/, so platform/ is
-// three levels up.
-
 const platformRoot =
   process.env.MAXY_PLATFORM_ROOT?.trim() ||
   resolve(__dirname, "..", "..", "..");
@@ -73,44 +54,41 @@ const PARSE_EXPORT_PATH = resolve(
   "dist",
   "index.js",
 );
-const ARCHIVE_WRITE_PATH = resolve(
+const NEO4J_LIB_PATH = resolve(
   platformRoot,
   "plugins",
   "memory",
   "mcp",
   "dist",
-  "tools",
-  "memory-archive-write.js",
+  "lib",
+  "neo4j.js",
 );
-const NEO4J_LIB_PATH = resolve(
+const LLM_CLASSIFIER_PATH = resolve(
   platformRoot,
   "plugins",
   "memory",
   "mcp",
   "dist",
   "lib",
-  "neo4j.js",
+  "llm-classifier.js",
 );
-// Task 870: pure key-derivation functions ship in the whatsapp-import lib.
-const DERIVE_KEYS_PATH = resolve(
+const MEMORY_INGEST_PATH = resolve(
   platformRoot,
   "plugins",
-  "whatsapp-import",
-  "lib",
+  "memory",
+  "mcp",
   "dist",
-  "derive-keys.js",
+  "tools",
+  "memory-ingest.js",
 );
 
 // ---------------------------------------------------------------------------
-// 2. Logger — every line `[whatsapp-ingest]`-prefixed to stderr (server.log).
+// 2. Logger
 // ---------------------------------------------------------------------------
-
 function log(line) {
-  process.stderr.write(`[whatsapp-ingest] ${line}\n`);
+  process.stderr.write(`[whatsapp-import] ${line}\n`);
 }
-
 function fail(phase, fields) {
-  // Single failure line. Used as the only stderr line on non-zero exit.
   const fieldStr = Object.entries(fields)
     .map(([k, v]) =>
       typeof v === "string" && (v.includes(" ") || v.includes("="))
@@ -118,14 +96,13 @@ function fail(phase, fields) {
         : `${k}=${v ?? "-"}`,
     )
     .join(" ");
-  process.stderr.write(`[whatsapp-ingest] FAIL phase=${phase} ${fieldStr}\n`);
+  process.stderr.write(`[whatsapp-import] FAIL phase=${phase} ${fieldStr}\n`);
   process.exit(1);
 }
 
 // ---------------------------------------------------------------------------
-// 3. Argv parsing.
+// 3. Argv parsing
 // ---------------------------------------------------------------------------
-
 function parseArgv(argv) {
   const args = argv.slice(2);
   let archive = null;
@@ -144,52 +121,33 @@ function parseArgv(argv) {
   }
   if (!archive) fail("argv", { reason: "missing positional <archive>" });
   if (!flags.ownerElementId) fail("argv", { reason: "missing --owner-element-id" });
+  if (!flags.participantPersonIds) {
+    fail("argv", {
+      reason: "missing --participant-person-ids (csv of operator-confirmed :Person/:AdminUser elementIds, owner excluded)",
+    });
+  }
   if (!flags.scope) fail("argv", { reason: "missing --scope" });
   if (flags.scope !== "admin" && flags.scope !== "public") {
     fail("argv", { reason: `invalid --scope "${flags.scope}" (admin|public)` });
   }
-  // Task 887 §A0 — DM scope: a single `--subject-person-id` identifies the
-  // third party in the conversation. The owner + subject pair is the
-  // canonical sender set; the writer rejects any other senderName as
-  // parser-miss. Group-chat ingest (>2 distinct senders) is a future task.
-  if (!flags.subjectPersonId) {
-    fail("argv", { reason: "missing --subject-person-id (Task 887: operator-confirmed third-party :Person elementId from preview histogram)" });
-  }
-  // Task 871: --filter is mandatory. The deterministic Bash entry refuses
-  // bulk archive writes without an operator-supplied filter — closes the
-  // doctrine gap named in feedback_compress_at_ingest_for_bulk_archives.md.
-  if (!flags.filter || !flags.filter.trim()) {
-    process.stderr.write(
-      `[whatsapp-ingest] FAIL filter-required reason="bulk-archive-gate (Task 871) — operator must specify --filter (one of all, senders=<csv>, date-range=<isoFrom>..<isoTo>)"\n`,
-    );
-    fail("argv", { reason: "--filter is required (one of all, senders=<csv>, date-range=<isoFrom>..<isoTo>)" });
-  }
   return { archive, flags };
 }
-
 function camelCase(s) {
   return s.replace(/-([a-z])/g, (_m, c) => c.toUpperCase());
 }
 
 // ---------------------------------------------------------------------------
-// 4. Archive resolution — accept .zip / dir / _chat.txt; return absolute
-//    path to _chat.txt and a cleanup callback for any tmp dir we created.
+// 4. Archive resolution
 // ---------------------------------------------------------------------------
-
 function resolveChatTxt(archivePath) {
   const abs = resolve(archivePath);
-  if (!existsSync(abs)) {
-    fail("argv", { reason: `archive path not found: ${abs}` });
-  }
+  if (!existsSync(abs)) fail("argv", { reason: `archive path not found: ${abs}` });
   const st = statSync(abs);
-
   if (st.isFile() && abs.endsWith(".zip")) {
     const tmp = mkdtempSync(join(tmpdir(), "whatsapp-ingest-"));
-    const unzip = spawnSync("unzip", ["-q", "-o", abs, "-d", tmp], {
-      encoding: "utf8",
-    });
+    const unzip = spawnSync("unzip", ["-q", "-o", abs, "-d", tmp], { encoding: "utf8" });
     if (unzip.status !== 0) {
-      rmSync(tmp, { recursive: true, force: true });
+      rmSync(tmp, { recursive: true });
       fail("argv", {
         reason: "unzip failed",
         archive: abs,
@@ -198,27 +156,22 @@ function resolveChatTxt(archivePath) {
     }
     const chat = findChatTxt(tmp);
     if (!chat) {
-      rmSync(tmp, { recursive: true, force: true });
+      rmSync(tmp, { recursive: true });
       fail("argv", { reason: "_chat.txt not found in zip", archive: abs });
     }
-    return { chatTxt: chat, cleanup: () => rmSync(tmp, { recursive: true, force: true }) };
+    return { chatTxt: chat, cleanup: () => rmSync(tmp, { recursive: true }) };
   }
-
   if (st.isDirectory()) {
     const chat = findChatTxt(abs);
     if (!chat) fail("argv", { reason: "_chat.txt not found in directory", archive: abs });
     return { chatTxt: chat, cleanup: () => {} };
   }
-
   if (st.isFile()) {
     return { chatTxt: abs, cleanup: () => {} };
   }
-
   fail("argv", { reason: `unsupported archive shape: ${abs}` });
-  // unreachable, but keeps tools happy
   return { chatTxt: abs, cleanup: () => {} };
 }
-
 function findChatTxt(dir) {
   const entries = readdirSync(dir, { withFileTypes: true });
   for (const e of entries) {
@@ -234,26 +187,19 @@ function findChatTxt(dir) {
 }
 
 // ---------------------------------------------------------------------------
-// 5. Account resolution — Phase 0 has one account dir. The wrapper passes
-//    --account-id when it is known; otherwise we pick the single dir under
-//    {install}/data/accounts/.
+// 5. Account resolution (Phase 0 = single account)
 // ---------------------------------------------------------------------------
-
 function resolveAccountId(flags) {
   if (flags.accountId && flags.accountId.trim()) return flags.accountId.trim();
   const installDir = resolve(platformRoot, "..");
   const accountsDir = join(installDir, "data", "accounts");
   if (!existsSync(accountsDir)) {
-    fail("argv", {
-      reason: `accounts dir not found: ${accountsDir}; pass --account-id explicitly`,
-    });
+    fail("argv", { reason: `accounts dir not found: ${accountsDir}; pass --account-id explicitly` });
   }
   const dirs = readdirSync(accountsDir, { withFileTypes: true })
     .filter((d) => d.isDirectory() && !d.name.startsWith("."))
     .map((d) => d.name);
-  if (dirs.length === 0) {
-    fail("argv", { reason: `no accounts found under ${accountsDir}` });
-  }
+  if (dirs.length === 0) fail("argv", { reason: `no accounts found under ${accountsDir}` });
   if (dirs.length > 1) {
     fail("argv", {
       reason: `multiple accounts under ${accountsDir} (${dirs.join(",")}); pass --account-id explicitly`,
@@ -263,26 +209,12 @@ function resolveAccountId(flags) {
 }
 
 // ---------------------------------------------------------------------------
-// 6. Bind senders to canonical {owner, subject} pair (Task 887 §A0).
+// 6. Bind canonical sender set (Task 887 §A0 preserved).
 //
-// Pre-887 behaviour: a `resolveParticipants` step ran each parsed senderName
-// against an account-wide canonical index, falling through to auto-create a
-// `:Person {participantStatus:'auto-created'}` node for any miss. That
-// fallback path was the structural defect: any parser failure that produced
-// a polluted senderName (Task 845's LRM-prefixed body glued onto the prior
-// header → senderName="\"Adam Mackay:\\n‎[04/02/2026, 11:52:16] Adam Mackay\"")
-// leaked one bogus :Person per distinct miss. 23 leaked from the Adam Mackay
-// archive in a single ingest.
-//
-// 887 §A0 closes the leak by deleting the fallback. The operator now passes
-// `--subject-person-id` (third-party Person elementId from preview), the
-// dispatch passes `--owner-element-id`, and the writer accepts EXACTLY the
-// name candidates of those two nodes. Any parsed senderName outside that
-// closed set LOUD-FAILs the ingest with the verbatim string — surfacing a
-// parser bug as a hard exit instead of as graph pollution.
+// All distinct parsed senderNames must resolve to {owner, participants...}.
+// Any miss LOUD-FAILs with `parser-miss reason="senderName=<...>"`.
 // ---------------------------------------------------------------------------
-
-const CANONICAL_PAIR_FETCH_CYPHER = `
+const CANONICAL_FETCH_CYPHER = `
 UNWIND $ids AS id
 MATCH (n) WHERE elementId(n) = id
 RETURN elementId(n) AS elemId,
@@ -293,12 +225,6 @@ RETURN elementId(n) AS elemId,
        coalesce(n.accountId, '') AS accountId
 `;
 
-/**
- * Sentinel error class so `main()`'s try/catch can recognise an operator
- * LOUD-FAIL (already-emitted FAIL line) and exit cleanly with cleanup.
- * Plain `process.exit(1)` from inside `bindCanonicalSenders` would skip
- * `main()`'s `cleanup()` (unzip tmp dir) and `session.close()` paths.
- */
 class IngestUserFacingError extends Error {
   constructor(message) {
     super(message);
@@ -311,61 +237,36 @@ async function bindCanonicalSenders({
   session,
   accountId,
   ownerElementId,
-  subjectPersonId,
+  participantElementIds,
   senderNames,
   normaliseSenderName,
 }) {
-  // Distinctness pre-flight — operator passing the same id for both flags
-  // collapses to a single bound participant; the drift between
-  // `participantCount=2` (from distinct senderNames) and
-  // `participantsAlreadyExisted=1` would propagate to the JSON summary
-  // silently. Refuse early, name the cause.
-  if (ownerElementId === subjectPersonId) {
+  const allIds = [ownerElementId, ...participantElementIds];
+  const distinctIds = Array.from(new Set(allIds));
+  if (distinctIds.length !== allIds.length) {
     throw new IngestUserFacingError(
-      `--owner-element-id and --subject-person-id must be distinct elementIds (both supplied as "${ownerElementId}")`,
+      `participant id list contains duplicates (owner appears in --participant-person-ids?)`,
     );
   }
-
-  const ids = [ownerElementId, subjectPersonId];
-  const res = await session.executeRead(async (tx) =>
-    tx.run(CANONICAL_PAIR_FETCH_CYPHER, { ids }),
-  );
-
-  // Build normalised-name → elementId index from owner + subject candidates.
-  // For each node we accept the full name, given name, family name, and
-  // "given family" composite as match candidates so an export header that
-  // says "Adam" or "Adam Mackay" both resolve to the same node.
-  const index = new Map();
+  const res = await session.executeRead(async (tx) => tx.run(CANONICAL_FETCH_CYPHER, { ids: distinctIds }));
   const seenIds = new Set();
   const labelByElemId = new Map();
+  const index = new Map();
   for (const r of res.records) {
     const elemId = r.get("elemId");
     const labels = r.get("labels") || [];
     const acct = r.get("accountId") || "";
-    // Empty accountId on a canonical Person/AdminUser is a graph-data
-    // defect (migration 004 normally prunes account-less nodes). Refuse
-    // rather than silently accept — bound-pair correctness depends on
-    // accountId being present and matching.
     if (!acct) {
-      throw new IngestUserFacingError(
-        `node ${elemId} has no accountId — corrupt canonical Person/AdminUser; refusing ingest`,
-      );
+      throw new IngestUserFacingError(`node ${elemId} has no accountId — corrupt canonical Person/AdminUser`);
     }
     if (acct !== accountId) {
-      throw new IngestUserFacingError(
-        `node ${elemId} belongs to account ${acct}, not ${accountId} — refusing cross-account ingest`,
-      );
+      throw new IngestUserFacingError(`node ${elemId} belongs to account ${acct}, not ${accountId}`);
     }
     if (!labels.includes("Person") && !labels.includes("AdminUser")) {
-      throw new IngestUserFacingError(
-        `node ${elemId} has labels [${labels.join(",")}]; expected :Person or :AdminUser`,
-      );
+      throw new IngestUserFacingError(`node ${elemId} has labels [${labels.join(",")}]; expected :Person or :AdminUser`);
     }
     seenIds.add(elemId);
-    labelByElemId.set(
-      elemId,
-      labels.includes("Person") ? "Person" : "AdminUser",
-    );
+    labelByElemId.set(elemId, labels.includes("Person") ? "Person" : "AdminUser");
     const candidates = [];
     const name = r.get("name") || "";
     const given = r.get("givenName") || "";
@@ -380,290 +281,336 @@ async function bindCanonicalSenders({
       if (!index.has(norm)) index.set(norm, elemId);
     }
   }
-
-  // Both ids must resolve. A typo'd or stale id is operator error; LOUD-FAIL
-  // before any parser work touches the graph.
-  if (!seenIds.has(ownerElementId)) {
-    throw new IngestUserFacingError(
-      `--owner-element-id ${ownerElementId} not found in graph`,
-    );
-  }
-  if (!seenIds.has(subjectPersonId)) {
-    throw new IngestUserFacingError(
-      `--subject-person-id ${subjectPersonId} not found in graph`,
-    );
-  }
-  // Subject must specifically be a `:Person` — the third party in a DM is
-  // never the operator's `:AdminUser`. (Owner can be either; both Adam and
-  // an external collaborator owning an export are operator-curated cases.)
-  if (labelByElemId.get(subjectPersonId) !== "Person") {
-    throw new IngestUserFacingError(
-      `--subject-person-id ${subjectPersonId} resolves to a :${labelByElemId.get(subjectPersonId)} — subject must be a :Person`,
-    );
-  }
-
-  // Group-chat early-detect: the singular `--subject-person-id` flag is
-  // DM-scoped. A `_chat.txt` carrying ≥3 distinct senders is an unsupported
-  // scope, NOT a parser bug. Emit a distinct reason so the operator does
-  // not chase a phantom parser regression. Group-chat support is the
-  // separate Task 889 lane.
-  if (senderNames.length > 2) {
-    throw new IngestUserFacingError(
-      `unsupported-scope reason="archive carries ${senderNames.length} distinct senders; --subject-person-id is DM-only (≤2 senders) — group-chat ingest is the separate Task 889 lane"`,
-    );
+  for (const id of distinctIds) {
+    if (!seenIds.has(id)) {
+      throw new IngestUserFacingError(`elementId ${id} not found in graph`);
+    }
   }
-
-  // Validate every distinct parsed senderName against the closed candidate
-  // set. The first miss is the LOUD-FAIL — operators see one parser-miss
-  // line per bad import, not 23.
-  const idsByName = new Map();
+  // Validate every distinct parsed senderName against the closed candidate set.
   for (const senderName of senderNames) {
     const norm = normaliseSenderName(senderName);
     const hit = index.get(norm);
     if (!hit) {
       throw new IngestUserFacingError(
-        `parser-miss reason="senderName=${senderName} not in preview histogram (parser failure — re-export or report)"`,
+        `parser-miss reason="senderName=${senderName} not in confirmed participant set (${distinctIds.length} confirmed elementIds; re-run with the missing :Person elementId in --participant-person-ids)"`,
       );
     }
-    idsByName.set(senderName, hit);
-    log(
-      `participant-resolved senderName="${senderName}" matched=canonical nodeId=${hit}`,
-    );
   }
-  // participantsAlreadyExisted = always 2 (owner + subject) under the
-  // bound-pair contract; expose it for the JSON summary's existing field.
-  return { idsByName, participantsAlreadyExisted: seenIds.size };
+  return { participantsResolved: seenIds.size };
+}
+
+// ---------------------------------------------------------------------------
+// 7. SHA-256 of file bytes (for archiveSha256)
+// ---------------------------------------------------------------------------
+async function fileSha256(filePath) {
+  return new Promise((resolveProm, rejectProm) => {
+    const hash = createHash("sha256");
+    const stream = createReadStream(filePath);
+    stream.on("data", (chunk) => hash.update(chunk));
+    stream.on("end", () => resolveProm(hash.digest("hex")));
+    stream.on("error", rejectProm);
+  });
 }
 
 // ---------------------------------------------------------------------------
-// 7. Main.
+// 8. Build natural-edge map for chat-mode classifier.
+// Chat mode doesn't use it (the chat prompt drops edge proposals), but
+// classifyDocument's signature still takes the param. Pass the empty string.
 // ---------------------------------------------------------------------------
 
+// ---------------------------------------------------------------------------
+// 9. Main
+// ---------------------------------------------------------------------------
 async function main() {
   const startedMs = Date.now();
   const { archive, flags } = parseArgv(process.argv);
   const ownerElementId = flags.ownerElementId;
-  const subjectPersonId = flags.subjectPersonId;
+  const participantElementIds = flags.participantPersonIds
+    .split(",")
+    .map((s) => s.trim())
+    .filter((s) => s.length > 0);
+  if (participantElementIds.length === 0) {
+    fail("argv", { reason: "--participant-person-ids must list at least one elementId" });
+  }
   const scope = flags.scope;
   const accountId = resolveAccountId(flags);
   const timezone = flags.timezone || "Europe/London";
   const dateFormat = flags.dateFormat;
+  const sessionGapHours = flags.sessionGapHours
+    ? parseFloat(flags.sessionGapHours)
+    : 12;
+  if (!Number.isFinite(sessionGapHours) || sessionGapHours <= 0) {
+    fail("argv", { reason: `invalid --session-gap-hours "${flags.sessionGapHours}" (must be positive number)` });
+  }
   const sessionId =
-    flags.sessionId || `whatsapp-ingest:${Date.now()}:${Math.random().toString(36).slice(2, 10)}`;
+    flags.sessionId || `whatsapp-import:${Date.now()}:${Math.random().toString(36).slice(2, 10)}`;
 
-  let parseExport, parseFilterArg, applyFilter, memoryArchiveWrite, getSession;
-  let normaliseSenderName, deriveMessageId;
+  // Imports — fail loudly if any compiled dist missing
+  let parseExport, sessionize, toClassifierInput, findDeltaCursor;
+  let normaliseSenderName, deriveConversationIdentity, deriveMessageContentHash;
+  let getSession, classifyDocument, memoryIngest;
   try {
-    ({ parseExport, parseFilterArg, applyFilter } = await import(PARSE_EXPORT_PATH));
-    ({ memoryArchiveWrite } = await import(ARCHIVE_WRITE_PATH));
+    ({
+      parseExport,
+      sessionize,
+      toClassifierInput,
+      findDeltaCursor,
+      normaliseSenderName,
+      deriveConversationIdentity,
+      deriveMessageContentHash,
+    } = await import(PARSE_EXPORT_PATH));
     ({ getSession } = await import(NEO4J_LIB_PATH));
-    ({ normaliseSenderName, deriveMessageId } = await import(DERIVE_KEYS_PATH));
+    ({ classifyDocument } = await import(LLM_CLASSIFIER_PATH));
+    ({ memoryIngest } = await import(MEMORY_INGEST_PATH));
   } catch (err) {
     fail("import", {
-      reason: `failed to import compiled dist`,
+      reason: "failed to import compiled dist",
       detail: err instanceof Error ? err.message : String(err),
     });
   }
 
-  // 7a. Parse the operator-supplied filter ahead of any IO so a malformed
-  // value LOUD-FAILs before unzip / Neo4j connection.
-  let filter;
-  try {
-    filter = parseFilterArg(flags.filter);
-  } catch (err) {
-    fail("argv", { reason: err instanceof Error ? err.message : String(err) });
-  }
-
-  // 7b. Resolve _chat.txt
+  // 9a. Resolve _chat.txt + sha256
   const { chatTxt, cleanup } = resolveChatTxt(archive);
-  const archiveBytes = statSync(chatTxt).size;
+  const archiveSha256 = await fileSha256(chatTxt);
+  const archiveSourceFile = basename(chatTxt);
   log(
-    `start file=${chatTxt} owner=${ownerElementId} scope=${scope} accountId=${accountId} archive-bytes=${archiveBytes} filter=${flags.filter}`,
+    `start file=${archiveSourceFile} owner=${ownerElementId} participants=${participantElementIds.length} scope=${scope} accountId=${accountId} archiveSha256=${archiveSha256.slice(0, 12)} session-gap-hours=${sessionGapHours}`,
   );
 
-  // 7c. Parse
+  // 9b. Parse
   let parseResult;
-  const parseStart = Date.now();
   try {
     parseResult = parseExport({ filePath: chatTxt, accountId, timezone, dateFormat });
   } catch (err) {
     cleanup();
     fail("parse", { reason: err instanceof Error ? err.message : String(err) });
   }
-  const parseMs = Date.now() - parseStart;
-  log(
-    `parsed lines=${parseResult.counters.parsed} media-skipped=${parseResult.counters.mediaSkipped} system-skipped=${parseResult.counters.systemSkipped} ms=${parseMs}`,
-  );
-
-  // 7d. Apply filter
-  const filteredLines = applyFilter(parseResult.parsedLines, filter);
+  const allLines = parseResult.parsedLines;
   log(
-    `filter-applied kind=${filter.kind} parsed=${parseResult.parsedLines.length} filtered=${filteredLines.length}`,
+    `parsed lines=${parseResult.counters.parsed} media-skipped=${parseResult.counters.mediaSkipped} system-skipped=${parseResult.counters.systemSkipped}`,
   );
-  if (filteredLines.length === 0) {
+  if (allLines.length === 0) {
     cleanup();
-    fail("filter", {
-      reason: `filter "${flags.filter}" matched zero rows from ${parseResult.parsedLines.length} parsed lines — refusing to write an empty conversation`,
-    });
+    fail("parse", { reason: "zero parsed lines after walking archive" });
   }
 
-  const firstTs = filteredLines[0].dateSent;
-  const lastTs = filteredLines[filteredLines.length - 1].dateSent;
-
-  // 7e. Auto-create participants and build rows
+  // 9c. Bind canonical senders against the confirmed set
+  const distinctSenderNames = Array.from(new Set(allLines.map((l) => l.senderName)));
+  const senderHistogram = computeSenderHistogram(allLines);
   let session = getSession();
-  let participantUpsert = { idsByName: new Map(), participantsAlreadyExisted: 0 };
-  const distinctSenderNames = Array.from(
-    new Set(filteredLines.map((l) => l.senderName)),
-  );
-
   try {
-    participantUpsert = await bindCanonicalSenders({
+    await bindCanonicalSenders({
       session,
       accountId,
       ownerElementId,
-      subjectPersonId,
+      participantElementIds,
       senderNames: distinctSenderNames,
       normaliseSenderName,
     });
   } catch (err) {
     await session.close().catch(() => {});
     cleanup();
-    // IngestUserFacingError carries a brief-shaped FAIL line (parser-miss /
-    // unsupported-scope / argv mismatch) that the operator's grep recipes
-    // already match on. Preserve it verbatim instead of wrapping in
-    // phase=archive-write — wrapping would defeat
-    // `grep '\[whatsapp-ingest\] FAIL parser-miss'` and friends.
     if (err && err.userFacing) {
-      process.stderr.write(`[whatsapp-ingest] FAIL ${err.message}\n`);
+      process.stderr.write(`[whatsapp-import] FAIL ${err.message}\n`);
       process.exit(1);
     }
-    fail("archive-write", {
-      phase: "bind-canonical-senders",
-      reason: err instanceof Error ? err.message : String(err),
-    });
+    fail("argv", { reason: err instanceof Error ? err.message : String(err) });
   }
 
-  const participantIds = participantUpsert.idsByName;
-  const participantsAlreadyExisted = participantUpsert.participantsAlreadyExisted;
-
-  if (participantIds.size !== distinctSenderNames.length) {
-    log(
-      `participant-resolve mismatch expected=${distinctSenderNames.length} got=${participantIds.size}`,
+  // 9d. Derive conversationIdentity and look up prior :ConversationArchive
+  const conversationIdentity = deriveConversationIdentity({
+    accountId,
+    participantElementIds: [ownerElementId, ...participantElementIds],
+  });
+  let priorArchive = null;
+  try {
+    const r = await session.run(
+      `MATCH (a:ConversationArchive { conversationIdentity: $cid })
+       RETURN elementId(a) AS elemId,
+              a.lastIngestedMessageHash AS lastHash,
+              a.lastIngestedMessageAt   AS lastAt LIMIT 1`,
+      { cid: conversationIdentity },
     );
+    if (r.records[0]) {
+      priorArchive = {
+        elemId: r.records[0].get("elemId"),
+        lastHash: r.records[0].get("lastHash"),
+        lastAt: r.records[0].get("lastAt"),
+      };
+    }
+  } catch (err) {
+    await session.close().catch(() => {});
+    cleanup();
+    fail("delta-cursor-missing", { reason: `conversationArchive lookup failed: ${err instanceof Error ? err.message : String(err)}` });
   }
+  await session.close().catch(() => {});
 
-  const conversationId = parseResult.conversationId;
-  // Task 870: messageId derives from a content-derived natural key. Stable
-  // across re-imports — same archive bytes + same operator timezone produce
-  // identical messageIds, so the existing MERGE in memory-archive-write is
-  // naturally idempotent.
-  const conversationSha256 = parseResult.archiveSourceFile.replace(
-    /^whatsapp-export:/,
-    "",
+  // 9e. Compute deltaStart
+  let deltaStart = 0;
+  let deltaKind = "first-ingest";
+  if (priorArchive && priorArchive.lastHash) {
+    const cursor = findDeltaCursor(allLines, priorArchive.lastHash);
+    if (cursor.kind === "missing") {
+      cleanup();
+      fail("delta-cursor-missing", {
+        reason: `prior cursor not found in re-export (operator deleted prior messages, or this is a different chat archive)`,
+        priorArchive: priorArchive.elemId,
+        lastIngestedMessageAt: priorArchive.lastAt,
+      });
+    }
+    if (cursor.kind === "empty") {
+      log(`noop reason="no new messages since ${priorArchive.lastAt}"`);
+      cleanup();
+      const totalMs = Date.now() - startedMs;
+      process.stdout.write(JSON.stringify({
+        archiveElementId: priorArchive.elemId,
+        conversationIdentity,
+        archiveSha256,
+        archiveSourceFile,
+        parsed: parseResult.counters.parsed,
+        mediaSkipped: parseResult.counters.mediaSkipped,
+        systemSkipped: parseResult.counters.systemSkipped,
+        delta: { kind: "empty-delta", deltaStart: allLines.length, deltaMessages: 0 },
+        sessions: 0,
+        chunks: 0,
+        nextEdgesCreated: 0,
+        participantsLinked: 0,
+        dateRange: { first: allLines[0].dateSent, last: allLines[allLines.length - 1].dateSent },
+        senderHistogram,
+        topicKeywords: [],
+        ms: totalMs,
+        priorLastIngestedMessageAt: priorArchive.lastAt,
+      }) + "\n");
+      process.exit(0);
+    }
+    deltaStart = cursor.deltaStart;
+    deltaKind = "delta";
+  }
+  const deltaLines = allLines.slice(deltaStart);
+  log(
+    `delta cursor=${priorArchive ? priorArchive.lastHash.slice(0, 12) : "(first-ingest)"} cursor-line=${deltaStart} delta-messages=${deltaLines.length}`,
   );
-  const rows = filteredLines.map((l) => ({
-    messageId: deriveMessageId({
-      conversationSha256,
-      dateSent: l.dateSent,
-      senderName: l.senderName,
-      body: l.body,
-    }),
-    conversationId,
-    senderNodeId: participantIds.get(l.senderName) || ownerElementId,
-    senderName: l.senderName,
-    dateSent: l.dateSent,
-    body: l.body,
-    // sequenceIndex preserved on the Message node for chain ordering tiebreaker
-    // in WHATSAPP_NEXT_CHAIN_CYPHER. NOT part of messageId — that would couple
-    // identity to array position and re-introduce shadowing.
-    sequenceIndex: l.sequenceIndex,
-  }));
 
-  const conversationBlock = {
-    conversationId,
-    archiveSourceFile: parseResult.archiveSourceFile,
-    firstMessageAt: firstTs,
-    lastMessageAt: lastTs,
-    participantCount: distinctSenderNames.length,
-    messageCount: rows.length,
-  };
+  // 9f. Sessionize delta
+  const sessions = sessionize(deltaLines, sessionGapHours);
+  log(
+    `sessionize file=${archiveSourceFile} archiveSha256=${archiveSha256.slice(0, 12)} messages=${deltaLines.length} sessions=${sessions.length} gap-hours=${sessionGapHours}`,
+  );
 
-  // Close the session before archive-write — memoryArchiveWrite opens its
-  // own session and closes it in finally.
-  await session.close().catch(() => {});
+  // 9g. Classify each session via Haiku (mode='chat')
+  const allChunks = [];
+  const allKeywords = new Set();
+  for (const s of sessions) {
+    const sessionStart = Date.now();
+    const text = toClassifierInput(s);
+    const result = await classifyDocument({
+      accountId,
+      mode: "chat",
+      anchorDescription: `WhatsApp conversation between ${[ownerElementId, ...participantElementIds].length} participants (session ${s.index + 1} of ${sessions.length})`,
+      ontologyLabels: new Set([]),
+      naturalEdgeMap: "",
+      documentText: text,
+    });
+    if (result.kind === "fallback") {
+      cleanup();
+      fail("classify", { reason: `Haiku fallback on session ${s.index}: ${result.reason}` });
+    }
+    const chunkCount = result.output.sections.length;
+    log(
+      `classify-session sessionIndex=${s.index + 1}/${sessions.length} messages=${s.messages.length} chars=${text.length} chunks=${chunkCount} ms=${Date.now() - sessionStart}`,
+    );
+    if (chunkCount === 0 && s.messages.length > 0) {
+      cleanup();
+      fail("classify", {
+        reason: `session ${s.index} of ${s.messages.length} messages produced zero chunks (classifier-prompt regression)`,
+      });
+    }
+    for (const sec of result.output.sections) {
+      allChunks.push(sec);
+    }
+    for (const kw of result.output.documentKeywords) {
+      allKeywords.add(kw);
+    }
+  }
 
-  // 7f. Archive-write
-  let archiveResult;
-  const writeStart = Date.now();
+  // 9h. Compute lastIngestedMessageHash from the last delta line
+  const lastLine = deltaLines[deltaLines.length - 1];
+  const lastIngestedMessageHash = deriveMessageContentHash({
+    dateSent: lastLine.dateSent,
+    senderName: lastLine.senderName,
+    body: lastLine.body,
+  });
+  const lastIngestedMessageAt = lastLine.dateSent;
+
+  // 9i. Aggregate document-level summary across sessions (concatenate first
+  //     two-three classifier summaries; the parent's summary is informational)
+  const documentSummary = sessions.length === 1
+    ? `${deltaLines.length} messages in 1 session, ${allChunks.length} chunks.`
+    : `${deltaLines.length} messages in ${sessions.length} sessions, ${allChunks.length} chunks.`;
+
+  // 9j. Call memoryIngest with parentLabel='ConversationArchive'
+  let ingestResult;
+  const ingestStart = Date.now();
   try {
-    archiveResult = await memoryArchiveWrite({
-      archiveType: "whatsapp-export",
-      ownerNodeId: ownerElementId,
+    ingestResult = await memoryIngest({
       accountId,
-      conversation: conversationBlock,
-      participantNodeIds: Array.from(participantIds.values()),
-      rows,
+      attachmentId: conversationIdentity, // semantic rename per parentLabel
+      parentLabel: "ConversationArchive",
+      documentSummary,
+      anchorNodeId: ownerElementId,
+      anchorLabel: "AdminUser", // unused in chat path but required by IngestParams
+      sections: allChunks,
+      scope,
       sessionId,
-      archiveFilePath: chatTxt,
+      documentKeywords: Array.from(allKeywords),
+      archiveSha256,
+      archiveSourceFile,
+      lastIngestedMessageHash,
+      lastIngestedMessageAt,
+      participantElementIds: [ownerElementId, ...participantElementIds],
     });
   } catch (err) {
     cleanup();
-    fail("archive-write", {
-      reason: err instanceof Error ? err.message : String(err),
-    });
+    fail("memory-ingest", { reason: err instanceof Error ? err.message : String(err) });
   }
-  const writeMs = Date.now() - writeStart;
-  const createdMessages = archiveResult.counters.createdMessages || 0;
-  const nextEdgesProcessed = archiveResult.counters.nextEdgesProcessed || 0;
-  const nextEdgesCreated = archiveResult.counters.nextEdgesCreated || 0;
-  const messagesAlreadyExisted = rows.length - createdMessages;
   log(
-    `archive-write created-messages=${createdMessages} messages-already-existed=${messagesAlreadyExisted} next-edges-processed=${nextEdgesProcessed} next-edges-created=${nextEdgesCreated} participants-already-existed=${participantsAlreadyExisted} ms=${writeMs}`,
+    `file=${archiveSourceFile} conversationIdentity=${conversationIdentity.slice(0, 12)} archiveElementId=${ingestResult.documentNodeId} chunks-written=${ingestResult.sectionCount} next-edges=${ingestResult.edgeBreakdown.NEXT ?? 0} participants=${ingestResult.edgeBreakdown.PARTICIPANT_IN ?? 0} ms=${Date.now() - ingestStart}`,
   );
 
-  // 7g. Resolve conversationElementId for the agent-return shape (Task 871.6).
-  // One small read-after-write — the operator's database-operator subagent
-  // surfaces this as the canonical handle for downstream Phase 2 calls.
-  let conversationElementId = null;
-  const ridSession = getSession();
-  try {
-    const res = await ridSession.run(
-      `MATCH (c:Conversation:WhatsAppConversation {conversationId: $cid})
-       RETURN elementId(c) AS elemId LIMIT 1`,
-      { cid: conversationId },
-    );
-    conversationElementId = res.records[0]?.get("elemId") ?? null;
-  } catch (err) {
-    log(`conversation-elementid-lookup failed reason="${err instanceof Error ? err.message : String(err)}"`);
-  } finally {
-    await ridSession.close().catch(() => {});
-  }
-
   cleanup();
-
   const totalMs = Date.now() - startedMs;
-  log(`done conversationId=${conversationId} total-ms=${totalMs} exit=0`);
-
-  process.stdout.write(
-    JSON.stringify({
-      conversationElementId,
-      conversationId,
-      parsed: parseResult.counters.parsed,
-      mediaSkipped: parseResult.counters.mediaSkipped,
-      systemSkipped: parseResult.counters.systemSkipped,
-      filtered: filteredLines.length,
-      written: createdMessages,
-      messagesAlreadyExisted,
-      nextEdgesProcessed,
-      nextEdgesCreated,
-      participantsAlreadyExisted,
-      ms: totalMs,
-    }) + "\n",
-  );
+  log(`done conversationIdentity=${conversationIdentity.slice(0, 12)} total-ms=${totalMs} exit=0`);
+
+  process.stdout.write(JSON.stringify({
+    archiveElementId: ingestResult.documentNodeId,
+    conversationIdentity,
+    archiveSha256,
+    archiveSourceFile,
+    parsed: parseResult.counters.parsed,
+    mediaSkipped: parseResult.counters.mediaSkipped,
+    systemSkipped: parseResult.counters.systemSkipped,
+    delta: { kind: deltaKind, deltaStart, deltaMessages: deltaLines.length },
+    sessions: sessions.length,
+    chunks: ingestResult.sectionCount,
+    nextEdgesCreated: ingestResult.edgeBreakdown.NEXT ?? 0,
+    participantsLinked: ingestResult.edgeBreakdown.PARTICIPANT_IN ?? 0,
+    dateRange: { first: allLines[0].dateSent, last: allLines[allLines.length - 1].dateSent },
+    senderHistogram,
+    topicKeywords: Array.from(allKeywords),
+    ms: totalMs,
+  }) + "\n");
   process.exit(0);
 }
 
+function computeSenderHistogram(lines) {
+  const counts = new Map();
+  for (const l of lines) {
+    counts.set(l.senderName, (counts.get(l.senderName) ?? 0) + 1);
+  }
+  return Array.from(counts.entries())
+    .map(([name, count]) => ({ name, count }))
+    .sort((a, b) => b.count - a.count);
+}
 
 main().catch((err) => {
   fail("uncaught", { reason: err instanceof Error ? err.message : String(err) });