@rubytech/create-maxy-code 0.1.0

package/payload/platform/plugins/memory/bin/conversation-archive-ingest.mjs

@@ -0,0 +1,879 @@
+#!/usr/bin/env node
+// =============================================================================
+// conversation-archive-ingest.mjs — in-process orchestrator for
+// conversation-archive-ingest.sh (Task 894 — supersedes whatsapp-import bin;
+// Task 900 — env-only identity, fsync-per-line progress sink, per-session
+// checkpointing).
+//
+// Source-agnostic. The same pipeline runs for every conversation source;
+// `--source <enum>` selects the normaliser at the top of the pipeline:
+//
+//   normalise  → bind canonical senders → derive conversationIdentity
+//   → look up prior :ConversationArchive (delta cursor)
+//   → sessionize delta at gap-hours boundaries
+//   → for each session: classify (mode='chat') → memoryIngest immediately
+//                       → advance lastIngestedMessageHash + lastIngestedMessageAt
+//
+// Identity (Task 900): `process.env.ACCOUNT_ID` and `process.env.USER_ID` are
+// the only inputs. Both are plumbed by `spawn-env.ts` into every Bash
+// subprocess; missing/malformed → LOUD-FAIL at phase=argv. `--account-id` and
+// `--owner-element-id` flags are GONE; the writer derives the AdminUser
+// elementId from (accountId, userId) one Cypher hop away.
+//
+// Argv (positional): <archive-path>
+// Argv (flags):       --source <enum>
+//                     --participant-person-ids <csv>
+//                     --scope <admin|public>
+//                     [--rebuild]                  (Task 902 — destructive rebuild)
+//                     [--timezone <iana>]
+//                     [--date-format <DD/MM/YY|MM/DD/YY|DD/MM/YYYY|MM/DD/YYYY>]
+//                     [--session-id <id>]
+//
+// Sessions split at SESSION_GAP_HOURS=8 (fixed constant — Task 902). The flag
+// `--session-gap-hours` is REMOVED; passing it FAILs at phase=argv. The
+// chunked chat-mode classify path (Task 902 sub-scope C) absorbs oversize
+// sessions internally — no operator lever required.
+//
+// Stdout (success): one JSON line — counters the skill needs to formulate
+// the three operator-facing messages. See SKILL.md for the shape.
+// Stderr + progress file: `[conversation-archive] ...` lines, fsync per write.
+// Progress path: data/accounts/<accountId>/logs/conversation-archive-<sessionId>.log
+// Wrapper sh prints `[conversation-archive] progress-file=<absolute-path>` on
+// stdout before exec node, so the agent computes nothing.
+// =============================================================================
+
+import {
+  closeSync,
+  existsSync,
+  fsyncSync,
+  mkdirSync,
+  mkdtempSync,
+  openSync,
+  readdirSync,
+  rmSync,
+  statSync,
+  writeSync,
+} from "node:fs";
+import { join, resolve, dirname } from "node:path";
+import { tmpdir } from "node:os";
+import { spawnSync } from "node:child_process";
+import { fileURLToPath } from "node:url";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+// Task 902 sub-scope D — fixed session gap. Pre-902 this was a CLI flag with a
+// 12h default; oversize sessions on dense archives drove operators to re-issue
+// with a smaller gap, which combined with cleanup-by-archiveSha256 to silently
+// destroy 138 chunks (Adam Mackay incident, 2026-05-04). The flag is gone;
+// chunked chat-mode classify (sub-scope C) absorbs oversize sessions
+// internally so the gap value is no longer an operational lever. 8h is a
+// tighter default than 12h and matches the operator-decided post-incident
+// constant.
+const SESSION_GAP_HOURS = 8;
+
+// ---------------------------------------------------------------------------
+// 1. Resolve dist paths.
+// ---------------------------------------------------------------------------
+const platformRoot =
+  process.env.MAXY_PLATFORM_ROOT?.trim() ||
+  resolve(__dirname, "..", "..", "..");
+
+const NORMALISERS_PATH = resolve(
+  platformRoot,
+  "plugins",
+  "memory",
+  "mcp",
+  "dist",
+  "lib",
+  "conversation-normalisers",
+  "index.js",
+);
+const PIPELINE_PATH = resolve(
+  platformRoot,
+  "plugins",
+  "memory",
+  "mcp",
+  "dist",
+  "lib",
+  "conversation-pipeline",
+);
+const NEO4J_LIB_PATH = resolve(
+  platformRoot,
+  "plugins",
+  "memory",
+  "mcp",
+  "dist",
+  "lib",
+  "neo4j.js",
+);
+const LLM_CLASSIFIER_PATH = resolve(
+  platformRoot,
+  "plugins",
+  "memory",
+  "mcp",
+  "dist",
+  "lib",
+  "llm-classifier.js",
+);
+const MEMORY_INGEST_PATH = resolve(
+  platformRoot,
+  "plugins",
+  "memory",
+  "mcp",
+  "dist",
+  "tools",
+  "memory-ingest.js",
+);
+const UUID_LIB_PATH = resolve(
+  platformRoot,
+  "plugins",
+  "memory",
+  "mcp",
+  "dist",
+  "lib",
+  "uuid.js",
+);
+
+// ---------------------------------------------------------------------------
+// 2. Progress sink + logger fan-out (Task 900 sub-scope A).
+//    `log()` and `fail()` write to stderr AND the progress file with fsync per
+//    write. Parent Bash returns nothing until child exits; the progress file
+//    is the only surface the agent's heartbeat poll can read mid-run.
+// ---------------------------------------------------------------------------
+let progressFd = -1;
+const startedMs = Date.now();
+
+function openProgressSink(absolutePath) {
+  mkdirSync(dirname(absolutePath), { recursive: true });
+  progressFd = openSync(absolutePath, "a");
+}
+
+function writeProgressLine(line) {
+  // stderr first — preserves existing observability for parents that read
+  // stderr after the child exits, AND makes the line visible via the
+  // mcp-spawn-tee'd device server.log without a separate sink.
+  process.stderr.write(line);
+  if (progressFd >= 0) {
+    try {
+      writeSync(progressFd, line);
+      fsyncSync(progressFd);
+    } catch {
+      // Disk pressure / EBADF: don't kill the run — stderr already carries
+      // the line.
+    }
+  }
+}
+
+function log(line) {
+  writeProgressLine(`[conversation-archive] ${line}\n`);
+}
+
+function endRun(exitCode) {
+  const totalMs = Date.now() - startedMs;
+  writeProgressLine(`[conversation-archive] end exit-code=${exitCode} ms=${totalMs}\n`);
+  if (progressFd >= 0) {
+    try { closeSync(progressFd); } catch { /* fd already closed */ }
+    progressFd = -1;
+  }
+  process.exit(exitCode);
+}
+
+function fail(phase, fields) {
+  const fieldStr = Object.entries(fields)
+    .map(([k, v]) =>
+      typeof v === "string" && (v.includes(" ") || v.includes("="))
+        ? `${k}="${v.replace(/"/g, '\\"')}"`
+        : `${k}=${v ?? "-"}`,
+    )
+    .join(" ");
+  writeProgressLine(`[conversation-archive] FAIL phase=${phase} ${fieldStr}\n`);
+  endRun(1);
+}
+
+// ---------------------------------------------------------------------------
+// 3. Argv parsing
+// ---------------------------------------------------------------------------
+const VALID_SOURCES = new Set([
+  "whatsapp",
+  "telegram",
+  "signal",
+  "linkedin-messages",
+  "zoom-transcript",
+  "meeting-minutes",
+  "imessage",
+  "slack",
+  "other",
+]);
+
+// Boolean flags carry no value; every other flag consumes the following arg.
+const BOOLEAN_FLAGS = new Set(["rebuild"]);
+
+function parseArgv(argv) {
+  const args = argv.slice(2);
+  let archive = null;
+  const flags = {};
+  for (let i = 0; i < args.length; i++) {
+    const a = args[i];
+    if (!a.startsWith("--")) {
+      if (archive == null) archive = a;
+      else fail("argv", { reason: `unexpected positional argument "${a}"` });
+      continue;
+    }
+    const key = a.slice(2);
+    // Task 902 sub-scope D — the flag re-introduced the surface that enabled
+    // the silent-partial-wipe failure (Defect B). Reject loudly with the
+    // remediation the operator should use instead.
+    if (key === "session-gap-hours") {
+      fail("argv", {
+        reason: `--session-gap-hours is no longer supported; gap is fixed at 8h. Use --rebuild on the prior export bytes if you need to re-classify.`,
+      });
+    }
+    if (BOOLEAN_FLAGS.has(camelCase(key))) {
+      flags[camelCase(key)] = true;
+      continue;
+    }
+    const v = args[++i];
+    if (v == null) fail("argv", { reason: `flag --${key} requires a value` });
+    flags[camelCase(key)] = v;
+  }
+  if (!archive) fail("argv", { reason: "missing positional <archive>" });
+  if (!flags.source) fail("argv", { reason: "missing --source" });
+  if (!VALID_SOURCES.has(flags.source)) {
+    fail("argv", { reason: `invalid --source "${flags.source}" (whatsapp|telegram|signal|linkedin-messages|zoom-transcript|meeting-minutes|imessage|slack|other)` });
+  }
+  if (!flags.participantPersonIds) {
+    fail("argv", {
+      reason: "missing --participant-person-ids (csv of operator-confirmed :Person/:AdminUser elementIds, owner excluded)",
+    });
+  }
+  if (!flags.scope) fail("argv", { reason: "missing --scope" });
+  if (flags.scope !== "admin" && flags.scope !== "public") {
+    fail("argv", { reason: `invalid --scope "${flags.scope}" (admin|public)` });
+  }
+  return { archive, flags };
+}
+function camelCase(s) {
+  return s.replace(/-([a-z])/g, (_m, c) => c.toUpperCase());
+}
+
+// ---------------------------------------------------------------------------
+// 4. Archive resolution. WhatsApp ships zip-or-dir-or-_chat.txt; other
+//    sources pass a single file path that the normaliser interprets directly.
+// ---------------------------------------------------------------------------
+function resolveSourceFile(archivePath, source) {
+  const abs = resolve(archivePath);
+  if (!existsSync(abs)) fail("argv", { reason: `archive path not found: ${abs}` });
+  const st = statSync(abs);
+
+  if (source === "whatsapp") {
+    if (st.isFile() && abs.endsWith(".zip")) {
+      const tmp = mkdtempSync(join(tmpdir(), "conversation-archive-"));
+      const unzip = spawnSync("unzip", ["-q", "-o", abs, "-d", tmp], { encoding: "utf8" });
+      if (unzip.status !== 0) {
+        rmSync(tmp, { recursive: true });
+        fail("argv", {
+          reason: "unzip failed",
+          archive: abs,
+          stderr: (unzip.stderr || "").slice(0, 200),
+        });
+      }
+      const chat = findChatTxt(tmp);
+      if (!chat) {
+        rmSync(tmp, { recursive: true });
+        fail("argv", { reason: "_chat.txt not found in zip", archive: abs });
+      }
+      return { sourceFile: chat, cleanup: () => rmSync(tmp, { recursive: true }) };
+    }
+    if (st.isDirectory()) {
+      const chat = findChatTxt(abs);
+      if (!chat) fail("argv", { reason: "_chat.txt not found in directory", archive: abs });
+      return { sourceFile: chat, cleanup: () => {} };
+    }
+  }
+
+  if (st.isFile()) {
+    return { sourceFile: abs, cleanup: () => {} };
+  }
+  fail("argv", { reason: `unsupported archive shape for source=${source}: ${abs}` });
+  return { sourceFile: abs, cleanup: () => {} };
+}
+function findChatTxt(dir) {
+  const entries = readdirSync(dir, { withFileTypes: true });
+  for (const e of entries) {
+    if (e.isFile() && e.name === "_chat.txt") return join(dir, e.name);
+  }
+  for (const e of entries) {
+    if (e.isDirectory()) {
+      const nested = findChatTxt(join(dir, e.name));
+      if (nested) return nested;
+    }
+  }
+  return null;
+}
+
+// ---------------------------------------------------------------------------
+// 4b. Stable archive title (Task 902 sub-scope A).
+//
+// Format: `<source> · <owner> ↔ <other1>, <other2>, … · <YYYY-MM-DD>→<YYYY-MM-DD>`.
+//
+// The title is computed once per ingest from inputs that are immutable for
+// the conversation (source, accountId, sortedParticipantElementIds, archive
+// dateRange) so re-ingests produce the same string. Memory-ingest writes it
+// to `:ConversationArchive.title` ON CREATE and COALESCE-on-MATCH so the UI
+// (display-helpers.ts) reads a stable label without ever falling through to
+// `summary`. Per-session counter strings stay in the progress file only.
+//
+// Names: AdminUser uses `displayName` then `slug`; Person composes
+// `givenName + familyName`. When neither resolves, fall back to a short
+// elementId prefix — same degraded format the migration uses on backfill,
+// so a re-ingest after the migration produces the same title shape that
+// already populated the node.
+// ---------------------------------------------------------------------------
+
+function isoToYmd(iso) {
+  // Read the YYYY-MM-DD prefix verbatim — same tz-stable contract as
+  // formatRunDate in display-helpers.ts. Parsing to Date + reformatting
+  // would flip the day between operator timezones.
+  const m = typeof iso === "string" ? iso.match(/^(\d{4}-\d{2}-\d{2})/) : null;
+  return m ? m[1] : "?";
+}
+
+function pickNameForRow(labels, props) {
+  if (Array.isArray(labels) && labels.includes("AdminUser")) {
+    const dn = typeof props.displayName === "string" ? props.displayName.trim() : "";
+    if (dn) return dn;
+    const slug = typeof props.slug === "string" ? props.slug.trim() : "";
+    if (slug) return slug;
+  }
+  if (Array.isArray(labels) && labels.includes("Person")) {
+    const given = typeof props.givenName === "string" ? props.givenName.trim() : "";
+    const family = typeof props.familyName === "string" ? props.familyName.trim() : "";
+    const full = [given, family].filter((s) => s.length > 0).join(" ");
+    if (full) return full;
+  }
+  return "";
+}
+
+async function computeArchiveTitle({
+  source,
+  accountId,
+  ownerElementId,
+  participantElementIds,
+  firstMessageAt,
+  lastMessageAt,
+  getSession,
+}) {
+  const session = getSession();
+  let nameByElementId = new Map();
+  try {
+    const ids = [ownerElementId, ...participantElementIds];
+    const r = await session.run(
+      `UNWIND $ids AS id
+       MATCH (n) WHERE elementId(n) = id AND n.accountId = $accountId
+       RETURN elementId(n) AS elemId, labels(n) AS labels, properties(n) AS props`,
+      { ids, accountId },
+    );
+    for (const rec of r.records) {
+      const elemId = rec.get("elemId");
+      const labels = rec.get("labels");
+      const props = rec.get("props") || {};
+      const name = pickNameForRow(labels, props);
+      if (name) nameByElementId.set(elemId, name);
+    }
+  } finally {
+    await session.close().catch(() => {});
+  }
+  const ownerName = nameByElementId.get(ownerElementId) || ownerElementId.slice(0, 8);
+  const otherNames = participantElementIds
+    .map((id) => nameByElementId.get(id) || id.slice(0, 8))
+    .join(", ");
+  const firstYmd = isoToYmd(firstMessageAt);
+  const lastYmd = isoToYmd(lastMessageAt);
+  return `${source} · ${ownerName} ↔ ${otherNames} · ${firstYmd}→${lastYmd}`;
+}
+
+// ---------------------------------------------------------------------------
+// 5. Main
+// ---------------------------------------------------------------------------
+async function main() {
+  const { archive, flags } = parseArgv(process.argv);
+  const source = flags.source;
+
+  // Identity is env-only (Task 900 sub-scope C). spawn-env.ts plumbs both
+  // ACCOUNT_ID and USER_ID into every Bash subprocess; the bin LOUD-FAILs if
+  // either is absent or malformed. The disk-scan resolver and the
+  // --account-id / --owner-element-id flags are gone.
+  let UUID_REGEX;
+  try {
+    ({ UUID_REGEX } = await import(UUID_LIB_PATH));
+  } catch (err) {
+    fail("import", {
+      reason: "failed to import compiled uuid lib",
+      detail: err instanceof Error ? err.message : String(err),
+    });
+  }
+
+  const accountId = process.env.ACCOUNT_ID?.trim();
+  if (!accountId) {
+    fail("argv", { reason: "ACCOUNT_ID env missing — bin must be invoked under an authenticated agent context" });
+  }
+  if (!UUID_REGEX.test(accountId)) {
+    fail("argv", { reason: `ACCOUNT_ID env malformed (not a UUID): "${accountId}"` });
+  }
+  const userId = process.env.USER_ID?.trim();
+  if (!userId) {
+    fail("argv", { reason: "USER_ID env missing — bin must be invoked under an authenticated user session" });
+  }
+  if (!UUID_REGEX.test(userId)) {
+    fail("argv", { reason: `USER_ID env malformed (not a UUID): "${userId}"` });
+  }
+
+  const participantElementIds = flags.participantPersonIds
+    .split(",")
+    .map((s) => s.trim())
+    .filter((s) => s.length > 0);
+  if (participantElementIds.length === 0) {
+    fail("argv", { reason: "--participant-person-ids must list at least one elementId" });
+  }
+  const scope = flags.scope;
+  const timezone = flags.timezone || "Europe/London";
+  const dateFormat = flags.dateFormat;
+  // Task 902 sub-scope B — `--rebuild` is the single opt-in for destructive
+  // re-run-with-same-bytes. Without it, re-issuing the bin on the same export
+  // is a no-op (chunk-count summary, exit 0). With it, the bin treats the run
+  // as first-ingest (skips prior-cursor lookup) and the FIRST session's
+  // memoryIngest call cleans prior chunks for the archiveSha256.
+  const rebuild = flags.rebuild === true;
+  const sessionId =
+    flags.sessionId ||
+    `conversation-archive:${source}:${Date.now()}:${Math.random().toString(36).slice(2, 10)}`;
+
+  // ---------------------------------------------------------------------------
+  // Open progress sink BEFORE any await. Path is deterministic from
+  // (accountId, sessionId); the wrapper sh prints the same path on stdout so
+  // the agent's heartbeat polls the right file.
+  // ---------------------------------------------------------------------------
+  const accountsDir = resolve(platformRoot, "..", "data", "accounts");
+  const logsDir = resolve(accountsDir, accountId, "logs");
+  const progressFilePath = resolve(logsDir, `conversation-archive-${sessionId}.log`);
+  openProgressSink(progressFilePath);
+
+  log(
+    `start session=${sessionId} source=${source} archive=${archive} accountId=${accountId} userId=${userId} participants=${participantElementIds.length} scope=${scope} pid=${process.pid}`,
+  );
+
+  // Imports — fail loudly if any compiled dist missing
+  let getNormaliser;
+  let sessionize, toTurnText, findDeltaCursor;
+  let deriveConversationIdentity, deriveMessageContentHash;
+  let verifyParticipantElementIds;
+  let getSession, classifyDocument, memoryIngest;
+  try {
+    ({ getNormaliser } = await import(NORMALISERS_PATH));
+    ({ sessionize } = await import(join(PIPELINE_PATH, "sessionize.js")));
+    ({ toTurnText } = await import(join(PIPELINE_PATH, "to-turn-text.js")));
+    ({ findDeltaCursor } = await import(join(PIPELINE_PATH, "delta-cursor.js")));
+    ({ deriveConversationIdentity, deriveMessageContentHash } =
+      await import(join(PIPELINE_PATH, "derive-keys.js")));
+    ({ verifyParticipantElementIds } = await import(join(PIPELINE_PATH, "sender-bind.js")));
+    ({ getSession } = await import(NEO4J_LIB_PATH));
+    ({ classifyDocument } = await import(LLM_CLASSIFIER_PATH));
+    ({ memoryIngest } = await import(MEMORY_INGEST_PATH));
+  } catch (err) {
+    fail("import", {
+      reason: "failed to import compiled dist",
+      detail: err instanceof Error ? err.message : String(err),
+    });
+  }
+
+  // Resolve owner AdminUser elementId from (accountId, userId). One Cypher
+  // hop replaces the deleted --owner-element-id flag — owner identity is env
+  // input, never an out-of-band flag.
+  let ownerElementId;
+  {
+    const lookupSession = getSession();
+    try {
+      const r = await lookupSession.run(
+        `MATCH (au:AdminUser { accountId: $accountId, userId: $userId })
+         RETURN elementId(au) AS ownerElementId LIMIT 1`,
+        { accountId, userId },
+      );
+      const row = r.records[0]?.get("ownerElementId");
+      if (typeof row !== "string" || !row) {
+        await lookupSession.close().catch(() => {});
+        fail("argv", {
+          reason: `USER_ID does not resolve to an AdminUser under ACCOUNT_ID (accountId=${accountId.slice(0, 8)}, userId=${userId.slice(0, 8)})`,
+        });
+      }
+      ownerElementId = row;
+    } catch (err) {
+      await lookupSession.close().catch(() => {});
+      fail("argv", {
+        reason: `AdminUser lookup failed: ${err instanceof Error ? err.message : String(err)}`,
+      });
+    }
+    await lookupSession.close().catch(() => {});
+  }
+  log(`owner-resolved ownerElementId=${ownerElementId.slice(0, 12)} userId=${userId.slice(0, 8)}`);
+
+  // 6a. Resolve source file
+  const { sourceFile, cleanup } = resolveSourceFile(archive, source);
+
+  // 6b. Run the source-specific normaliser
+  let normaliserResult;
+  try {
+    const normaliser = getNormaliser(source);
+    normaliserResult = await normaliser({
+      filePath: sourceFile,
+      accountId,
+      timezone,
+      opts: dateFormat ? { dateFormat } : undefined,
+    });
+  } catch (err) {
+    cleanup();
+    fail("parse", { reason: err instanceof Error ? err.message : String(err) });
+  }
+  const allLines = normaliserResult.parsedLines;
+  const archiveSha256 = normaliserResult.archiveSha256;
+  const archiveSourceFile = normaliserResult.archiveSourceFile;
+  log(
+    `source=${source} file=${archiveSourceFile} owner=${ownerElementId} participants=${participantElementIds.length} scope=${scope} accountId=${accountId} archiveSha256=${archiveSha256.slice(0, 12)} gap=${SESSION_GAP_HOURS} rebuild=${rebuild}`,
+  );
+  log(
+    `parsed lines=${normaliserResult.counters.parsed} media-skipped=${normaliserResult.counters.mediaSkipped} system-skipped=${normaliserResult.counters.systemSkipped}`,
+  );
+  // Task 897: zero matched timestamps means this isn't a chat archive — the
+  // file should route through document-ingest instead. The agent's
+  // database-operator skill picks the right entry; this LOUD-FAIL surfaces
+  // misroute as an actionable signal rather than a silent empty ingest.
+  if (allLines.length === 0) {
+    cleanup();
+    fail("parse", {
+      reason: `no timestamp prefixes detected — this archive contains no chat messages with the expected source=${source} grammar; route through document-ingest with mode='document', not conversation-archive-ingest`,
+    });
+  }
+
+  // 6c. Verify participant elementIds resolve to graph nodes with the right
+  // accountId and labels. Per-line senderName binding is gone (Task 897);
+  // operator-confirmed participants attach to the parent :ConversationArchive.
+  let session = getSession();
+  try {
+    await verifyParticipantElementIds({
+      session,
+      accountId,
+      ownerElementId,
+      participantElementIds,
+    });
+  } catch (err) {
+    await session.close().catch(() => {});
+    cleanup();
+    if (err && err.userFacing) {
+      writeProgressLine(`[conversation-archive] FAIL ${err.message}\n`);
+      endRun(1);
+    }
+    fail("argv", { reason: err instanceof Error ? err.message : String(err) });
+  }
+
+  // 6d. Derive conversationIdentity and look up prior :ConversationArchive.
+  // Task 902 sub-scope B — under `--rebuild` we skip the prior-cursor lookup
+  // entirely; the run is treated as first-ingest, the first session cleans
+  // prior chunks (cleanupPriorChunks=true), and the cursor is overwritten
+  // by that session's MERGE. Without `--rebuild`, the prior cursor drives
+  // delta-append OR (Task 902 sub-scope B) the same-bytes no-op exit when
+  // chunks already exist for this archiveSha256.
+  const conversationIdentity = deriveConversationIdentity({
+    accountId,
+    participantElementIds: [ownerElementId, ...participantElementIds],
+  });
+  let priorArchive = null;
+  let priorChunkCountForSha = 0;
+  if (!rebuild) {
+    try {
+      const r = await session.run(
+        `MATCH (a:ConversationArchive { conversationIdentity: $cid })
+         OPTIONAL MATCH (a)-[:HAS_SECTION]->(c:Section:Conversation { archiveSha256: $sha })
+         RETURN elementId(a) AS elemId,
+                a.lastIngestedMessageHash AS lastHash,
+                a.lastIngestedMessageAt   AS lastAt,
+                count(c)                  AS chunksForSha LIMIT 1`,
+        { cid: conversationIdentity, sha: archiveSha256 },
+      );
+      if (r.records[0] && r.records[0].get("elemId")) {
+        priorArchive = {
+          elemId: r.records[0].get("elemId"),
+          lastHash: r.records[0].get("lastHash"),
+          lastAt: r.records[0].get("lastAt"),
+        };
+        const cs = r.records[0].get("chunksForSha");
+        priorChunkCountForSha =
+          typeof cs === "number" ? cs : cs?.toNumber?.() ?? 0;
+      }
+    } catch (err) {
+      await session.close().catch(() => {});
+      cleanup();
+      fail("delta-cursor-missing", { reason: `conversationArchive lookup failed: ${err instanceof Error ? err.message : String(err)}` });
+    }
+  }
+  await session.close().catch(() => {});
+
+  // 6e. Compute deltaStart. Three branches:
+  //   (i) --rebuild: deltaStart=0, kind=rebuild, classify entire archive.
+  //  (ii) no prior archive (or no cursor): first-ingest, deltaStart=0.
+  // (iii) prior archive + cursor: standard delta-append. Same-bytes (no
+  //       cursor advance possible) AND chunks exist for this sha → no-op.
+  let deltaStart = 0;
+  let deltaKind = rebuild ? "rebuild" : "first-ingest";
+  if (!rebuild && priorArchive && priorArchive.lastHash) {
+    const cursor = findDeltaCursor(allLines, priorArchive.lastHash);
+    if (cursor.kind === "missing") {
+      cleanup();
+      fail("delta-cursor-missing", {
+        reason: `prior cursor not found in re-export (operator deleted prior messages, or this is a different chat archive)`,
+        priorArchive: priorArchive.elemId,
+        lastIngestedMessageAt: priorArchive.lastAt,
+      });
+    }
+    if (cursor.kind === "empty") {
+      log(`noop reason="no new messages since ${priorArchive.lastAt}" prior-chunks=${priorChunkCountForSha}`);
+      cleanup();
+      const totalMs = Date.now() - startedMs;
+      process.stdout.write(JSON.stringify({
+        archiveElementId: priorArchive.elemId,
+        conversationIdentity,
+        archiveSha256,
+        archiveSourceFile,
+        source,
+        parsed: normaliserResult.counters.parsed,
+        mediaSkipped: normaliserResult.counters.mediaSkipped,
+        systemSkipped: normaliserResult.counters.systemSkipped,
+        delta: { kind: "empty-delta", deltaStart: allLines.length, deltaMessages: 0 },
+        sessions: 0,
+        chunks: priorChunkCountForSha,
+        nextEdgesCreated: 0,
+        participantsLinked: 0,
+        dateRange: { first: allLines[0].dateSent, last: allLines[allLines.length - 1].dateSent },
+        parsedLineCount: allLines.length,
+        sessionsUnenriched: 0,
+        topicKeywords: [],
+        ms: totalMs,
+        priorLastIngestedMessageAt: priorArchive.lastAt,
+      }) + "\n");
+      endRun(0);
+    }
+    deltaStart = cursor.deltaStart;
+    deltaKind = "delta";
+  }
+  const deltaLines = allLines.slice(deltaStart);
+  log(
+    `delta kind=${deltaKind} cursor=${priorArchive && priorArchive.lastHash && !rebuild ? priorArchive.lastHash.slice(0, 12) : "(first-ingest)"} cursor-line=${deltaStart} delta-messages=${deltaLines.length}`,
+  );
+
+  // 6f. Sessionize delta at the fixed 8h gap (Task 902 sub-scope D).
+  const sessions = sessionize(deltaLines, SESSION_GAP_HOURS);
+  log(
+    `sessionize source=${source} archiveSha256=${archiveSha256.slice(0, 12)} messages=${deltaLines.length} sessions=${sessions.length} gap-hours=${SESSION_GAP_HOURS}`,
+  );
+
+  // Task 902 sub-scope A — compute the stable archive title once, before
+  // the per-session loop. The title is `<source> · <owner> ↔ <others> ·
+  // <YYYY-MM-DD>→<YYYY-MM-DD>`. Operator-confirmed participant names come
+  // from one Cypher hop; absent canonical names fall back to elementId
+  // prefixes so the migration's degraded-title path mirrors live behaviour.
+  const archiveTitle = await computeArchiveTitle({
+    source,
+    accountId,
+    ownerElementId,
+    participantElementIds,
+    firstMessageAt: allLines[0].dateSent,
+    lastMessageAt: allLines[allLines.length - 1].dateSent,
+    getSession,
+  });
+  log(`archive-title=${archiveTitle}`);
+
+  // 6g. Per-session classify + immediate memoryIngest (Task 900 sub-scope E).
+  // Each session commits atomically: chunks + cursor advance happen together
+  // via memoryIngest. A kill mid-loop leaves a partial archive whose cursor
+  // is at session N-1's last message; the next invocation slices from there
+  // and re-classifies only session N onward. degrade-on-error from Task 897
+  // is preserved: a Haiku error on session N still emits an unenriched chunk
+  // and advances the cursor, so resumption never re-attempts the same Haiku
+  // error.
+  const allKeywords = new Set();
+  let sessionsUnenriched = 0;
+  let totalChunksWritten = 0;
+  let totalNextEdges = 0;
+  let participantsLinked = 0;
+  let archiveElementIdLatest = priorArchive ? priorArchive.elemId : "";
+
+  for (let sIdx = 0; sIdx < sessions.length; sIdx++) {
+    const s = sessions[sIdx];
+    const isFirstSessionInRun = sIdx === 0;
+    const sessionStart = Date.now();
+    const text = toTurnText(s);
+    const result = await classifyDocument({
+      accountId,
+      mode: "chat",
+      anchorDescription: `Conversation transcript (${[ownerElementId, ...participantElementIds].length} participants, session ${s.index + 1} of ${sessions.length})`,
+      ontologyLabels: new Set([]),
+      naturalEdgeMap: "",
+      documentText: text,
+    });
+
+    let sessionChunks;
+    let sessionKeywords = [];
+    if (result.kind === "error") {
+      sessionsUnenriched += 1;
+      log(`session-unenriched session=${s.index + 1}/${sessions.length} reason="${result.reason.replace(/"/g, '\\"')}"`);
+      // Emit one raw :Section:Conversation chunk covering the whole session.
+      // No summary, no topicKeywords — a follow-up pass can re-enrich.
+      sessionChunks = [{
+        kind: "Conversation",
+        title: `Unenriched session ${s.index + 1}`,
+        body: text,
+        summary: "",
+        sourceStart: 0,
+        sourceEnd: text.length,
+        properties: {
+          unenriched: true,
+          unenrichedReason: result.reason,
+          firstMessageAt: s.firstMessageAt,
+          lastMessageAt: s.lastMessageAt,
+          messageCount: s.messages.length,
+        },
+        anchorEdge: null,
+      }];
+    } else {
+      const chunkCount = result.output.sections.length;
+      log(
+        `classify-session sessionIndex=${s.index + 1}/${sessions.length} messages=${s.messages.length} chars=${text.length} chunks=${chunkCount} ms=${Date.now() - sessionStart}`,
+      );
+      if (chunkCount === 0 && s.messages.length > 0) {
+        cleanup();
+        fail("classify", {
+          reason: `session ${s.index} of ${s.messages.length} messages produced zero chunks (classifier-prompt regression)`,
+        });
+      }
+      sessionChunks = result.output.sections;
+      sessionKeywords = result.output.documentKeywords ?? [];
+      for (const kw of sessionKeywords) allKeywords.add(kw);
+    }
+
+    // Per-session cursor: advance to THIS session's last message. Atomic with
+    // chunk writes inside memoryIngest's MERGE/CREATE Cypher transaction.
+    const lastSessionLine = s.messages[s.messages.length - 1];
+    const sessionLastHash = deriveMessageContentHash({
+      dateSent: lastSessionLine.dateSent,
+      body: lastSessionLine.body,
+    });
+    const sessionLastAt = lastSessionLine.dateSent;
+
+    let ingestResult;
+    const ingestStart = Date.now();
+    try {
+      ingestResult = await memoryIngest({
+        accountId,
+        attachmentId: conversationIdentity,
+        parentLabel: "ConversationArchive",
+        source,
+        // Task 902 sub-scope A — `archiveTitle` is the stable label written
+        // to `:ConversationArchive.title` (UI prefers it over `summary`).
+        // `documentSummary` is also the stable title rather than the
+        // per-session counter that previously locked the parent's first
+        // checkpoint into "Session 1/N: …".
+        archiveTitle,
+        documentSummary: archiveTitle,
+        anchorNodeId: ownerElementId,
+        anchorLabel: "AdminUser",
+        sections: sessionChunks,
+        scope,
+        sessionId,
+        documentKeywords: Array.from(allKeywords),
+        archiveSha256,
+        archiveSourceFile,
+        lastIngestedMessageHash: sessionLastHash,
+        lastIngestedMessageAt: sessionLastAt,
+        participantElementIds: [ownerElementId, ...participantElementIds],
+        // Task 902 sub-scope B — cleanup-by-archiveSha256 runs ONLY under
+        // `--rebuild`, AND only on the first session of the run (subsequent
+        // sessions skip cleanup or they would delete chunks the earlier
+        // sessions in the same run just wrote).
+        cleanupPriorChunks: rebuild && isFirstSessionInRun,
+      });
+    } catch (err) {
+      cleanup();
+      fail("memory-ingest", {
+        reason: err instanceof Error ? err.message : String(err),
+        sessionIndex: s.index + 1,
+        sessionsTotal: sessions.length,
+      });
+    }
+
+    archiveElementIdLatest = ingestResult.documentNodeId;
+    totalChunksWritten += ingestResult.sectionCount;
+    totalNextEdges += ingestResult.edgeBreakdown.NEXT ?? 0;
+    if (isFirstSessionInRun) {
+      participantsLinked = ingestResult.edgeBreakdown.PARTICIPANT_IN ?? 0;
+      // Task 902 sub-scope E — surface destructive cleanup as a WARN line in
+      // the progress sink (not just memory-ingest's stderr). The agent's
+      // heartbeat grep-matches this exact prefix and elevates to operator
+      // chat. Cleanup only ever fires under `--rebuild` per sub-scope B,
+      // so seeing the line outside of an operator-issued `--rebuild` is a
+      // doctrine violation worth surfacing loudly.
+      const cleaned = ingestResult.cleanedPriorChunks ?? 0;
+      if (cleaned > 0) {
+        log(
+          `WARN cleanup-dropped chunks=${cleaned} archiveSha256=${archiveSha256.slice(0, 12)} — prior data deleted, expected only under --rebuild`,
+        );
+      }
+    }
+    log(
+      `session-committed sessionIndex=${s.index + 1}/${sessions.length} chunks-written=${ingestResult.sectionCount} cursor-at="${sessionLastAt}" ms=${Date.now() - ingestStart}`,
+    );
+    // Task 902 sub-scope F — derived progress line. The agent's heartbeat
+    // surfaces only `progress` lines as chat tokens; raw `classify-session`
+    // and `session-committed` lines remain on disk for diagnosis. The render
+    // is computed in the bin (deterministic) instead of in SKILL.md prose
+    // (LLM drift) — sessionIndex/N → P% with running totals comes from one
+    // place, not from per-tick agent arithmetic.
+    const pct = Math.round((100 * (s.index + 1)) / sessions.length);
+    log(
+      `progress sessionIndex=${s.index + 1}/${sessions.length} pct=${pct} chunks-so-far=${totalChunksWritten} elapsed-ms=${Date.now() - startedMs}`,
+    );
+  }
+
+  if (sessionsUnenriched > 0) {
+    log(`classify-summary sessions=${sessions.length} unenriched=${sessionsUnenriched} enriched=${sessions.length - sessionsUnenriched}`);
+  }
+
+  cleanup();
+  const totalMs = Date.now() - startedMs;
+  log(
+    `done source=${source} conversationIdentity=${conversationIdentity.slice(0, 12)} archiveElementId=${archiveElementIdLatest} chunks-written=${totalChunksWritten} sessions=${sessions.length} total-ms=${totalMs}`,
+  );
+
+  process.stdout.write(JSON.stringify({
+    archiveElementId: archiveElementIdLatest,
+    conversationIdentity,
+    archiveSha256,
+    archiveSourceFile,
+    source,
+    parsed: normaliserResult.counters.parsed,
+    mediaSkipped: normaliserResult.counters.mediaSkipped,
+    systemSkipped: normaliserResult.counters.systemSkipped,
+    delta: { kind: deltaKind, deltaStart, deltaMessages: deltaLines.length },
+    sessions: sessions.length,
+    chunks: totalChunksWritten,
+    nextEdgesCreated: totalNextEdges,
+    participantsLinked,
+    dateRange: { first: allLines[0].dateSent, last: allLines[allLines.length - 1].dateSent },
+    parsedLineCount: allLines.length,
+    sessionsUnenriched,
+    topicKeywords: Array.from(allKeywords),
+    ms: totalMs,
+  }) + "\n");
+  endRun(0);
+}
+
+main().catch((err) => {
+  fail("uncaught", { reason: err instanceof Error ? err.message : String(err) });
+});