@chainlesschain/personal-data-hub 0.1.0 → 0.2.0

package/scripts/smoke-phase-5-7.js

@@ -0,0 +1,181 @@
+#!/usr/bin/env node
+/**
+ * Phase 5.7 smoke — exercises retry-with-backoff + onProgress through a
+ * flaky session. No real IMAP needed.
+ *
+ *   1. First 2 connect attempts throw ECONNRESET (transient)
+ *   2. 3rd attempt succeeds, yields 5 envelopes
+ *   3. Verify progress events fire in order: connecting → error → connecting
+ *      → error → connecting → connected → mailbox-opened → fetching × 5 → done
+ *   4. Verify retry was capped (3 attempts total)
+ *   5. Verify AUTH_FAILED does NOT retry (separate run)
+ */
+
+"use strict";
+
+const { EmailAdapter } = require("../lib/adapters/email-imap/email-adapter");
+const { ImapAuthFailedError } = require("../lib/adapters/email-imap/imap-session");
+
+function makeFlakyFactory(failuresFirst, envelopes) {
+  const failures = failuresFirst.slice();
+  const recorder = { attempts: 0 };
+  const factory = () => {
+    let openMb = null;
+    return {
+      async connect() {
+        recorder.attempts += 1;
+        if (failures.length > 0) {
+          throw failures.shift();
+        }
+      },
+      async openMailbox(name) {
+        openMb = name;
+        return { uidValidity: 1, uidNext: 9999, exists: envelopes.length };
+      },
+      async *fetchFullSince(sinceUid = 0) {
+        for (const env of envelopes) {
+          if (env.uid > sinceUid) yield { ...env, source: env.source || Buffer.alloc(0) };
+        }
+      },
+      async close() {},
+    };
+  };
+  return { factory, recorder };
+}
+
+function makeEnv(uid) {
+  return {
+    uid,
+    internalDate: new Date(`2026-05-${String(uid).padStart(2, "0")}T10:00:00Z`),
+    flags: ["\\Seen"],
+    messageId: `<m-${uid}@x>`,
+    subject: `Subject ${uid}`,
+    from: [{ address: `s${uid}@example.com` }],
+    to: [{ address: "me@example.com" }],
+    cc: [],
+    date: new Date(`2026-05-${String(uid).padStart(2, "0")}T10:00:00Z`),
+    size: 1024,
+  };
+}
+
+async function scenarioRetrySucceeds() {
+  console.log("\n=== Scenario A — 2 fails then success, 5 envelopes ===");
+  const transient = Object.assign(new Error("ECONNRESET"), { code: "ECONNRESET" });
+  const envs = [1, 2, 3, 4, 5].map(makeEnv);
+  const { factory, recorder } = makeFlakyFactory([transient, transient], envs);
+
+  const events = [];
+  const a = new EmailAdapter({
+    account: { provider: "qq", email: "me@qq.com", authCode: "x", folders: ["INBOX"] },
+    sessionFactory: factory,
+    parser: async () => ({ textBody: "", attachments: [] }),
+    maxConnectRetries: 3,
+    retryBaseDelayMs: 5,
+    onProgress: (e) => events.push(e),
+  });
+
+  console.log("adapter.version =", a.version);
+  console.log("adapter.capabilities (Phase 5.7) =", a.capabilities.filter((c) => c.startsWith("sync:")).join(", "));
+
+  const raws = [];
+  for await (const r of a.sync()) raws.push(r);
+  console.log(`emitted ${raws.length} raws (expected 5)`);
+  console.log(`connect attempts: ${recorder.attempts} (expected 3)`);
+
+  const phaseSeq = events.map((e) => `${e.phase}${e.attempt ? "(" + e.attempt + ")" : ""}`);
+  console.log("phase sequence:");
+  for (const p of phaseSeq) console.log("  -", p);
+
+  // Verify expected phase order
+  const errs = events.filter((e) => e.phase === "error");
+  if (errs.length !== 2) {
+    console.error(`FAIL: expected 2 error events, got ${errs.length}`);
+    process.exitCode = 1;
+  } else if (!errs.every((e) => e.retriable === true)) {
+    console.error("FAIL: error events should be retriable=true during first 2 attempts");
+    process.exitCode = 1;
+  } else {
+    console.log("error events: ✓ both marked retriable");
+  }
+
+  const done = events.find((e) => e.phase === "done");
+  if (!done || done.emitted !== 5) {
+    console.error(`FAIL: expected done event with emitted=5, got ${JSON.stringify(done)}`);
+    process.exitCode = 1;
+  } else {
+    console.log(`done event: ✓ emitted=${done.emitted} durationMs=${done.durationMs}`);
+  }
+
+  const fetches = events.filter((e) => e.phase === "fetching");
+  if (fetches.length !== 5) {
+    console.error(`FAIL: expected 5 fetching events, got ${fetches.length}`);
+    process.exitCode = 1;
+  } else if (fetches[0].total !== 5 || fetches[4].current !== 5) {
+    console.error(`FAIL: fetching events should run 1..5 of 5`);
+    process.exitCode = 1;
+  } else {
+    console.log("fetching events: ✓ 5 events with current/total");
+  }
+}
+
+async function scenarioAuthFailedNoRetry() {
+  console.log("\n=== Scenario B — AUTH_FAILED never retries ===");
+  const authErr = new ImapAuthFailedError("bad creds");
+  const { factory, recorder } = makeFlakyFactory(
+    [authErr, authErr, authErr], // shouldn't matter — first one stops us
+    [],
+  );
+
+  const events = [];
+  const a = new EmailAdapter({
+    account: { provider: "qq", email: "me@qq.com", authCode: "x", folders: ["INBOX"] },
+    sessionFactory: factory,
+    parser: async () => ({}),
+    maxConnectRetries: 3,
+    retryBaseDelayMs: 1,
+    onProgress: (e) => events.push(e),
+  });
+
+  let caught = null;
+  try {
+    for await (const _r of a.sync()) { /* drain */ }
+  } catch (err) {
+    caught = err;
+  }
+  console.log(`connect attempts: ${recorder.attempts} (expected 1)`);
+  console.log(`caught.code: ${caught && caught.code}`);
+  if (recorder.attempts !== 1) {
+    console.error("FAIL: AUTH_FAILED should not retry");
+    process.exitCode = 1;
+  } else if (!caught || caught.code !== "AUTH_FAILED") {
+    console.error("FAIL: error should propagate as AUTH_FAILED");
+    process.exitCode = 1;
+  } else {
+    console.log("AUTH short-circuit: ✓");
+  }
+
+  const errEvent = events.find((e) => e.phase === "error");
+  if (!errEvent) {
+    console.error("FAIL: error progress event missing");
+    process.exitCode = 1;
+  } else if (errEvent.retriable !== false) {
+    console.error("FAIL: AUTH_FAILED error event should have retriable=false");
+    process.exitCode = 1;
+  } else {
+    console.log("AUTH error event: ✓ retriable=false");
+  }
+}
+
+async function main() {
+  console.log("== Phase 5.7 smoke (retry + progress) ==");
+  await scenarioRetrySucceeds();
+  await scenarioAuthFailedNoRetry();
+  if (!process.exitCode) {
+    console.log("\n== Phase 5.7 smoke PASSED ==");
+  }
+}
+
+main().catch((err) => {
+  console.error("smoke crashed:", err);
+  process.exitCode = 1;
+});

package/scripts/smoke-system-data-contacts.js

@@ -0,0 +1,309 @@
+#!/usr/bin/env node
+/**
+ * Smoke / real-device runner for Phase 4.5.2 — Contacts extraction.
+ *
+ * Drives the full vertical:
+ *
+ *   ┌──────────────────────────────────────────────────────────────┐
+ *   │ 1. (optional) android.list_devices                           │
+ *   │ 2. (optional) android.pull_file  /data/.../contacts2.db      │
+ *   │ 3. system.parse_contacts → Persons                           │
+ *   │ 4. hub-side UnifiedSchema validatePerson() on every row       │
+ *   │ 5. write NormalizedBatch JSON to ./out/<timestamp>/           │
+ *   └──────────────────────────────────────────────────────────────┘
+ *
+ * Three modes:
+ *
+ *   --db <path>          Skip ADB entirely; parse a contacts2.db already
+ *                        on disk. Best for first-run sanity on the dev box.
+ *
+ *   --serial <serial>    Run `adb pull` first. Requires `adb root` (most
+ *                        retail builds reject this) OR a userdebug build.
+ *                        On a stock Redmi 24115RA8EC use --workaround.
+ *
+ *   --workaround sdcard  Look for a contacts2.db copy at
+ *                        /sdcard/Download/contacts2.db (you copied it
+ *                        out via Termux + tsu, or via Mi cloud export,
+ *                        per docs/design/Adapter_System_Data.md §2.1).
+ *
+ * Usage examples:
+ *
+ *     # Local fixture
+ *     node scripts/smoke-system-data-contacts.js --db ./fixtures/contacts2.db
+ *
+ *     # List devices, then prompt for serial
+ *     node scripts/smoke-system-data-contacts.js --list
+ *
+ *     # Real device with /sdcard workaround
+ *     node scripts/smoke-system-data-contacts.js \
+ *         --serial 24115RA8ECabc123 --workaround sdcard
+ *
+ * Exits non-zero on any sidecar error or schema validation failure.
+ */
+
+"use strict";
+
+const path = require("node:path");
+const fs = require("node:fs");
+const os = require("node:os");
+
+const { SidecarSupervisor } = require("../lib/sidecar");
+const { validatePerson } = require("../lib/schemas");
+
+const SIDECAR_ROOT = path.resolve(__dirname, "..", "..", "personal-data-hub-bridge");
+const PYTHON = process.env.FORENSICS_BRIDGE_PYTHON || "python";
+
+const SDCARD_WORKAROUND_PATH = "/sdcard/Download/contacts2.db";
+const SYSTEM_PROVIDER_PATH =
+  "/data/data/com.android.providers.contacts/databases/contacts2.db";
+
+// ---------------------------------------------------------------------------
+// CLI parsing — kept dependency-free
+// ---------------------------------------------------------------------------
+
+function parseArgs(argv) {
+  const out = {
+    db: null,
+    serial: null,
+    workaround: null,
+    list: false,
+    outDir: null,
+    help: false,
+  };
+  for (let i = 0; i < argv.length; i += 1) {
+    const a = argv[i];
+    switch (a) {
+      case "--db":
+        out.db = argv[++i];
+        break;
+      case "--serial":
+        out.serial = argv[++i];
+        break;
+      case "--workaround":
+        out.workaround = argv[++i];
+        break;
+      case "--list":
+        out.list = true;
+        break;
+      case "--out":
+        out.outDir = argv[++i];
+        break;
+      case "-h":
+      case "--help":
+        out.help = true;
+        break;
+      default:
+        if (a.startsWith("--")) {
+          throw new Error(`unknown flag: ${a}`);
+        }
+    }
+  }
+  return out;
+}
+
+function printHelp() {
+  process.stdout.write(`
+smoke-system-data-contacts — drive sidecar end-to-end for contacts.
+
+  --db <path>             Parse a contacts2.db already on disk (skip ADB).
+  --serial <serial>       Target this ADB device for the pull step.
+  --workaround sdcard     Pull from ${SDCARD_WORKAROUND_PATH} instead of /data/data.
+                          Required on stock Android (no adb root).
+  --list                  Just list ADB devices and exit.
+  --out <dir>             Write NormalizedBatch JSON here. Default: ./out/<ts>.
+  -h, --help              Show this help.
+
+Env:
+  FORENSICS_BRIDGE_PYTHON   override Python interpreter (default: python).
+
+Exit codes:
+  0  success
+  1  sidecar / hub error
+  2  invalid Persons (schema validation failed)
+`);
+}
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function timestampSlug() {
+  const d = new Date();
+  const z = (n) => String(n).padStart(2, "0");
+  return (
+    `${d.getFullYear()}${z(d.getMonth() + 1)}${z(d.getDate())}-` +
+    `${z(d.getHours())}${z(d.getMinutes())}${z(d.getSeconds())}`
+  );
+}
+
+function makeSupervisor() {
+  return new SidecarSupervisor({
+    command: PYTHON,
+    args: ["-u", "-m", "forensics_bridge.ipc_server"],
+    cwd: SIDECAR_ROOT,
+    healthCheckIntervalMs: 0,
+    env: { PYTHONPATH: SIDECAR_ROOT },
+  });
+}
+
+function log(level, msg, extra = {}) {
+  const line = JSON.stringify({
+    ts: new Date().toISOString(),
+    level,
+    msg,
+    ...extra,
+  });
+  if (level === "error") process.stderr.write(line + "\n");
+  else process.stdout.write(line + "\n");
+}
+
+// ---------------------------------------------------------------------------
+// Main
+// ---------------------------------------------------------------------------
+
+async function main(rawArgs) {
+  let args;
+  try {
+    args = parseArgs(rawArgs);
+  } catch (err) {
+    console.error(err.message);
+    printHelp();
+    process.exit(2);
+  }
+  if (args.help) {
+    printHelp();
+    return;
+  }
+
+  const outDir = path.resolve(
+    args.outDir || path.join(process.cwd(), "out", timestampSlug()),
+  );
+  fs.mkdirSync(outDir, { recursive: true });
+  log("info", "output directory ready", { outDir });
+
+  const sup = makeSupervisor();
+  // Stream sidecar pino-style logs out as ndjson so the user sees timing.
+  sup.on("log", (line) => process.stderr.write(`[sidecar] ${line}\n`));
+
+  await sup.start({ readyTimeoutMs: 10_000 });
+  log("info", "sidecar ready");
+
+  try {
+    // ---------- list-only path ----------
+    if (args.list) {
+      const devices = await sup.invoke("android.list_devices");
+      log("info", "adb devices", devices);
+      console.log(JSON.stringify(devices, null, 2));
+      return;
+    }
+
+    // ---------- choose source for contacts2.db ----------
+    let dbPath = args.db ? path.resolve(args.db) : null;
+
+    if (!dbPath) {
+      if (!args.serial) {
+        throw new Error(
+          "neither --db nor --serial provided; nothing to extract",
+        );
+      }
+      const remotePath =
+        args.workaround === "sdcard" ? SDCARD_WORKAROUND_PATH : SYSTEM_PROVIDER_PATH;
+      log("info", "pulling from device", { serial: args.serial, remotePath });
+      const pulled = await sup.invoke(
+        "android.pull_file",
+        {
+          serial: args.serial,
+          remote_path: remotePath,
+          local_dir: outDir,
+        },
+        { timeoutMs: 60_000 },
+      );
+      log("info", "pull completed", pulled);
+      dbPath = pulled.local;
+    }
+
+    if (!fs.existsSync(dbPath)) {
+      throw new Error(`contacts db not found at ${dbPath}`);
+    }
+
+    // ---------- parse + validate ----------
+    const persons = [];
+    let chunks = 0;
+    const t0 = Date.now();
+    const parseResult = await sup.invoke(
+      "system.parse_contacts",
+      {
+        data_path: dbPath,
+        device_serial: args.serial || null,
+      },
+      {
+        timeoutMs: 120_000,
+        onProgress: (p) => log("info", "progress", p),
+        onChunk: (batch) => {
+          chunks += 1;
+          for (const person of batch.persons || []) persons.push(person);
+        },
+      },
+    );
+    const wallMs = Date.now() - t0;
+    log("info", "parse completed", {
+      ...parseResult,
+      chunks,
+      wallMs,
+      personsCollected: persons.length,
+    });
+
+    // ---------- hub-side schema check ----------
+    const invalid = [];
+    for (const p of persons) {
+      const v = validatePerson(p);
+      if (!v.valid) invalid.push({ id: p.id, errors: v.errors });
+    }
+    if (invalid.length) {
+      log("error", "validation failed", { count: invalid.length });
+      fs.writeFileSync(
+        path.join(outDir, "validation-errors.json"),
+        JSON.stringify(invalid, null, 2),
+      );
+      process.exitCode = 2;
+    } else {
+      log("info", "all persons passed UnifiedSchema validation");
+    }
+
+    // ---------- persist for inspection ----------
+    const dump = {
+      schemaVersion: "0.1.0",
+      generatedAt: new Date().toISOString(),
+      sidecar: { pythonRoot: SIDECAR_ROOT },
+      input: {
+        dbPath,
+        serial: args.serial || null,
+        workaround: args.workaround || null,
+      },
+      parseResult,
+      wallMs,
+      persons,
+    };
+    const dumpPath = path.join(outDir, "contacts-normalized-batch.json");
+    fs.writeFileSync(dumpPath, JSON.stringify(dump, null, 2));
+    log("info", "wrote dump", { dumpPath, bytes: fs.statSync(dumpPath).size });
+
+    // ---------- compact summary ----------
+    log("info", "summary", {
+      totalPersons: parseResult.totalPersons,
+      withPhone: parseResult.stats?.with_phone,
+      withEmail: parseResult.stats?.with_email,
+      starred: parseResult.stats?.starred,
+      invalidPersons: invalid.length,
+      outDir,
+    });
+  } finally {
+    await sup.stop({ graceMs: 2000 });
+  }
+}
+
+main(process.argv.slice(2)).catch((err) => {
+  log("error", "fatal", { name: err.name, message: err.message, code: err.code });
+  if (err.stack) process.stderr.write(err.stack + "\n");
+  process.exit(1);
+});