@chainlesschain/personal-data-hub 0.1.0 → 0.2.1

package/scripts/evaluate-entity-resolver.js

@@ -0,0 +1,213 @@
+#!/usr/bin/env node
+/**
+ * Phase 8.8 — EntityResolver evaluation runner.
+ *
+ * Reads a labeled pair fixture, runs each pair through the configured
+ * stages, computes recall / accuracy / per-stage breakdown, and exits
+ * non-zero if the CI gate (recall ≥ 80%, accuracy ≥ 90%) fails.
+ *
+ * Usage:
+ *   node scripts/evaluate-entity-resolver.js \
+ *     [--fixture <path>] [--use-embedding] [--use-llm] [--require-pass]
+ *
+ * Defaults:
+ *   - fixture: __tests__/fixtures/entity-resolver-200-mock.json
+ *   - --use-embedding: skipped unless flag set (needs Ollama running)
+ *   - --use-llm: skipped unless flag set (needs Ollama + chat model)
+ *   - --require-pass: exit 1 when gate fails (use in CI; otherwise warn-only)
+ */
+
+"use strict";
+
+const fs = require("node:fs");
+const path = require("node:path");
+
+const {
+  entityResolverRuleStage: ruleStage,
+} = require("../lib/entity-resolver");
+
+const args = parseArgs(process.argv.slice(2));
+const fixturePath = args.fixture || path.join(
+  __dirname, "..", "__tests__", "fixtures", "entity-resolver-200-mock.json"
+);
+const useEmbedding = !!args["use-embedding"];
+const useLlm = !!args["use-llm"];
+const requirePass = !!args["require-pass"];
+
+const RECALL_GATE = 0.80;
+const ACCURACY_GATE = 0.90;
+
+async function main() {
+  console.log("== EntityResolver evaluation ==");
+  console.log("fixture:", fixturePath);
+  if (!fs.existsSync(fixturePath)) {
+    console.error("\nFAIL: fixture not found");
+    process.exit(2);
+  }
+  const data = JSON.parse(fs.readFileSync(fixturePath, "utf-8"));
+  if (!Array.isArray(data.pairs) || data.pairs.length === 0) {
+    console.error("\nFAIL: fixture has no pairs");
+    process.exit(2);
+  }
+
+  const stages = { embedding: null, llm: null };
+  if (useEmbedding) {
+    const { EntityResolverEmbeddingStage } = require("../lib/entity-resolver");
+    stages.embedding = new EntityResolverEmbeddingStage({}).asStageFn();
+    console.log("embedding stage: Ollama nomic-embed-text @ localhost:11434");
+  }
+  if (useLlm) {
+    const { EntityResolverLLMStage } = require("../lib/entity-resolver");
+    const { OllamaClient } = require("../lib/llm-client");
+    const llm = new OllamaClient({ baseUrl: "http://localhost:11434", model: "qwen2.5:7b-instruct" });
+    stages.llm = new EntityResolverLLMStage({ llm }).asStageFn();
+    console.log("llm stage: Ollama qwen2.5:7b-instruct");
+  }
+
+  // Per-stage counters
+  const counts = {
+    ruleSame: 0, ruleDifferent: 0, ruleUncertain: 0,
+    embeddingSame: 0, embeddingDifferent: 0, embeddingUncertain: 0,
+    llmSame: 0, llmDifferent: 0, llmMaybe: 0,
+  };
+
+  // Confusion matrix
+  const confusion = { tp: 0, fp: 0, fn: 0, tn: 0, unresolved: 0 };
+
+  // Per-pair breakdown
+  const results = [];
+
+  for (const pair of data.pairs) {
+    const truth = pair.groundTruth; // "same" | "different"
+    const ruleVerdict = ruleStage(pair.a, pair.b).verdict;
+    let finalVerdict = ruleVerdict;
+    let stage = "rule";
+
+    if (ruleVerdict === "same") counts.ruleSame += 1;
+    else if (ruleVerdict === "different") counts.ruleDifferent += 1;
+    else counts.ruleUncertain += 1;
+
+    if (ruleVerdict === "uncertain" && stages.embedding) {
+      const e = await stages.embedding(pair.a, pair.b);
+      if (e.sim >= 0.85) {
+        finalVerdict = "same";
+        counts.embeddingSame += 1;
+        stage = "embedding";
+      } else if (e.sim < 0.55) {
+        finalVerdict = "different";
+        counts.embeddingDifferent += 1;
+        stage = "embedding";
+      } else {
+        counts.embeddingUncertain += 1;
+        if (stages.llm) {
+          const v = await stages.llm(pair.a, pair.b);
+          if (v.verdict === "yes" && v.confidence >= 0.7) {
+            finalVerdict = "same";
+            counts.llmSame += 1;
+            stage = "llm";
+          } else if (v.verdict === "no" && v.confidence >= 0.7) {
+            finalVerdict = "different";
+            counts.llmDifferent += 1;
+            stage = "llm";
+          } else {
+            finalVerdict = "review";
+            counts.llmMaybe += 1;
+            stage = "llm-review";
+          }
+        }
+      }
+    }
+
+    // Tally confusion (only counting decided verdicts)
+    if (finalVerdict === "same" && truth === "same") confusion.tp += 1;
+    else if (finalVerdict === "same" && truth === "different") confusion.fp += 1;
+    else if (finalVerdict === "different" && truth === "same") confusion.fn += 1;
+    else if (finalVerdict === "different" && truth === "different") confusion.tn += 1;
+    else confusion.unresolved += 1;
+
+    results.push({ id: pair.id, truth, finalVerdict, stage, category: pair.category });
+  }
+
+  const total = data.pairs.length;
+  const resolved = total - confusion.unresolved;
+  const accuracy = resolved > 0 ? (confusion.tp + confusion.tn) / resolved : 0;
+  const recall = (confusion.tp + confusion.fn) > 0
+    ? confusion.tp / (confusion.tp + confusion.fn)
+    : 1;
+  const precision = (confusion.tp + confusion.fp) > 0
+    ? confusion.tp / (confusion.tp + confusion.fp)
+    : 1;
+  const resolveRate = resolved / total;
+
+  // ── Report ──
+  console.log("\nPair counts:", { total, resolved, unresolved: confusion.unresolved });
+  console.log("\nPipeline stages:");
+  console.log(`  Rule:      same=${counts.ruleSame} different=${counts.ruleDifferent} uncertain=${counts.ruleUncertain}`);
+  if (stages.embedding) console.log(`  Embedding: same=${counts.embeddingSame} different=${counts.embeddingDifferent} uncertain=${counts.embeddingUncertain}`);
+  if (stages.llm) console.log(`  LLM:       same=${counts.llmSame} different=${counts.llmDifferent} maybe=${counts.llmMaybe}`);
+
+  console.log("\nConfusion (decided only):");
+  console.log(`  TP=${confusion.tp}  FP=${confusion.fp}  FN=${confusion.fn}  TN=${confusion.tn}`);
+  console.log(`  unresolved=${confusion.unresolved}  (rule "uncertain" with no embedding/llm wired)`);
+
+  console.log("\nMetrics:");
+  const recallStr = (recall * 100).toFixed(1) + "%";
+  const accStr = (accuracy * 100).toFixed(1) + "%";
+  const precStr = (precision * 100).toFixed(1) + "%";
+  const resolveStr = (resolveRate * 100).toFixed(1) + "%";
+  console.log(`  Recall:       ${recallStr}  (target ≥ ${(RECALL_GATE * 100).toFixed(0)}%) ${recall >= RECALL_GATE ? "✓" : "✗"}`);
+  console.log(`  Accuracy:     ${accStr}  (target ≥ ${(ACCURACY_GATE * 100).toFixed(0)}%) ${accuracy >= ACCURACY_GATE ? "✓" : "✗"}`);
+  console.log(`  Precision:    ${precStr}`);
+  console.log(`  Resolve rate: ${resolveStr}`);
+
+  // Per-category breakdown (useful for spotting weak spots)
+  const byCategory = {};
+  for (const r of results) {
+    const cat = r.category || "(uncat)";
+    if (!byCategory[cat]) byCategory[cat] = { total: 0, correct: 0, unresolved: 0 };
+    byCategory[cat].total += 1;
+    if (r.finalVerdict === "review" || r.finalVerdict === "uncertain") {
+      byCategory[cat].unresolved += 1;
+    } else if (r.finalVerdict === r.truth) {
+      byCategory[cat].correct += 1;
+    }
+  }
+  console.log("\nPer-category:");
+  for (const cat of Object.keys(byCategory)) {
+    const s = byCategory[cat];
+    const accStr = s.total - s.unresolved > 0
+      ? ((s.correct / (s.total - s.unresolved)) * 100).toFixed(0) + "%"
+      : "N/A";
+    console.log(`  ${cat}: ${s.correct}/${s.total - s.unresolved} correct (${accStr}); ${s.unresolved} unresolved`);
+  }
+
+  // Gate decision
+  const passed = recall >= RECALL_GATE && accuracy >= ACCURACY_GATE;
+  console.log(`\n${passed ? "✓ PASS" : "✗ FAIL"} — recall ${recallStr} / accuracy ${accStr}`);
+
+  if (!passed && requirePass) {
+    process.exit(1);
+  }
+}
+
+function parseArgs(argv) {
+  const out = {};
+  for (let i = 0; i < argv.length; i += 1) {
+    const a = argv[i];
+    if (a.startsWith("--")) {
+      const k = a.slice(2);
+      if (argv[i + 1] && !argv[i + 1].startsWith("--")) {
+        out[k] = argv[i + 1];
+        i += 1;
+      } else {
+        out[k] = true;
+      }
+    }
+  }
+  return out;
+}
+
+main().catch((err) => {
+  console.error("\nFATAL:", err && err.message ? err.message : err);
+  process.exit(2);
+});

package/scripts/smoke-phase-5-5.js

@@ -0,0 +1,196 @@
+#!/usr/bin/env node
+/**
+ * Phase 5.5 smoke — drives EmailAdapter end-to-end with a mock encrypted
+ * PDF attachment, exercising:
+ *   - password-trial loop (3 wrong → 1 right)
+ *   - text extraction
+ *   - transactions regex (3 rows from a 招行-style statement)
+ *   - merging transactions[] into bill template fields
+ *   - per-attachment pdfExtraction summary
+ *   - attachment buffer stripping before raw-event emission
+ *
+ * Uses an INJECTED pdfExtractor so the smoke runs without pulling the
+ * heavy pdfjs dep. The shape of the injected output matches what
+ * `extractPdfText` from pdf-extractor.js would return.
+ */
+
+"use strict";
+
+const { EmailAdapter } = require("../lib/adapters/email-imap/email-adapter");
+const { extractTransactions } = require("../lib/adapters/email-imap/transactions");
+
+const PDF_TEXT = [
+  "招商银行信用卡 11 月对账单",
+  "持卡人: 张三  尾号 1234",
+  "账单周期: 2026-10-26 至 2026-11-25",
+  "最后还款日: 2026-12-05  应还金额: ¥3,000.00",
+  "",
+  "交易明细:",
+  "2026-10-30  星巴克 上海中山公园店             -39.00      2,961.00",
+  "2026-11-05  京东自营                           -899.00     2,062.00",
+  "2026-11-12  退款 淘宝                          +50.00      2,112.00",
+  "2026-11-18  美团外卖                           -85.00      2,027.00",
+  "",
+  "第 1 页 共 1 页",
+].join("\n");
+
+const PDF_PASSWORDS = ["wrong1", "wrong2", "wrong3", "987654"];
+
+function makeSession() {
+  const env = {
+    uid: 1,
+    internalDate: new Date("2026-11-26T10:00:00Z"),
+    flags: ["\\Seen"],
+    messageId: "<bill-cmb-11@x>",
+    subject: "招商银行信用卡 11 月对账单",
+    from: [{ name: "招商银行", address: "ebank@cmbchina.com" }],
+    to: [{ address: "me@example.com" }],
+    cc: [],
+    date: new Date("2026-11-26T10:00:00Z"),
+    size: 8192,
+    source: Buffer.from("RAW", "utf8"),
+  };
+  return () => ({
+    async connect() {},
+    async openMailbox(_name) {
+      return { uidValidity: 1, uidNext: 9999, exists: 1 };
+    },
+    async *fetchFullSince(sinceUid = 0) {
+      if (env.uid > sinceUid) yield env;
+    },
+    async close() {},
+  });
+}
+
+let trialCount = 0;
+async function mockPdfExtractor(buffer, opts) {
+  trialCount = 0;
+  for (const pw of ["", ...(opts.passwords || [])]) {
+    trialCount += 1;
+    if (pw === "987654") {
+      return {
+        decrypted: true,
+        text: PDF_TEXT,
+        password: pw,
+        attempted: trialCount,
+        wasEncrypted: true,
+        pageCount: 1,
+      };
+    }
+  }
+  return {
+    decrypted: false,
+    text: "",
+    attempted: trialCount,
+    wasEncrypted: true,
+    pageCount: 0,
+    error: "all passwords failed",
+  };
+}
+
+async function main() {
+  console.log("== Phase 5.5 smoke ==");
+
+  // First validate the standalone transactions parser on the fixture text
+  console.log("\n— Standalone transactions regex —");
+  const standaloneTxns = extractTransactions(PDF_TEXT);
+  console.log(`extracted ${standaloneTxns.length} transactions:`);
+  for (const t of standaloneTxns) {
+    const dir = t.amount.direction || "?";
+    const date = new Date(t.occurredAtMs).toISOString().slice(0, 10);
+    console.log(`  ${date}  ${dir.padEnd(3)}  ¥${t.amount.value.toFixed(2).padStart(8)}  ${t.description}`);
+  }
+  if (standaloneTxns.length !== 4) {
+    console.log(`FAIL: expected 4 transactions, got ${standaloneTxns.length}`);
+    process.exitCode = 1;
+    return;
+  }
+
+  // Now full pipeline
+  console.log("\n— Full adapter pipeline —");
+
+  const a = new EmailAdapter({
+    account: { provider: "qq", email: "me@qq.com", authCode: "x", folders: ["INBOX"] },
+    sessionFactory: makeSession(),
+    parser: async () => ({
+      textBody: "您的招商银行信用卡 11 月对账单已生成，详情见附件 PDF。",
+      attachments: [{
+        filename: "招行账单_11月.pdf",
+        contentType: "application/pdf",
+        contentDisposition: "attachment",
+        size: 78_456,
+        sha256: "abc123sha256deadbeef",
+        isInline: false,
+        isEncrypted: true,
+        buffer: Buffer.from("FAKE-PDF-BYTES-DO-NOT-LEAK"),
+      }],
+    }),
+    pdfExtractor: mockPdfExtractor,
+    pdfPasswordHints: { idCardLast6: "987654", phoneLast6: "555000" },
+    pdfPasswords: ["wrong1", "wrong2", "wrong3"], // tried before hints
+  });
+
+  console.log("adapter.version =", a.version);
+  console.log("adapter.capabilities =", a.capabilities.join(", "));
+  console.log("pdfPasswords (merged) =", a._pdfPasswords);
+
+  let count = 0;
+  for await (const raw of a.sync()) {
+    count += 1;
+    const ext = raw.payload.extraction;
+    console.log(`\nemail #${count} subject: ${raw.payload.subject}`);
+    console.log("  classification.category:", raw.payload.classification.category);
+    console.log("  extraction.template    :", ext.template);
+    console.log("  extraction.confidence  :", ext.confidence);
+    console.log("  extraction.fields keys :", Object.keys(ext.fields || {}).join(", "));
+    if (ext.fields.transactions) {
+      console.log(`  transactions[] count   : ${ext.fields.transactions.length}`);
+      for (const t of ext.fields.transactions) {
+        const date = new Date(t.occurredAtMs).toISOString().slice(0, 10);
+        console.log(`    ${date} ¥${t.amount.value} ${t.amount.direction} ${t.description}`);
+      }
+    }
+    console.log("  pdfExtraction[]        :");
+    for (const p of ext.pdfExtraction || []) {
+      console.log(`    ${p.filename}: decrypted=${p.decrypted} attempted=${p.attempted} txns=${p.transactionsExtracted ?? "-"}`);
+    }
+
+    // Buffer-leakage check
+    const serialized = JSON.stringify(raw);
+    if (serialized.includes("FAKE-PDF-BYTES-DO-NOT-LEAK")) {
+      console.log("\nBUFFER LEAK ✗ — raw PDF bytes survived into payload!");
+      process.exitCode = 1;
+    } else {
+      console.log("  buffer stripping       : ✓ no PDF bytes in payload");
+    }
+    // Password-leakage check
+    if (serialized.match(/987654/)) {
+      console.log("  PASSWORD LEAK ✗ — real password survived into payload");
+      process.exitCode = 1;
+    } else {
+      console.log("  password redaction     : ✓ real password not in payload");
+    }
+
+    // Normalize and confirm transactions land in extra.fields
+    const batch = a.normalize(raw);
+    const ev = batch.events[0];
+    if (ev.extra.fields && Array.isArray(ev.extra.fields.transactions)) {
+      console.log(`  normalize → extra.fields.transactions: ${ev.extra.fields.transactions.length} rows ✓`);
+    } else {
+      console.log("  normalize MISSING transactions in extra.fields ✗");
+      process.exitCode = 1;
+    }
+  }
+
+  if (count === 1 && !process.exitCode) {
+    console.log("\n== Phase 5.5 smoke PASSED ==");
+  } else if (!process.exitCode) {
+    console.log(`expected 1 email, got ${count}`);
+    process.exitCode = 1;
+  }
+}
+
+main().catch((err) => {
+  console.error("smoke failed:", err);
+  process.exitCode = 1;
+});

package/scripts/smoke-phase-5-7.js

@@ -0,0 +1,181 @@
+#!/usr/bin/env node
+/**
+ * Phase 5.7 smoke — exercises retry-with-backoff + onProgress through a
+ * flaky session. No real IMAP needed.
+ *
+ *   1. First 2 connect attempts throw ECONNRESET (transient)
+ *   2. 3rd attempt succeeds, yields 5 envelopes
+ *   3. Verify progress events fire in order: connecting → error → connecting
+ *      → error → connecting → connected → mailbox-opened → fetching × 5 → done
+ *   4. Verify retry was capped (3 attempts total)
+ *   5. Verify AUTH_FAILED does NOT retry (separate run)
+ */
+
+"use strict";
+
+const { EmailAdapter } = require("../lib/adapters/email-imap/email-adapter");
+const { ImapAuthFailedError } = require("../lib/adapters/email-imap/imap-session");
+
+function makeFlakyFactory(failuresFirst, envelopes) {
+  const failures = failuresFirst.slice();
+  const recorder = { attempts: 0 };
+  const factory = () => {
+    let openMb = null;
+    return {
+      async connect() {
+        recorder.attempts += 1;
+        if (failures.length > 0) {
+          throw failures.shift();
+        }
+      },
+      async openMailbox(name) {
+        openMb = name;
+        return { uidValidity: 1, uidNext: 9999, exists: envelopes.length };
+      },
+      async *fetchFullSince(sinceUid = 0) {
+        for (const env of envelopes) {
+          if (env.uid > sinceUid) yield { ...env, source: env.source || Buffer.alloc(0) };
+        }
+      },
+      async close() {},
+    };
+  };
+  return { factory, recorder };
+}
+
+function makeEnv(uid) {
+  return {
+    uid,
+    internalDate: new Date(`2026-05-${String(uid).padStart(2, "0")}T10:00:00Z`),
+    flags: ["\\Seen"],
+    messageId: `<m-${uid}@x>`,
+    subject: `Subject ${uid}`,
+    from: [{ address: `s${uid}@example.com` }],
+    to: [{ address: "me@example.com" }],
+    cc: [],
+    date: new Date(`2026-05-${String(uid).padStart(2, "0")}T10:00:00Z`),
+    size: 1024,
+  };
+}
+
+async function scenarioRetrySucceeds() {
+  console.log("\n=== Scenario A — 2 fails then success, 5 envelopes ===");
+  const transient = Object.assign(new Error("ECONNRESET"), { code: "ECONNRESET" });
+  const envs = [1, 2, 3, 4, 5].map(makeEnv);
+  const { factory, recorder } = makeFlakyFactory([transient, transient], envs);
+
+  const events = [];
+  const a = new EmailAdapter({
+    account: { provider: "qq", email: "me@qq.com", authCode: "x", folders: ["INBOX"] },
+    sessionFactory: factory,
+    parser: async () => ({ textBody: "", attachments: [] }),
+    maxConnectRetries: 3,
+    retryBaseDelayMs: 5,
+    onProgress: (e) => events.push(e),
+  });
+
+  console.log("adapter.version =", a.version);
+  console.log("adapter.capabilities (Phase 5.7) =", a.capabilities.filter((c) => c.startsWith("sync:")).join(", "));
+
+  const raws = [];
+  for await (const r of a.sync()) raws.push(r);
+  console.log(`emitted ${raws.length} raws (expected 5)`);
+  console.log(`connect attempts: ${recorder.attempts} (expected 3)`);
+
+  const phaseSeq = events.map((e) => `${e.phase}${e.attempt ? "(" + e.attempt + ")" : ""}`);
+  console.log("phase sequence:");
+  for (const p of phaseSeq) console.log("  -", p);
+
+  // Verify expected phase order
+  const errs = events.filter((e) => e.phase === "error");
+  if (errs.length !== 2) {
+    console.error(`FAIL: expected 2 error events, got ${errs.length}`);
+    process.exitCode = 1;
+  } else if (!errs.every((e) => e.retriable === true)) {
+    console.error("FAIL: error events should be retriable=true during first 2 attempts");
+    process.exitCode = 1;
+  } else {
+    console.log("error events: ✓ both marked retriable");
+  }
+
+  const done = events.find((e) => e.phase === "done");
+  if (!done || done.emitted !== 5) {
+    console.error(`FAIL: expected done event with emitted=5, got ${JSON.stringify(done)}`);
+    process.exitCode = 1;
+  } else {
+    console.log(`done event: ✓ emitted=${done.emitted} durationMs=${done.durationMs}`);
+  }
+
+  const fetches = events.filter((e) => e.phase === "fetching");
+  if (fetches.length !== 5) {
+    console.error(`FAIL: expected 5 fetching events, got ${fetches.length}`);
+    process.exitCode = 1;
+  } else if (fetches[0].total !== 5 || fetches[4].current !== 5) {
+    console.error(`FAIL: fetching events should run 1..5 of 5`);
+    process.exitCode = 1;
+  } else {
+    console.log("fetching events: ✓ 5 events with current/total");
+  }
+}
+
+async function scenarioAuthFailedNoRetry() {
+  console.log("\n=== Scenario B — AUTH_FAILED never retries ===");
+  const authErr = new ImapAuthFailedError("bad creds");
+  const { factory, recorder } = makeFlakyFactory(
+    [authErr, authErr, authErr], // shouldn't matter — first one stops us
+    [],
+  );
+
+  const events = [];
+  const a = new EmailAdapter({
+    account: { provider: "qq", email: "me@qq.com", authCode: "x", folders: ["INBOX"] },
+    sessionFactory: factory,
+    parser: async () => ({}),
+    maxConnectRetries: 3,
+    retryBaseDelayMs: 1,
+    onProgress: (e) => events.push(e),
+  });
+
+  let caught = null;
+  try {
+    for await (const _r of a.sync()) { /* drain */ }
+  } catch (err) {
+    caught = err;
+  }
+  console.log(`connect attempts: ${recorder.attempts} (expected 1)`);
+  console.log(`caught.code: ${caught && caught.code}`);
+  if (recorder.attempts !== 1) {
+    console.error("FAIL: AUTH_FAILED should not retry");
+    process.exitCode = 1;
+  } else if (!caught || caught.code !== "AUTH_FAILED") {
+    console.error("FAIL: error should propagate as AUTH_FAILED");
+    process.exitCode = 1;
+  } else {
+    console.log("AUTH short-circuit: ✓");
+  }
+
+  const errEvent = events.find((e) => e.phase === "error");
+  if (!errEvent) {
+    console.error("FAIL: error progress event missing");
+    process.exitCode = 1;
+  } else if (errEvent.retriable !== false) {
+    console.error("FAIL: AUTH_FAILED error event should have retriable=false");
+    process.exitCode = 1;
+  } else {
+    console.log("AUTH error event: ✓ retriable=false");
+  }
+}
+
+async function main() {
+  console.log("== Phase 5.7 smoke (retry + progress) ==");
+  await scenarioRetrySucceeds();
+  await scenarioAuthFailedNoRetry();
+  if (!process.exitCode) {
+    console.log("\n== Phase 5.7 smoke PASSED ==");
+  }
+}
+
+main().catch((err) => {
+  console.error("smoke crashed:", err);
+  process.exitCode = 1;
+});