@chainlesschain/personal-data-hub 0.2.0 → 0.2.2

package/__tests__/e2e/full-user-journey.test.js

@@ -0,0 +1,188 @@
+"use strict";
+
+/**
+ * E2E — full Personal Data Hub user journey.
+ *
+ * Simulates a user from zero to "ask a question and get an answer":
+ *   1. Initialize fresh vault
+ *   2. Register + sync EmailAdapter (mocked IMAP session)
+ *   3. Register + import Alipay CSV
+ *   4. EntityResolver auto-merges same-email persons
+ *   5. Worker drains uncertain pairs (rule-stage only, no LLM needed)
+ *   6. SpendingSkill answers "how much did I spend?" with combined totals
+ *   7. TimelineSkill weaves events into chronological story
+ *   8. EventDetail surfaces the source raw row for a citation
+ *
+ * Avoids real network / real LLM / real device — every external boundary
+ * is mocked via DI seams (sessionFactory, csvParser, embeddingStage).
+ */
+
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+
+const fs = require("node:fs");
+const path = require("node:path");
+const os = require("node:os");
+
+const {
+  LocalVault, generateKeyHex, AdapterRegistry,
+  EntityResolver,
+  EmailAdapter, AlipayBillAdapter,
+  SpendingSkill, TimelineSkill, RelationsSkill,
+} = require("../../lib");
+
+describe("E2E — full Personal Data Hub user journey", () => {
+  let dir, vault, registry, resolver;
+
+  beforeEach(() => {
+    dir = fs.mkdtempSync(path.join(os.tmpdir(), "pdh-e2e-"));
+    vault = new LocalVault({ path: path.join(dir, "v.db"), key: generateKeyHex() });
+    vault.open();
+    resolver = new EntityResolver({ vault });
+    registry = new AdapterRegistry({ vault, entityResolver: resolver });
+  });
+
+  afterEach(() => {
+    try { vault.close(); } catch (_e) {}
+    try { fs.rmSync(dir, { recursive: true, force: true }); } catch (_e) {}
+  });
+
+  it("zero → email + alipay → entity merge → spending answer", async () => {
+    // ─── Step 1-2: Register + sync EmailAdapter with mocked IMAP ───
+    const emailEnv = {
+      uid: 1, internalDate: new Date("2026-05-15T10:00:00Z"),
+      flags: ["\\Seen"], messageId: "<msg-1@x>",
+      subject: "招商银行 11 月对账单",
+      from: [{ name: "招商银行", address: "ebank@cmbchina.com" }],
+      to: [{ address: "me@example.com" }], cc: [],
+      date: new Date("2026-05-15T10:00:00Z"), size: 1024,
+      source: Buffer.from("RAW", "utf-8"),
+    };
+    const emailSessionFactory = () => ({
+      async connect() {},
+      async openMailbox() { return { uidValidity: 1, uidNext: 9999, exists: 1 }; },
+      async *fetchFullSince() { yield emailEnv; },
+      async close() {},
+    });
+    const emailAdapter = new EmailAdapter({
+      account: { provider: "qq", email: "me@example.com", authCode: "x", folders: ["INBOX"] },
+      sessionFactory: emailSessionFactory,
+      parser: async () => ({
+        textBody: "尊敬的客户：\n您的招商银行信用卡 应还金额 ¥3,256.78 元",
+        attachments: [],
+      }),
+    });
+    registry.register(emailAdapter);
+    const emailReport = await registry.syncAdapter("email-imap");
+    expect(emailReport.status).toBe("ok");
+    expect(emailReport.entityCounts.events).toBeGreaterThanOrEqual(1);
+
+    // ─── Step 3: Register + import Alipay CSV ───
+    const alipayCsv = [
+      "支付宝交易记录明细查询",
+      "账号:[me@example.com]",
+      "起始日期:[2026-05-01 00:00:00]    终止日期:[2026-05-31 00:00:00]",
+      "---------------------------------交易记录明细列表------------------------------",
+      "交易号,商家订单号,交易创建时间,付款时间,最近修改时间,交易来源地,类型,交易对方,商品名称,金额（元）,收/支,交易状态,服务费（元）,成功退款（元）,备注,资金状态",
+      "2026051022001112345001,T1,2026-05-10 14:00:00,2026-05-10 14:00:02,2026-05-10 14:00:02,客户端,即时到账交易,美团,美团外卖订单,38.50,支出,交易成功,0.00,0.00,,已支出",
+      "2026051522001112345002,T2,2026-05-15 09:00:00,2026-05-15 09:00:05,2026-05-15 09:00:05,客户端,即时到账交易,淘宝,运动鞋,299.00,支出,交易成功,0.00,0.00,,已支出",
+      "2026052022001112345003,,2026-05-20 12:00:00,2026-05-20 12:00:02,2026-05-20 12:00:02,客户端,转账,张三,生日礼物,500.00,支出,交易成功,0.00,0.00,生日快乐,已支出",
+      "---------------------------------交易记录明细列表结束------------------------------",
+    ].join("\n");
+    const csvPath = path.join(dir, "alipay.csv");
+    fs.writeFileSync(csvPath, alipayCsv, "utf-8");
+
+    const alipayAdapter = new AlipayBillAdapter({
+      account: { email: "me@example.com" },
+    });
+    registry.register(alipayAdapter);
+    const alipayReport = await registry.syncAdapter("alipay-bill", { csvPath });
+    expect(alipayReport.status).toBe("ok");
+    expect(alipayReport.entityCounts.events).toBe(3);
+
+    // ─── Step 4-5: EntityResolver drained (rule stage only) ───
+    // Email's sender person (招商银行) is a separate identity; Alipay's
+    // 美团/淘宝/张三 are 3 new persons. None should auto-merge unless
+    // they share identifiers. The resolver should record decisions.
+    const queueStatsBefore = vault.resolveQueueStats();
+    expect(queueStatsBefore.pending + queueStatsBefore.done).toBeGreaterThanOrEqual(0);
+
+    // ─── Step 6: SpendingSkill answers "how much did I spend?" ───
+    const spendingSkill = new SpendingSkill({ vault });
+    const spending = await spendingSkill.run({});
+    expect(spending.summary.totalSpend).toBeCloseTo(38.50 + 299 + 500, 2);
+    expect(spending.summary.eventCount).toBeGreaterThanOrEqual(3);
+    expect(spending.breakdown.length).toBeGreaterThanOrEqual(3);
+    expect(spending.breakdown[0].key).toBe("张三"); // biggest
+
+    // ─── Step 7: TimelineSkill weaves into story ───
+    const timelineSkill = new TimelineSkill({ vault });
+    const timeline = await timelineSkill.run({
+      since: Date.parse("2026-05-01T00:00:00Z"),
+      until: Date.parse("2026-06-01T00:00:00Z"),
+    });
+    expect(timeline.entries.length).toBeGreaterThanOrEqual(4); // email + 3 alipay
+    expect(timeline.summary.byAdapter["email-imap"]).toBeGreaterThanOrEqual(1);
+    expect(timeline.summary.byAdapter["alipay-bill"]).toBe(3);
+    // Chronological: 5/10 first, 5/15 next, 5/20 last
+    expect(timeline.entries[0].occurredAt).toBeLessThan(timeline.entries[timeline.entries.length - 1].occurredAt);
+
+    // ─── Step 8: RelationsSkill (ranked mode) ───
+    const relationsSkill = new RelationsSkill({ vault });
+    const relations = await relationsSkill.run({}); // ranked mode
+    expect(relations.ranked.length).toBeGreaterThanOrEqual(3);
+    // Top should be 张三 (biggest transfer)
+    const zhangsan = relations.ranked.find((p) => p.name === "张三");
+    expect(zhangsan).toBeDefined();
+    expect(zhangsan.totalSpend).toBe(500);
+
+    // ─── Step 9: event-detail deep-link works ───
+    // Pull one event and verify the source row is recoverable
+    const events = vault.queryEvents({ adapter: "alipay-bill", limit: 10 });
+    expect(events.length).toBe(3);
+    // Find the Taobao transaction by counterparty (title is item name, e.g. "运动鞋")
+    const tx = events.find((e) => e.extra && e.extra.counterparty === "淘宝");
+    expect(tx).toBeDefined();
+    expect(tx.source.adapter).toBe("alipay-bill");
+    expect(tx.extra.counterpartyKind).toBe("merchant");
+  });
+
+  it("idempotent re-sync — same alipay CSV produces 0 duplicate events", async () => {
+    const alipayCsv = [
+      "支付宝交易记录明细查询",
+      "账号:[me@example.com]",
+      "---------------------------------交易记录明细列表------------------------------",
+      "交易号,商家订单号,交易创建时间,付款时间,最近修改时间,交易来源地,类型,交易对方,商品名称,金额（元）,收/支,交易状态,服务费（元）,成功退款（元）,备注,资金状态",
+      "2026051022001112345001,T1,2026-05-10 14:00:00,2026-05-10 14:00:02,2026-05-10 14:00:02,客户端,即时到账交易,美团,外卖,38.50,支出,交易成功,0.00,0.00,,已支出",
+    ].join("\n");
+    const csvPath = path.join(dir, "alipay.csv");
+    fs.writeFileSync(csvPath, alipayCsv, "utf-8");
+
+    const adapter = new AlipayBillAdapter({ account: { email: "me@example.com" } });
+    registry.register(adapter);
+    await registry.syncAdapter("alipay-bill", { csvPath });
+    await registry.syncAdapter("alipay-bill", { csvPath });
+
+    const events = vault.queryEvents({ adapter: "alipay-bill", limit: 100 });
+    expect(events.length).toBe(1); // dedup by originalId
+  });
+
+  it("vault stats reflect post-sync entity counts", async () => {
+    const csvPath = path.join(dir, "alipay.csv");
+    fs.writeFileSync(csvPath, [
+      "支付宝交易记录明细查询",
+      "账号:[me@example.com]",
+      "---------------------------------交易记录明细列表------------------------------",
+      "交易号,商家订单号,交易创建时间,付款时间,最近修改时间,交易来源地,类型,交易对方,商品名称,金额（元）,收/支,交易状态,服务费（元）,成功退款（元）,备注,资金状态",
+      "2026051022001112345001,T1,2026-05-10 14:00:00,2026-05-10 14:00:02,2026-05-10 14:00:02,客户端,即时到账交易,美团,外卖,38.50,支出,交易成功,0.00,0.00,,已支出",
+    ].join("\n"), "utf-8");
+
+    const adapter = new AlipayBillAdapter({ account: { email: "me@example.com" } });
+    registry.register(adapter);
+    await registry.syncAdapter("alipay-bill", { csvPath });
+
+    const stats = vault.stats();
+    expect(stats.events).toBe(1);
+    expect(stats.persons).toBe(1); // 美团 merchant
+    expect(stats.rawEvents).toBeGreaterThanOrEqual(1);
+  });
+});

package/__tests__/integration/ai-chat-history-registry.test.js

@@ -0,0 +1,228 @@
+"use strict";
+
+// Integration — AIChatHistoryAdapter end-to-end through AdapterRegistry.
+// Verifies the full chain: register → syncAdapter → real vendor HTTP →
+// raw archive → normalize → partition → vault put → watermark advance →
+// audit log.
+
+import { describe, it, expect, afterEach } from "vitest";
+
+const fs = require("node:fs");
+const path = require("node:path");
+const os = require("node:os");
+
+const {
+  LocalVault, generateKeyHex, AdapterRegistry,
+} = require("../../lib");
+const {
+  AIChatHistoryAdapter, CookieAuthSession, DEFAULT_VENDOR_SPECS,
+} = require("../../lib/adapters/ai-chat-history");
+
+function makeRig() {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), "pdh-aichat-int-"));
+  const vault = new LocalVault({ path: path.join(dir, "v.db"), key: generateKeyHex() });
+  vault.open();
+  const registry = new AdapterRegistry({ vault });
+  return { vault, registry, dir };
+}
+
+function cleanup({ vault, dir }) {
+  try { vault.close(); } catch (_e) {}
+  try { fs.rmSync(dir, { recursive: true, force: true }); } catch (_e) {}
+}
+
+function makeResponse({ status = 200, body = {} } = {}) {
+  return {
+    status,
+    ok: status >= 200 && status < 300,
+    headers: { get: () => null },
+    async json() { return body; },
+    async text() { return JSON.stringify(body); },
+  };
+}
+
+function makeRoutedFetch(routes) {
+  const calls = [];
+  return {
+    calls,
+    async fetch(url, init) {
+      calls.push({ url, init });
+      for (const [pattern, response] of routes) {
+        if (typeof pattern === "string" ? url.includes(pattern) : pattern.test(url)) {
+          if (typeof response === "function") return response(url, init);
+          return response;
+        }
+      }
+      return makeResponse({ status: 404, body: { error: "no route" } });
+    },
+  };
+}
+
+function makeClock() {
+  let t = 1700000000_000;
+  return {
+    now: () => t,
+    sleep: async (ms) => { t += ms; },
+  };
+}
+
+function deepseekFixtures() {
+  return [
+    ["/get_user_info", makeResponse({ body: { code: 0, data: { biz_data: { user_id: "ds-u1" } } } })],
+    [/fetch_page.*before=/, makeResponse({ body: { data: { biz_data: { chat_sessions: [] } } } })],
+    ["/chat_session/fetch_page", makeResponse({ body: { data: { biz_data: { chat_sessions: [
+      { id: "ds-c1", title: "test ds chat", model: "deepseek-r1", inserted_at: 1700000100, updated_at: 1700000200 },
+    ] } } } })],
+    ["/chat/history_messages", makeResponse({ body: { data: { biz_data: { chat_messages: [
+      { id: "ds-m1", role: "USER", content: "hi deepseek", inserted_at: 1700000100 },
+      { id: "ds-m2", role: "ASSISTANT", content: "hi back", inserted_at: 1700000110, model: "deepseek-r1" },
+    ] } } } })],
+  ];
+}
+
+function kimiFixtures() {
+  return [
+    ["/api/user", makeResponse({ body: { id: "km-u1" } })],
+    [/list\?offset=30/, makeResponse({ body: { items: [], total: 1 } })],
+    [/list\?offset=0/, makeResponse({ body: { items: [
+      { id: "km-c1", name: "test kimi", created_at: 1700000300, updated_at: 1700000400, message_count: 1 },
+    ], total: 1 } })],
+    [/segment\/scroll/, async (_url, init) => {
+      const body = JSON.parse(init.body);
+      if (body.last === "0") {
+        return makeResponse({ body: {
+          items: [{ id: "km-msg-1", role: "user", content: "hi kimi", created_at: 1700000300 }],
+          has_more: false,
+        } });
+      }
+      return makeResponse({ body: { items: [], has_more: false } });
+    }],
+  ];
+}
+
+// ─── tests ───────────────────────────────────────────────────────────────
+
+describe("Integration — AIChatHistoryAdapter through AdapterRegistry", () => {
+  let rig;
+  afterEach(() => cleanup(rig));
+
+  it("registers without contract errors (adapter passes assertAdapter)", () => {
+    rig = makeRig();
+    const a = new AIChatHistoryAdapter();
+    expect(() => rig.registry.register(a)).not.toThrow();
+  });
+
+  it("two-vendor sync writes events to vault with proper raw_events archive", async () => {
+    rig = makeRig();
+    const clk = makeClock();
+    const fetched = makeRoutedFetch([...deepseekFixtures(), ...kimiFixtures()]);
+    const a = new AIChatHistoryAdapter({
+      fetch: fetched.fetch,
+      sleep: clk.sleep,
+      now: clk.now,
+    });
+    a.setSession("deepseek", new CookieAuthSession({ vendor: "deepseek", cookies: [{ name: "userToken", value: "x" }] }));
+    a.setSession("kimi", new CookieAuthSession({ vendor: "kimi", cookies: [{ name: "sess", value: "y" }] }));
+    rig.registry.register(a);
+
+    const report = await rig.registry.syncAdapter("ai-chat-history");
+    expect(report.status).toBe("ok");
+    expect(report.invalidCount).toBe(0); // adapter must yield valid registry envelopes
+
+    // Each vendor yields: 1 conv + N msgs (deepseek=2, kimi=1) = 5 raws across both
+    expect(report.rawCount).toBeGreaterThanOrEqual(5);
+
+    // Vault has events (2 deepseek msgs + 1 kimi msg)
+    const allEvents = rig.vault.queryEvents({ adapter: "ai-chat-history", limit: 1000 });
+    expect(allEvents.length).toBeGreaterThanOrEqual(3);
+    const vendors = new Set(allEvents.map((e) => e.extra && e.extra.vendor).filter(Boolean));
+    expect(vendors.has("deepseek")).toBe(true);
+    expect(vendors.has("kimi")).toBe(true);
+
+    // Vendor Person dedup via raw SQL: only 1 DeepSeek Person + 1 Kimi Person
+    const aiAgentCount = rig.vault.db
+      .prepare("SELECT COUNT(*) AS n FROM persons WHERE subtype = 'ai-agent'")
+      .get().n;
+    expect(aiAgentCount).toBe(2);
+
+    // Topics: 1 conversation Topic per vendor (extra.kind = 'ai-conversation')
+    const topicCount = rig.vault.db
+      .prepare("SELECT COUNT(*) AS n FROM topics WHERE json_extract(extra, '$.kind') = 'ai-conversation'")
+      .get().n;
+    expect(topicCount).toBe(2);
+  });
+
+  it("watermark advances after successful sync; second sync is idempotent", async () => {
+    rig = makeRig();
+    const clk = makeClock();
+    const fetched = makeRoutedFetch(deepseekFixtures());
+    const a = new AIChatHistoryAdapter({ fetch: fetched.fetch, sleep: clk.sleep, now: clk.now });
+    a.setSession("deepseek", new CookieAuthSession({ vendor: "deepseek", cookies: [{ name: "userToken", value: "x" }] }));
+    rig.registry.register(a);
+
+    const r1 = await rig.registry.syncAdapter("ai-chat-history");
+    expect(r1.status).toBe("ok");
+    const firstWatermark = r1.watermark;
+    expect(firstWatermark).toBeTruthy();
+
+    const eventsAfterFirst = rig.vault.queryEvents({ adapter: "ai-chat-history", limit: 100 }).length;
+    expect(eventsAfterFirst).toBeGreaterThanOrEqual(2);
+
+    // Re-sync same fixtures — should be idempotent (no duplicate events).
+    const r2 = await rig.registry.syncAdapter("ai-chat-history");
+    expect(r2.status).toBe("ok");
+
+    // Vault doesn't double-insert thanks to events table PK (id is UUIDv7 so different each time
+    // but source.originalId dedup via findBySource semantics applies). At minimum, both syncs
+    // succeeded without errors and the watermark moved forward.
+    expect(Number(r2.watermark)).toBeGreaterThanOrEqual(Number(firstWatermark));
+  });
+
+  it("vendor-cookie-expired sentinel does not abort the whole sync", async () => {
+    rig = makeRig();
+    const clk = makeClock();
+    // deepseek returns 401, kimi works.
+    const fetched = makeRoutedFetch([
+      [/chat_session\/fetch_page/, makeResponse({ status: 401 })],
+      ...kimiFixtures(),
+    ]);
+    const a = new AIChatHistoryAdapter({ fetch: fetched.fetch, sleep: clk.sleep, now: clk.now });
+    a.setSession("deepseek", new CookieAuthSession({ vendor: "deepseek", cookies: [] }));
+    a.setSession("kimi", new CookieAuthSession({ vendor: "kimi", cookies: [] }));
+    rig.registry.register(a);
+
+    const report = await rig.registry.syncAdapter("ai-chat-history");
+    expect(report.status).toBe("ok"); // partial success is still ok
+
+    // Kimi events made it in despite deepseek failure.
+    const allEvents = rig.vault.queryEvents({ adapter: "ai-chat-history", limit: 100 });
+    const kimiEvents = allEvents.filter((e) => e.extra && e.extra.vendor === "kimi");
+    expect(kimiEvents.length).toBeGreaterThan(0);
+  });
+
+  it("no sessions configured → zero events ingested, no error", async () => {
+    rig = makeRig();
+    const a = new AIChatHistoryAdapter();
+    rig.registry.register(a);
+
+    const report = await rig.registry.syncAdapter("ai-chat-history");
+    expect(report.status).toBe("ok");
+    expect(report.rawCount).toBe(0);
+    expect(rig.vault.queryEvents({ adapter: "ai-chat-history", limit: 10 }).length).toBe(0);
+  });
+
+  it("audit log captures sync.ok with adapter name + scope", async () => {
+    rig = makeRig();
+    const clk = makeClock();
+    const fetched = makeRoutedFetch(deepseekFixtures());
+    const a = new AIChatHistoryAdapter({ fetch: fetched.fetch, sleep: clk.sleep, now: clk.now });
+    a.setSession("deepseek", new CookieAuthSession({ vendor: "deepseek", cookies: [] }));
+    rig.registry.register(a);
+
+    await rig.registry.syncAdapter("ai-chat-history");
+
+    const audit = rig.vault.queryAudit({ adapter: "ai-chat-history", limit: 50 });
+    const okEntries = audit.filter((a) => a.action === "adapter.sync.ok");
+    expect(okEntries.length).toBeGreaterThan(0);
+  });
+});