@chainlesschain/personal-data-hub 0.1.0 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (154) hide show
  1. package/__tests__/adapters/ai-chat-cookie-capture-spec.test.js +211 -0
  2. package/__tests__/adapters/ai-chat-health-checker.test.js +262 -0
  3. package/__tests__/adapters/ai-chat-history.test.js +396 -0
  4. package/__tests__/adapters/ai-chat-http-client.test.js +242 -0
  5. package/__tests__/adapters/ai-chat-vendors.test.js +874 -0
  6. package/__tests__/adapters/alipay-bill-adapter.test.js +538 -0
  7. package/__tests__/adapters/email-adapter.test.js +138 -1
  8. package/__tests__/adapters/email-classifier.test.js +347 -0
  9. package/__tests__/adapters/email-pdf-extractor.test.js +529 -0
  10. package/__tests__/adapters/email-retry-progress.test.js +294 -0
  11. package/__tests__/adapters/email-templates.test.js +699 -0
  12. package/__tests__/adapters/social-toutiao-kuaishou-scaffold.test.js +269 -0
  13. package/__tests__/adapters/system-data-adapter.test.js +440 -0
  14. package/__tests__/adapters/system-data-android-ingest.test.js +144 -0
  15. package/__tests__/adapters/system-data-android.test.js +387 -0
  16. package/__tests__/adapters/system-data-disclosure.test.js +153 -0
  17. package/__tests__/adapters/wechat-bootstrap.test.js +240 -0
  18. package/__tests__/adapters/wechat-env-probe.test.js +162 -0
  19. package/__tests__/adapters/wechat-frida-agent.test.js +191 -0
  20. package/__tests__/adapters/wechat-frida-integration.test.js +149 -0
  21. package/__tests__/adapters/wechat-frida-key-provider.test.js +188 -0
  22. package/__tests__/adapters/wechat-md5-key-provider.test.js +101 -0
  23. package/__tests__/analysis-skills.test.js +556 -0
  24. package/__tests__/analysis.test.js +329 -1
  25. package/__tests__/e2e/ai-chat-cross-source-journey.test.js +213 -0
  26. package/__tests__/e2e/full-user-journey.test.js +188 -0
  27. package/__tests__/entity-resolver-ingest-hook.test.js +177 -0
  28. package/__tests__/entity-resolver-stages.test.js +411 -0
  29. package/__tests__/entity-resolver-vault.test.js +246 -0
  30. package/__tests__/entity-resolver.test.js +526 -0
  31. package/__tests__/fixtures/entity-resolver-200-mock.json +96 -0
  32. package/__tests__/integration/ai-chat-history-registry.test.js +228 -0
  33. package/__tests__/integration/aichat-wizard-end-to-end.test.js +282 -0
  34. package/__tests__/integration/cross-adapter-pipelines.test.js +396 -0
  35. package/__tests__/integration/wechat-bootstrap-end-to-end.test.js +390 -0
  36. package/__tests__/longtail-adapters.test.js +217 -0
  37. package/__tests__/mobile-extractor.test.js +288 -0
  38. package/__tests__/registry.test.js +4 -2
  39. package/__tests__/shopping-adapters.test.js +296 -0
  40. package/__tests__/sidecar-contacts-cross-validate.test.js +163 -0
  41. package/__tests__/sidecar-supervisor.test.js +120 -0
  42. package/__tests__/social-adapters.test.js +206 -0
  43. package/__tests__/travel-adapters.test.js +325 -0
  44. package/__tests__/vault.test.js +3 -3
  45. package/__tests__/wechat-adapter.test.js +476 -0
  46. package/__tests__/whatsapp-adapter.test.js +135 -0
  47. package/lib/adapter-spec.js +12 -0
  48. package/lib/adapters/_python-sidecar-base.js +207 -0
  49. package/lib/adapters/ai-chat-history/ai-chat-adapter.js +374 -0
  50. package/lib/adapters/ai-chat-history/cookie-auth.js +109 -0
  51. package/lib/adapters/ai-chat-history/cookie-capture-spec.js +331 -0
  52. package/lib/adapters/ai-chat-history/health-checker.js +210 -0
  53. package/lib/adapters/ai-chat-history/http-client.js +211 -0
  54. package/lib/adapters/ai-chat-history/index.js +28 -0
  55. package/lib/adapters/ai-chat-history/schema-map.js +258 -0
  56. package/lib/adapters/ai-chat-history/vendor-spec.js +86 -0
  57. package/lib/adapters/ai-chat-history/vendors/coze.js +179 -0
  58. package/lib/adapters/ai-chat-history/vendors/deepseek.js +199 -0
  59. package/lib/adapters/ai-chat-history/vendors/doubao.js +255 -0
  60. package/lib/adapters/ai-chat-history/vendors/dreamina.js +174 -0
  61. package/lib/adapters/ai-chat-history/vendors/hunyuan.js +176 -0
  62. package/lib/adapters/ai-chat-history/vendors/kimi.js +182 -0
  63. package/lib/adapters/ai-chat-history/vendors/qianfan.js +160 -0
  64. package/lib/adapters/ai-chat-history/vendors/tongyi.js +193 -0
  65. package/lib/adapters/ai-chat-history/vendors/zhipu.js +202 -0
  66. package/lib/adapters/ai-chat-history/wizard-controller.js +473 -0
  67. package/lib/adapters/alipay-bill/alipay-bill-adapter.js +311 -0
  68. package/lib/adapters/alipay-bill/counterparty.js +129 -0
  69. package/lib/adapters/alipay-bill/csv-parser.js +217 -0
  70. package/lib/adapters/alipay-bill/index.js +41 -0
  71. package/lib/adapters/alipay-bill/zip-decryptor.js +111 -0
  72. package/lib/adapters/email-imap/classifier.js +495 -0
  73. package/lib/adapters/email-imap/email-adapter.js +419 -8
  74. package/lib/adapters/email-imap/index.js +42 -0
  75. package/lib/adapters/email-imap/pdf-extractor.js +192 -0
  76. package/lib/adapters/email-imap/templates/bill.js +232 -0
  77. package/lib/adapters/email-imap/templates/government.js +120 -0
  78. package/lib/adapters/email-imap/templates/index.js +78 -0
  79. package/lib/adapters/email-imap/templates/order.js +186 -0
  80. package/lib/adapters/email-imap/templates/other.js +114 -0
  81. package/lib/adapters/email-imap/templates/register.js +113 -0
  82. package/lib/adapters/email-imap/templates/travel.js +157 -0
  83. package/lib/adapters/email-imap/templates/utils.js +275 -0
  84. package/lib/adapters/email-imap/transactions.js +234 -0
  85. package/lib/adapters/messaging-qq/index.js +158 -0
  86. package/lib/adapters/messaging-telegram/index.js +142 -0
  87. package/lib/adapters/messaging-whatsapp/index.js +189 -0
  88. package/lib/adapters/shopping-base/index.js +208 -0
  89. package/lib/adapters/shopping-jd/index.js +150 -0
  90. package/lib/adapters/shopping-meituan/index.js +154 -0
  91. package/lib/adapters/shopping-taobao/index.js +176 -0
  92. package/lib/adapters/social-bilibili/index.js +171 -0
  93. package/lib/adapters/social-douyin/index.js +116 -0
  94. package/lib/adapters/social-kuaishou/index.js +237 -0
  95. package/lib/adapters/social-toutiao/index.js +236 -0
  96. package/lib/adapters/social-weibo/index.js +164 -0
  97. package/lib/adapters/social-xiaohongshu/index.js +96 -0
  98. package/lib/adapters/system-data/disclosure.js +166 -0
  99. package/lib/adapters/system-data/index.js +34 -0
  100. package/lib/adapters/system-data/system-data-adapter.js +344 -0
  101. package/lib/adapters/system-data-android/adapter.js +348 -0
  102. package/lib/adapters/system-data-android/index.js +76 -0
  103. package/lib/adapters/travel-12306/index.js +151 -0
  104. package/lib/adapters/travel-amap/index.js +164 -0
  105. package/lib/adapters/travel-baidu-map/index.js +162 -0
  106. package/lib/adapters/travel-base/index.js +240 -0
  107. package/lib/adapters/travel-ctrip/index.js +151 -0
  108. package/lib/adapters/wechat/bootstrap.js +146 -0
  109. package/lib/adapters/wechat/content-parser.js +326 -0
  110. package/lib/adapters/wechat/db-reader.js +209 -0
  111. package/lib/adapters/wechat/env-probe.js +218 -0
  112. package/lib/adapters/wechat/frida-agent/loader.js +67 -0
  113. package/lib/adapters/wechat/frida-agent/wechat-key-hook.js +126 -0
  114. package/lib/adapters/wechat/index.js +37 -0
  115. package/lib/adapters/wechat/key-extractor.js +158 -0
  116. package/lib/adapters/wechat/key-providers/frida-key-provider.js +244 -0
  117. package/lib/adapters/wechat/key-providers/index.js +22 -0
  118. package/lib/adapters/wechat/key-providers/key-provider-base.js +44 -0
  119. package/lib/adapters/wechat/key-providers/md5-key-provider.js +81 -0
  120. package/lib/adapters/wechat/normalize.js +220 -0
  121. package/lib/adapters/wechat/wechat-adapter.js +205 -0
  122. package/lib/analysis-skills/base.js +113 -0
  123. package/lib/analysis-skills/footprint.js +167 -0
  124. package/lib/analysis-skills/index.js +58 -0
  125. package/lib/analysis-skills/interests.js +161 -0
  126. package/lib/analysis-skills/relations.js +226 -0
  127. package/lib/analysis-skills/spending.js +219 -0
  128. package/lib/analysis-skills/timeline.js +167 -0
  129. package/lib/analysis.js +191 -2
  130. package/lib/entity-resolver/embedding-stage.js +198 -0
  131. package/lib/entity-resolver/entity-resolver.js +384 -0
  132. package/lib/entity-resolver/index.js +42 -0
  133. package/lib/entity-resolver/llm-stage.js +191 -0
  134. package/lib/entity-resolver/rule-stage.js +208 -0
  135. package/lib/entity-resolver/worker.js +149 -0
  136. package/lib/index.js +131 -0
  137. package/lib/migrations.js +73 -0
  138. package/lib/mobile-extractor/android.js +193 -0
  139. package/lib/mobile-extractor/index.js +9 -0
  140. package/lib/mobile-extractor/ios.js +223 -0
  141. package/lib/prompt-builder.js +11 -1
  142. package/lib/query-parser.js +7 -1
  143. package/lib/registry.js +42 -0
  144. package/lib/sidecar/index.js +15 -0
  145. package/lib/sidecar/supervisor.js +359 -0
  146. package/lib/vault.js +343 -0
  147. package/package.json +36 -3
  148. package/scripts/_make-fixture-all.js +126 -0
  149. package/scripts/_make-fixture-contacts.js +84 -0
  150. package/scripts/evaluate-entity-resolver.js +213 -0
  151. package/scripts/smoke-phase-5-5.js +196 -0
  152. package/scripts/smoke-phase-5-7.js +181 -0
  153. package/scripts/smoke-system-data-contacts.js +309 -0
  154. package/scripts/smoke-system-data.js +312 -0
package/lib/vault.js CHANGED
@@ -37,6 +37,13 @@ const DEFAULT_CIPHER_PAGE_SIZE = 4096;
37
37
 
38
38
  // ─── Helpers ─────────────────────────────────────────────────────────────
39
39
 
40
+ function newGroupId() {
41
+ // Lightweight uuid v4-ish for merge_groups.id. Doesn't need crypto
42
+ // strength — uniqueness within one user's vault is enough.
43
+ const r = () => Math.random().toString(16).slice(2, 10);
44
+ return `mg-${r()}${r()}-${Date.now().toString(36)}`;
45
+ }
46
+
40
47
  function loadDriver() {
41
48
  // Lazy require so consumers that only need schemas don't pay for the
42
49
  // native binding load. Errors surface here with a precise message.
@@ -598,6 +605,83 @@ class LocalVault {
598
605
  .map((row) => this._rowToEvent(row));
599
606
  }
600
607
 
608
+ /**
609
+ * queryPersons — list person entities (contacts, family, colleagues...).
610
+ * Phase 14.x Path C — needed so AnalysisEngine can answer questions about
611
+ * "how many contacts" / "did I call mom last week" without missing the
612
+ * persons-table half of the world.
613
+ *
614
+ * @param {object} q
615
+ * @param {string} [q.subtype] e.g. "contact" / "family" / "colleague"
616
+ * @param {string} [q.adapter] source_adapter filter
617
+ * @param {number} [q.limit=100]
618
+ * @param {number} [q.offset=0]
619
+ */
620
+ queryPersons(q = {}) {
621
+ const where = [];
622
+ const params = {};
623
+ if (q.subtype) {
624
+ where.push("subtype = @subtype");
625
+ params.subtype = q.subtype;
626
+ }
627
+ if (q.adapter) {
628
+ where.push("source_adapter = @adapter");
629
+ params.adapter = q.adapter;
630
+ }
631
+ const limit = Number.isInteger(q.limit) && q.limit > 0 ? Math.min(q.limit, 10000) : 100;
632
+ const offset = Number.isInteger(q.offset) && q.offset >= 0 ? q.offset : 0;
633
+ params.limit = limit;
634
+ params.offset = offset;
635
+ const sql =
636
+ "SELECT * FROM persons" +
637
+ (where.length ? " WHERE " + where.join(" AND ") : "") +
638
+ " ORDER BY ingested_at DESC LIMIT @limit OFFSET @offset";
639
+ return this._requireOpen()
640
+ .prepare(sql)
641
+ .all(params)
642
+ .map((row) => this._rowToPerson(row));
643
+ }
644
+
645
+ /**
646
+ * queryItems — list item entities (installed apps, purchases, media...).
647
+ * Pairs with queryPersons for AnalysisEngine fact gathering.
648
+ *
649
+ * @param {object} q
650
+ * @param {string} [q.subtype]
651
+ * @param {string} [q.adapter]
652
+ * @param {string} [q.category]
653
+ * @param {number} [q.limit=100]
654
+ * @param {number} [q.offset=0]
655
+ */
656
+ queryItems(q = {}) {
657
+ const where = [];
658
+ const params = {};
659
+ if (q.subtype) {
660
+ where.push("subtype = @subtype");
661
+ params.subtype = q.subtype;
662
+ }
663
+ if (q.adapter) {
664
+ where.push("source_adapter = @adapter");
665
+ params.adapter = q.adapter;
666
+ }
667
+ if (q.category) {
668
+ where.push("category = @category");
669
+ params.category = q.category;
670
+ }
671
+ const limit = Number.isInteger(q.limit) && q.limit > 0 ? Math.min(q.limit, 10000) : 100;
672
+ const offset = Number.isInteger(q.offset) && q.offset >= 0 ? q.offset : 0;
673
+ params.limit = limit;
674
+ params.offset = offset;
675
+ const sql =
676
+ "SELECT * FROM items" +
677
+ (where.length ? " WHERE " + where.join(" AND ") : "") +
678
+ " ORDER BY ingested_at DESC LIMIT @limit OFFSET @offset";
679
+ return this._requireOpen()
680
+ .prepare(sql)
681
+ .all(params)
682
+ .map((row) => this._rowToItem(row));
683
+ }
684
+
601
685
  countEvents(q = {}) {
602
686
  const where = [];
603
687
  const params = {};
@@ -716,6 +800,9 @@ class LocalVault {
716
800
  stats() {
717
801
  const db = this._requireOpen();
718
802
  const count = (tbl) => db.prepare(`SELECT COUNT(*) as n FROM ${tbl}`).get().n;
803
+ const safeCount = (tbl) => {
804
+ try { return count(tbl); } catch (_e) { return 0; }
805
+ };
719
806
  return {
720
807
  schemaVersion: getSchemaVersion(db),
721
808
  events: count("events"),
@@ -726,9 +813,265 @@ class LocalVault {
726
813
  rawEvents: count("raw_events"),
727
814
  auditLog: count("audit_log"),
728
815
  watermarks: count("sync_watermarks"),
816
+ // Phase 8 — EntityResolver tables (safeCount because v1 vaults
817
+ // don't have these yet until migrate).
818
+ mergeGroups: safeCount("merge_groups"),
819
+ mergeMembers: safeCount("merge_members"),
820
+ resolveQueue: safeCount("resolve_queue"),
821
+ reviewQueue: safeCount("review_queue"),
822
+ resolveDecisions: safeCount("resolve_decisions"),
729
823
  };
730
824
  }
731
825
 
826
+ // ─── Phase 8 EntityResolver helpers ───────────────────────────────────
827
+
828
+ /**
829
+ * Insert a new pending row into resolve_queue. Idempotent — already-
830
+ * pending rows for the same person are not duplicated. Returns the
831
+ * row id (existing or newly inserted).
832
+ */
833
+ enqueueResolve(personId) {
834
+ if (typeof personId !== "string" || personId.length === 0) {
835
+ throw new Error("enqueueResolve: personId required");
836
+ }
837
+ const db = this._requireOpen();
838
+ const existing = db.prepare(
839
+ "SELECT id FROM resolve_queue WHERE person_id = ? AND status IN ('pending','in-progress')"
840
+ ).get(personId);
841
+ if (existing) return existing.id;
842
+ const info = db.prepare(
843
+ "INSERT INTO resolve_queue (person_id, enqueued_at, status) VALUES (?, ?, 'pending')"
844
+ ).run(personId, Date.now());
845
+ return info.lastInsertRowid;
846
+ }
847
+
848
+ /**
849
+ * Pull up to `limit` pending rows + atomically mark them in-progress.
850
+ * Returns [{id, person_id, attempts}, ...].
851
+ */
852
+ claimResolveBatch(limit = 50) {
853
+ const db = this._requireOpen();
854
+ const tx = db.transaction(() => {
855
+ const rows = db.prepare(
856
+ "SELECT id, person_id, attempts FROM resolve_queue WHERE status = 'pending' ORDER BY enqueued_at LIMIT ?"
857
+ ).all(limit);
858
+ if (rows.length === 0) return [];
859
+ const stmt = db.prepare(
860
+ "UPDATE resolve_queue SET status = 'in-progress', attempts = attempts + 1 WHERE id = ?"
861
+ );
862
+ for (const r of rows) stmt.run(r.id);
863
+ return rows;
864
+ });
865
+ return tx();
866
+ }
867
+
868
+ /**
869
+ * Mark a resolve_queue row as done (success path).
870
+ */
871
+ completeResolve(queueId) {
872
+ const db = this._requireOpen();
873
+ db.prepare("UPDATE resolve_queue SET status = 'done' WHERE id = ?").run(queueId);
874
+ }
875
+
876
+ /**
877
+ * Mark a resolve_queue row as errored (retry-eligible if attempts < 3).
878
+ */
879
+ errorResolve(queueId, errMsg) {
880
+ const db = this._requireOpen();
881
+ // If attempts < 3, leave status 'pending' for retry; else 'error'
882
+ db.prepare(
883
+ `UPDATE resolve_queue
884
+ SET status = CASE WHEN attempts >= 3 THEN 'error' ELSE 'pending' END,
885
+ last_error = ?
886
+ WHERE id = ?`
887
+ ).run(errMsg || "unknown", queueId);
888
+ }
889
+
890
+ /**
891
+ * Record a resolve_decisions row. Lex-orders the two ids so each pair
892
+ * is stored only once. Returns inserted-or-updated row.
893
+ */
894
+ recordResolveDecision({ aId, bId, verdict, confidence, decidedBy, reason }) {
895
+ const db = this._requireOpen();
896
+ const [lo, hi] = aId < bId ? [aId, bId] : [bId, aId];
897
+ db.prepare(
898
+ `INSERT INTO resolve_decisions
899
+ (a_person_id, b_person_id, verdict, confidence, decided_at, decided_by, reason)
900
+ VALUES (?, ?, ?, ?, ?, ?, ?)
901
+ ON CONFLICT(a_person_id, b_person_id) DO UPDATE SET
902
+ verdict = excluded.verdict,
903
+ confidence = excluded.confidence,
904
+ decided_at = excluded.decided_at,
905
+ decided_by = excluded.decided_by,
906
+ reason = excluded.reason`
907
+ ).run(lo, hi, verdict, confidence, Date.now(), decidedBy || "rule", reason || null);
908
+ }
909
+
910
+ getResolveDecision(aId, bId) {
911
+ const db = this._requireOpen();
912
+ const [lo, hi] = aId < bId ? [aId, bId] : [bId, aId];
913
+ return db.prepare(
914
+ "SELECT * FROM resolve_decisions WHERE a_person_id = ? AND b_person_id = ?"
915
+ ).get(lo, hi);
916
+ }
917
+
918
+ /**
919
+ * Merge a pair into a merge_group. If either side already belongs to a
920
+ * group, the other side joins it (and the two groups merge if both
921
+ * already existed). Returns the resulting group_id.
922
+ */
923
+ mergePair({ aId, bId, joinedBy = "rule" }) {
924
+ const db = this._requireOpen();
925
+ const tx = db.transaction(() => {
926
+ const aGroup = db.prepare("SELECT group_id FROM merge_members WHERE person_id = ?").get(aId);
927
+ const bGroup = db.prepare("SELECT group_id FROM merge_members WHERE person_id = ?").get(bId);
928
+ const now = Date.now();
929
+
930
+ if (aGroup && bGroup && aGroup.group_id === bGroup.group_id) {
931
+ return aGroup.group_id; // already same group
932
+ }
933
+ if (aGroup && bGroup) {
934
+ // Merge two existing groups → keep aGroup, move bGroup members in
935
+ db.prepare(
936
+ "UPDATE merge_members SET group_id = ? WHERE group_id = ?"
937
+ ).run(aGroup.group_id, bGroup.group_id);
938
+ db.prepare("DELETE FROM merge_groups WHERE id = ?").run(bGroup.group_id);
939
+ db.prepare(
940
+ "UPDATE merge_groups SET member_count = (SELECT COUNT(*) FROM merge_members WHERE group_id = ?), last_updated = ? WHERE id = ?"
941
+ ).run(aGroup.group_id, now, aGroup.group_id);
942
+ return aGroup.group_id;
943
+ }
944
+ if (aGroup) {
945
+ // Add b to a's group
946
+ db.prepare(
947
+ "INSERT INTO merge_members (group_id, person_id, joined_at, joined_by) VALUES (?, ?, ?, ?)"
948
+ ).run(aGroup.group_id, bId, now, joinedBy);
949
+ db.prepare(
950
+ "UPDATE merge_groups SET member_count = member_count + 1, last_updated = ? WHERE id = ?"
951
+ ).run(now, aGroup.group_id);
952
+ return aGroup.group_id;
953
+ }
954
+ if (bGroup) {
955
+ db.prepare(
956
+ "INSERT INTO merge_members (group_id, person_id, joined_at, joined_by) VALUES (?, ?, ?, ?)"
957
+ ).run(bGroup.group_id, aId, now, joinedBy);
958
+ db.prepare(
959
+ "UPDATE merge_groups SET member_count = member_count + 1, last_updated = ? WHERE id = ?"
960
+ ).run(now, bGroup.group_id);
961
+ return bGroup.group_id;
962
+ }
963
+ // Neither in any group — create new
964
+ const groupId = newGroupId();
965
+ db.prepare(
966
+ "INSERT INTO merge_groups (id, primary_id, member_count, created_at, last_updated) VALUES (?, ?, 2, ?, ?)"
967
+ ).run(groupId, aId, now, now);
968
+ const ins = db.prepare(
969
+ "INSERT INTO merge_members (group_id, person_id, joined_at, joined_by) VALUES (?, ?, ?, ?)"
970
+ );
971
+ ins.run(groupId, aId, now, joinedBy);
972
+ ins.run(groupId, bId, now, joinedBy);
973
+ return groupId;
974
+ });
975
+ return tx();
976
+ }
977
+
978
+ /**
979
+ * Remove a person from its merge group (unmerge). If only one member
980
+ * remains, the group is deleted entirely.
981
+ */
982
+ unmergePerson(personId) {
983
+ const db = this._requireOpen();
984
+ const tx = db.transaction(() => {
985
+ const row = db.prepare(
986
+ "SELECT group_id FROM merge_members WHERE person_id = ?"
987
+ ).get(personId);
988
+ if (!row) return { ok: false, reason: "not in any group" };
989
+ const groupId = row.group_id;
990
+ db.prepare("DELETE FROM merge_members WHERE person_id = ?").run(personId);
991
+ const remaining = db.prepare(
992
+ "SELECT COUNT(*) as n FROM merge_members WHERE group_id = ?"
993
+ ).get(groupId).n;
994
+ if (remaining < 2) {
995
+ // Group of 1 or 0 — delete the group + remaining member row
996
+ db.prepare("DELETE FROM merge_members WHERE group_id = ?").run(groupId);
997
+ db.prepare("DELETE FROM merge_groups WHERE id = ?").run(groupId);
998
+ } else {
999
+ db.prepare(
1000
+ "UPDATE merge_groups SET member_count = ?, last_updated = ? WHERE id = ?"
1001
+ ).run(remaining, Date.now(), groupId);
1002
+ }
1003
+ return { ok: true, groupId, remaining };
1004
+ });
1005
+ return tx();
1006
+ }
1007
+
1008
+ /**
1009
+ * Get all person ids in the same merge group as the given person.
1010
+ * Returns [personId, ...] including the input (whether or not it's in
1011
+ * a group — a "group of 1" is just `[personId]`).
1012
+ */
1013
+ getMergeGroupMembers(personId) {
1014
+ const db = this._requireOpen();
1015
+ const groupRow = db.prepare(
1016
+ "SELECT group_id FROM merge_members WHERE person_id = ?"
1017
+ ).get(personId);
1018
+ if (!groupRow) return [personId];
1019
+ return db.prepare(
1020
+ "SELECT person_id FROM merge_members WHERE group_id = ? ORDER BY joined_at"
1021
+ ).all(groupRow.group_id).map((r) => r.person_id);
1022
+ }
1023
+
1024
+ /**
1025
+ * Insert a row into review_queue when the LLM stage returns "maybe".
1026
+ * UI lists these for user one-click decisions.
1027
+ */
1028
+ enqueueReview({ aId, bId, embedSim, llmVerdict, llmReason, llmConfidence }) {
1029
+ const db = this._requireOpen();
1030
+ const [lo, hi] = aId < bId ? [aId, bId] : [bId, aId];
1031
+ const info = db.prepare(
1032
+ `INSERT INTO review_queue
1033
+ (a_person_id, b_person_id, embed_sim, llm_verdict, llm_reason, llm_confidence, enqueued_at)
1034
+ VALUES (?, ?, ?, ?, ?, ?, ?)`
1035
+ ).run(lo, hi, embedSim || null, llmVerdict || null, llmReason || null, llmConfidence || null, Date.now());
1036
+ return info.lastInsertRowid;
1037
+ }
1038
+
1039
+ /**
1040
+ * List pending review rows (oldest first).
1041
+ */
1042
+ listReviewQueue({ limit = 50 } = {}) {
1043
+ const db = this._requireOpen();
1044
+ return db.prepare(
1045
+ "SELECT * FROM review_queue WHERE reviewed_at IS NULL ORDER BY enqueued_at ASC LIMIT ?"
1046
+ ).all(Math.min(limit, 1000));
1047
+ }
1048
+
1049
+ /**
1050
+ * Mark a review row as decided by the user.
1051
+ */
1052
+ recordReviewDecision({ reviewId, decision }) {
1053
+ if (!["same", "different", "skip"].includes(decision)) {
1054
+ throw new Error(`invalid review decision: ${decision}`);
1055
+ }
1056
+ const db = this._requireOpen();
1057
+ const row = db.prepare("SELECT * FROM review_queue WHERE id = ?").get(reviewId);
1058
+ if (!row) throw new Error(`review row ${reviewId} not found`);
1059
+ db.prepare(
1060
+ "UPDATE review_queue SET reviewed_at = ?, user_decision = ? WHERE id = ?"
1061
+ ).run(Date.now(), decision, reviewId);
1062
+ return row;
1063
+ }
1064
+
1065
+ resolveQueueStats() {
1066
+ const db = this._requireOpen();
1067
+ const rows = db.prepare(
1068
+ "SELECT status, COUNT(*) as n FROM resolve_queue GROUP BY status"
1069
+ ).all();
1070
+ const out = { pending: 0, "in-progress": 0, done: 0, error: 0 };
1071
+ for (const r of rows) out[r.status] = r.n;
1072
+ return out;
1073
+ }
1074
+
732
1075
  // ─── Key rotation ──────────────────────────────────────────────────────
733
1076
 
734
1077
  /**
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@chainlesschain/personal-data-hub",
3
- "version": "0.1.0",
3
+ "version": "0.2.1",
4
4
  "description": "Personal Data Hub — UnifiedSchema + validators + KG ingest helpers for the data-back-to-the-individual middleware",
5
5
  "type": "commonjs",
6
6
  "main": "lib/index.js",
@@ -26,7 +26,38 @@
26
26
  "./bridges/cc-llm-adapter": "./lib/bridges/cc-llm-adapter.js",
27
27
  "./bridges/cc-kg-sink": "./lib/bridges/cc-kg-sink.js",
28
28
  "./bridges/cc-rag-sink": "./lib/bridges/cc-rag-sink.js",
29
- "./adapters/email-imap": "./lib/adapters/email-imap/index.js"
29
+ "./adapters/email-imap": "./lib/adapters/email-imap/index.js",
30
+ "./adapters/alipay-bill": "./lib/adapters/alipay-bill/index.js",
31
+ "./adapters/system-data": "./lib/adapters/system-data/index.js",
32
+ "./adapters/system-data-android": "./lib/adapters/system-data-android/index.js",
33
+ "./entity-resolver": "./lib/entity-resolver/index.js",
34
+ "./analysis-skills": "./lib/analysis-skills/index.js",
35
+ "./mobile-extractor": "./lib/mobile-extractor/index.js",
36
+ "./adapters/wechat": "./lib/adapters/wechat/index.js",
37
+ "./adapters/ai-chat-history": "./lib/adapters/ai-chat-history/index.js",
38
+ "./adapters/ai-chat-history/cookie-capture-spec": "./lib/adapters/ai-chat-history/cookie-capture-spec.js",
39
+ "./adapters/ai-chat-history/wizard-controller": "./lib/adapters/ai-chat-history/wizard-controller.js",
40
+ "./adapters/ai-chat-history/health-checker": "./lib/adapters/ai-chat-history/health-checker.js",
41
+ "./lib/adapters/ai-chat-history/cookie-capture-spec": "./lib/adapters/ai-chat-history/cookie-capture-spec.js",
42
+ "./lib/adapters/ai-chat-history/wizard-controller": "./lib/adapters/ai-chat-history/wizard-controller.js",
43
+ "./lib/adapters/ai-chat-history/health-checker": "./lib/adapters/ai-chat-history/health-checker.js",
44
+ "./adapters/travel-base": "./lib/adapters/travel-base/index.js",
45
+ "./adapters/travel-12306": "./lib/adapters/travel-12306/index.js",
46
+ "./adapters/travel-ctrip": "./lib/adapters/travel-ctrip/index.js",
47
+ "./adapters/travel-amap": "./lib/adapters/travel-amap/index.js",
48
+ "./adapters/travel-baidu-map": "./lib/adapters/travel-baidu-map/index.js",
49
+ "./adapters/shopping-base": "./lib/adapters/shopping-base/index.js",
50
+ "./adapters/shopping-taobao": "./lib/adapters/shopping-taobao/index.js",
51
+ "./adapters/shopping-jd": "./lib/adapters/shopping-jd/index.js",
52
+ "./adapters/shopping-meituan": "./lib/adapters/shopping-meituan/index.js",
53
+ "./adapters/social-bilibili": "./lib/adapters/social-bilibili/index.js",
54
+ "./adapters/social-weibo": "./lib/adapters/social-weibo/index.js",
55
+ "./adapters/social-douyin": "./lib/adapters/social-douyin/index.js",
56
+ "./adapters/social-xiaohongshu": "./lib/adapters/social-xiaohongshu/index.js",
57
+ "./adapters/messaging-qq": "./lib/adapters/messaging-qq/index.js",
58
+ "./adapters/messaging-telegram": "./lib/adapters/messaging-telegram/index.js",
59
+ "./adapters/messaging-whatsapp": "./lib/adapters/messaging-whatsapp/index.js",
60
+ "./sidecar": "./lib/sidecar/index.js"
30
61
  },
31
62
  "scripts": {
32
63
  "test": "vitest run",
@@ -55,7 +86,9 @@
55
86
  "mailparser": "^3.7.1"
56
87
  },
57
88
  "optionalDependencies": {
58
- "imapflow": "^1.0.183"
89
+ "imapflow": "^1.0.183",
90
+ "adm-zip": "^0.5.16",
91
+ "iconv-lite": "^0.6.3"
59
92
  },
60
93
  "devDependencies": {
61
94
  "vitest": "^4.1.5"
@@ -0,0 +1,126 @@
1
+ #!/usr/bin/env node
2
+ /**
3
+ * Build a complete fixture directory with all 4 system-data sources:
4
+ *
5
+ * <out>/
6
+ * contacts2.db (raw_contacts + data + mimetypes)
7
+ * mmssms.db (sms)
8
+ * wifi/
9
+ * WifiConfigStore.xml
10
+ *
11
+ * Usage:
12
+ * node scripts/_make-fixture-all.js ./fixtures
13
+ */
14
+
15
+ "use strict";
16
+
17
+ const fs = require("node:fs");
18
+ const path = require("node:path");
19
+ const Database = require("better-sqlite3-multiple-ciphers");
20
+
21
+ const outDir = path.resolve(process.argv[2] || "./fixtures");
22
+ fs.mkdirSync(path.join(outDir, "wifi"), { recursive: true });
23
+
24
+ // ── contacts2.db ──────────────────────────────────────────────────────────
25
+ const contactsPath = path.join(outDir, "contacts2.db");
26
+ if (fs.existsSync(contactsPath)) fs.unlinkSync(contactsPath);
27
+ const contacts = new Database(contactsPath);
28
+ contacts.exec(`
29
+ CREATE TABLE raw_contacts (
30
+ _id INTEGER PRIMARY KEY, display_name TEXT, starred INTEGER DEFAULT 0, deleted INTEGER DEFAULT 0
31
+ );
32
+ CREATE TABLE mimetypes (_id INTEGER PRIMARY KEY, mimetype TEXT NOT NULL UNIQUE);
33
+ CREATE TABLE data (
34
+ _id INTEGER PRIMARY KEY, raw_contact_id INTEGER NOT NULL, mimetype_id INTEGER NOT NULL, data1 TEXT
35
+ );
36
+ CREATE TABLE calls (
37
+ _id INTEGER PRIMARY KEY, number TEXT, type INTEGER, duration INTEGER, date INTEGER, name TEXT, is_read INTEGER DEFAULT 1
38
+ );
39
+ `);
40
+ const MT = { phone: 5, email: 1, org: 4, note: 10, photo: 14 };
41
+ const mi = contacts.prepare("INSERT INTO mimetypes (_id, mimetype) VALUES (?, ?)");
42
+ mi.run(MT.phone, "vnd.android.cursor.item/phone_v2");
43
+ mi.run(MT.email, "vnd.android.cursor.item/email_v2");
44
+ mi.run(MT.org, "vnd.android.cursor.item/organization");
45
+ mi.run(MT.note, "vnd.android.cursor.item/note");
46
+ mi.run(MT.photo, "vnd.android.cursor.item/photo");
47
+ const ci = contacts.prepare(
48
+ "INSERT INTO raw_contacts (_id, display_name, starred, deleted) VALUES (?, ?, ?, 0)",
49
+ );
50
+ ci.run(1, "妈妈", 1);
51
+ ci.run(2, "张三", 0);
52
+ ci.run(3, "李四 Manager", 0);
53
+ ci.run(5, "工商银行客服", 0);
54
+ const di = contacts.prepare(
55
+ "INSERT INTO data (raw_contact_id, mimetype_id, data1) VALUES (?, ?, ?)",
56
+ );
57
+ di.run(1, MT.phone, "13800001111");
58
+ di.run(1, MT.phone, "13900002222");
59
+ di.run(1, MT.email, "mom@example.com");
60
+ di.run(1, MT.note, "亲妈,过年回家");
61
+ di.run(2, MT.phone, "13711112222");
62
+ di.run(3, MT.phone, "13822223333");
63
+ di.run(3, MT.email, "lisi@corp.example.com");
64
+ di.run(3, MT.org, "Example Corp");
65
+ di.run(5, MT.phone, "95588");
66
+ // Calls table inside contacts2.db (pre-Android-11 location)
67
+ const li = contacts.prepare(
68
+ "INSERT INTO calls (_id, number, type, duration, date, name, is_read) VALUES (?, ?, ?, ?, ?, ?, ?)",
69
+ );
70
+ li.run(1, "13800001111", 1, 120, 1737000000000, "妈妈", 1);
71
+ li.run(2, "13800001111", 2, 45, 1737010000000, "妈妈", 1);
72
+ li.run(3, "13999998888", 3, 0, 1737020000000, "", 0);
73
+ li.run(4, "10086", 1, 8, 1737030000000, "中国移动", 1);
74
+ contacts.close();
75
+ console.log("wrote:", contactsPath);
76
+
77
+ // ── mmssms.db ─────────────────────────────────────────────────────────────
78
+ const smsPath = path.join(outDir, "mmssms.db");
79
+ if (fs.existsSync(smsPath)) fs.unlinkSync(smsPath);
80
+ const sms = new Database(smsPath);
81
+ sms.exec(`
82
+ CREATE TABLE sms (
83
+ _id INTEGER PRIMARY KEY, thread_id INTEGER, address TEXT, body TEXT,
84
+ type INTEGER, date INTEGER, read INTEGER
85
+ );
86
+ `);
87
+ const si = sms.prepare(
88
+ "INSERT INTO sms (_id, thread_id, address, body, type, date, read) VALUES (?, ?, ?, ?, ?, ?, ?)",
89
+ );
90
+ si.run(1, 100, "13800001111", "妈妈我到家了", 2, 1737000000000, 1);
91
+ si.run(2, 100, "13800001111", "好的,注意安全", 1, 1737000010000, 1);
92
+ si.run(3, 200, "10086", "【中国移动】您的话费余额为 ¥36.50", 1, 1737000020000, 1);
93
+ si.run(4, 300, "95588", "【工商银行】您的验证码为 123456,3 分钟内有效", 1, 1737000030000, 0);
94
+ sms.close();
95
+ console.log("wrote:", smsPath);
96
+
97
+ // ── wifi/WifiConfigStore.xml ──────────────────────────────────────────────
98
+ const wifiXml = path.join(outDir, "wifi", "WifiConfigStore.xml");
99
+ fs.writeFileSync(
100
+ wifiXml,
101
+ `<?xml version='1.0' encoding='UTF-8'?>
102
+ <WifiConfigStoreData>
103
+ <NetworkList>
104
+ <Network>
105
+ <WifiConfiguration>
106
+ <string name="SSID">"Home_5G"</string>
107
+ <string name="PreSharedKey">"secret"</string>
108
+ <string name="KeyMgmt">WPA-PSK</string>
109
+ <boolean name="HiddenSSID">false</boolean>
110
+ </WifiConfiguration>
111
+ </Network>
112
+ <Network>
113
+ <WifiConfiguration>
114
+ <string name="SSID">"Starbucks Free"</string>
115
+ <string name="KeyMgmt">NONE</string>
116
+ <boolean name="HiddenSSID">false</boolean>
117
+ </WifiConfiguration>
118
+ </Network>
119
+ </NetworkList>
120
+ </WifiConfigStoreData>
121
+ `,
122
+ "utf-8",
123
+ );
124
+ console.log("wrote:", wifiXml);
125
+
126
+ console.log("\nAll fixtures ready under:", outDir);
@@ -0,0 +1,84 @@
1
+ #!/usr/bin/env node
2
+ /**
3
+ * Build a synthetic Android contacts2.db at the given path.
4
+ *
5
+ * Only used by the smoke runner / docs walkthrough — production code never
6
+ * relies on this. Mirrors the fixture from
7
+ * packages/personal-data-hub-bridge/tests/test_parsers_system_contacts.py.
8
+ *
9
+ * Usage:
10
+ * node scripts/_make-fixture-contacts.js ./fixtures/contacts2.db
11
+ */
12
+
13
+ "use strict";
14
+
15
+ const fs = require("node:fs");
16
+ const path = require("node:path");
17
+ const Database = require("better-sqlite3-multiple-ciphers");
18
+
19
+ const target = path.resolve(process.argv[2] || "./fixtures/contacts2.db");
20
+ fs.mkdirSync(path.dirname(target), { recursive: true });
21
+ if (fs.existsSync(target)) fs.unlinkSync(target);
22
+
23
+ const db = new Database(target);
24
+ try {
25
+ db.exec(`
26
+ CREATE TABLE raw_contacts (
27
+ _id INTEGER PRIMARY KEY,
28
+ display_name TEXT,
29
+ starred INTEGER DEFAULT 0,
30
+ deleted INTEGER DEFAULT 0
31
+ );
32
+ CREATE TABLE mimetypes (
33
+ _id INTEGER PRIMARY KEY,
34
+ mimetype TEXT NOT NULL UNIQUE
35
+ );
36
+ CREATE TABLE data (
37
+ _id INTEGER PRIMARY KEY,
38
+ raw_contact_id INTEGER NOT NULL,
39
+ mimetype_id INTEGER NOT NULL,
40
+ data1 TEXT
41
+ );
42
+ `);
43
+
44
+ const MT = {
45
+ phone: 5,
46
+ email: 1,
47
+ org: 4,
48
+ note: 10,
49
+ photo: 14,
50
+ };
51
+ const insertMime = db.prepare(
52
+ "INSERT INTO mimetypes (_id, mimetype) VALUES (?, ?)",
53
+ );
54
+ insertMime.run(MT.phone, "vnd.android.cursor.item/phone_v2");
55
+ insertMime.run(MT.email, "vnd.android.cursor.item/email_v2");
56
+ insertMime.run(MT.org, "vnd.android.cursor.item/organization");
57
+ insertMime.run(MT.note, "vnd.android.cursor.item/note");
58
+ insertMime.run(MT.photo, "vnd.android.cursor.item/photo");
59
+
60
+ const insertC = db.prepare(
61
+ "INSERT INTO raw_contacts (_id, display_name, starred, deleted) VALUES (?, ?, ?, 0)",
62
+ );
63
+ insertC.run(1, "妈妈", 1);
64
+ insertC.run(2, "张三", 0);
65
+ insertC.run(3, "李四 Manager", 0);
66
+ insertC.run(4, "", 0); // nameless — skipped by parser
67
+ insertC.run(5, "工商银行客服", 0);
68
+
69
+ const insertD = db.prepare(
70
+ "INSERT INTO data (raw_contact_id, mimetype_id, data1) VALUES (?, ?, ?)",
71
+ );
72
+ insertD.run(1, MT.phone, "13800001111");
73
+ insertD.run(1, MT.phone, "13900002222");
74
+ insertD.run(1, MT.email, "mom@example.com");
75
+ insertD.run(1, MT.note, "亲妈,过年回家");
76
+ insertD.run(2, MT.phone, "13711112222");
77
+ insertD.run(3, MT.phone, "13822223333");
78
+ insertD.run(3, MT.email, "lisi@corp.example.com");
79
+ insertD.run(3, MT.org, "Example Corp");
80
+ insertD.run(5, MT.phone, "95588");
81
+ } finally {
82
+ db.close();
83
+ }
84
+ console.log("fixture written:", target);