@chainlesschain/personal-data-hub 0.4.29 → 0.4.31

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (199) hide show
  1. package/lib/forensics/qq-nt-collect.js +190 -0
  2. package/lib/prompt-builder.js +15 -1
  3. package/package.json +8 -3
  4. package/__tests__/adapter-guide.test.js +0 -47
  5. package/__tests__/adapter-spec.test.js +0 -78
  6. package/__tests__/adapters/ai-chat-cookie-capture-spec.test.js +0 -211
  7. package/__tests__/adapters/ai-chat-health-checker.test.js +0 -262
  8. package/__tests__/adapters/ai-chat-history.test.js +0 -396
  9. package/__tests__/adapters/ai-chat-http-client.test.js +0 -242
  10. package/__tests__/adapters/ai-chat-vendors.test.js +0 -874
  11. package/__tests__/adapters/alipay-bill-adapter.test.js +0 -538
  12. package/__tests__/adapters/apple-health.test.js +0 -95
  13. package/__tests__/adapters/bank-family.test.js +0 -125
  14. package/__tests__/adapters/biz-tianyancha.test.js +0 -159
  15. package/__tests__/adapters/browser-history-chrome.test.js +0 -377
  16. package/__tests__/adapters/browser-history-edge.test.js +0 -159
  17. package/__tests__/adapters/car-mercedesme.test.js +0 -74
  18. package/__tests__/adapters/doc-baidu-netdisk.test.js +0 -102
  19. package/__tests__/adapters/doc-camscanner.test.js +0 -147
  20. package/__tests__/adapters/doc-platforms.test.js +0 -177
  21. package/__tests__/adapters/edu-huawei-learning-live.test.js +0 -198
  22. package/__tests__/adapters/edu-zuoyebang-live.test.js +0 -226
  23. package/__tests__/adapters/email-adapter-snapshot.test.js +0 -237
  24. package/__tests__/adapters/email-adapter.test.js +0 -742
  25. package/__tests__/adapters/email-classifier.test.js +0 -347
  26. package/__tests__/adapters/email-imap-session.test.js +0 -334
  27. package/__tests__/adapters/email-parser.test.js +0 -244
  28. package/__tests__/adapters/email-pdf-extractor.test.js +0 -529
  29. package/__tests__/adapters/email-providers.test.js +0 -84
  30. package/__tests__/adapters/email-retry-progress.test.js +0 -294
  31. package/__tests__/adapters/email-templates.test.js +0 -822
  32. package/__tests__/adapters/family-23-collectors-scaffold.test.js +0 -182
  33. package/__tests__/adapters/finance-alipay-live.test.js +0 -258
  34. package/__tests__/adapters/finance-dcep.test.js +0 -74
  35. package/__tests__/adapters/fitness-joyrun.test.js +0 -82
  36. package/__tests__/adapters/game-genshin-live.test.js +0 -238
  37. package/__tests__/adapters/game-genshin-scaffold.test.js +0 -108
  38. package/__tests__/adapters/game-honor-of-kings-live.test.js +0 -230
  39. package/__tests__/adapters/git-activity.test.js +0 -222
  40. package/__tests__/adapters/gov-12123.test.js +0 -103
  41. package/__tests__/adapters/gov-ixiamen.test.js +0 -150
  42. package/__tests__/adapters/gov-tax.test.js +0 -135
  43. package/__tests__/adapters/health-meiyou.test.js +0 -125
  44. package/__tests__/adapters/local-files.test.js +0 -264
  45. package/__tests__/adapters/local-im-pc.test.js +0 -154
  46. package/__tests__/adapters/messaging-whatsapp.test.js +0 -289
  47. package/__tests__/adapters/music-kugou.test.js +0 -187
  48. package/__tests__/adapters/music-qq.test.js +0 -112
  49. package/__tests__/adapters/netease-music-live.test.js +0 -244
  50. package/__tests__/adapters/netease-music.test.js +0 -74
  51. package/__tests__/adapters/pc-local-discovery.test.js +0 -141
  52. package/__tests__/adapters/qq-pc-direct-read.test.js +0 -227
  53. package/__tests__/adapters/reading-family.test.js +0 -108
  54. package/__tests__/adapters/recruit-boss.test.js +0 -180
  55. package/__tests__/adapters/shell-history.test.js +0 -180
  56. package/__tests__/adapters/shopping-base.test.js +0 -179
  57. package/__tests__/adapters/shopping-dianping.test.js +0 -239
  58. package/__tests__/adapters/social-bilibili-adb-api-client.test.js +0 -721
  59. package/__tests__/adapters/social-bilibili-adb-chromium-cookies-reader.test.js +0 -346
  60. package/__tests__/adapters/social-bilibili-adb-collector.test.js +0 -284
  61. package/__tests__/adapters/social-bilibili-adb-cookies-extension.test.js +0 -343
  62. package/__tests__/adapters/social-bilibili-adb-snapshot-builder.test.js +0 -296
  63. package/__tests__/adapters/social-csdn.test.js +0 -175
  64. package/__tests__/adapters/social-dongchedi.test.js +0 -165
  65. package/__tests__/adapters/social-douyin-adb-aweme-detail.test.js +0 -165
  66. package/__tests__/adapters/social-douyin-adb-collector.test.js +0 -254
  67. package/__tests__/adapters/social-douyin-adb-db-extension.test.js +0 -114
  68. package/__tests__/adapters/social-douyin-adb-im-db-parser.test.js +0 -304
  69. package/__tests__/adapters/social-douyin-adb-snapshot-builder.test.js +0 -216
  70. package/__tests__/adapters/social-douyin-adb-usage-profile.test.js +0 -229
  71. package/__tests__/adapters/social-douyin-adb-watch-history.test.js +0 -269
  72. package/__tests__/adapters/social-kuaishou-adb-api-client.test.js +0 -496
  73. package/__tests__/adapters/social-kuaishou-adb-collector.test.js +0 -276
  74. package/__tests__/adapters/social-kuaishou-adb-cookies-extension.test.js +0 -152
  75. package/__tests__/adapters/social-kuaishou-adb-snapshot-builder.test.js +0 -178
  76. package/__tests__/adapters/social-toutiao-adb-account-reader.test.js +0 -135
  77. package/__tests__/adapters/social-toutiao-adb-api-client.test.js +0 -626
  78. package/__tests__/adapters/social-toutiao-adb-article.test.js +0 -155
  79. package/__tests__/adapters/social-toutiao-adb-collector.test.js +0 -378
  80. package/__tests__/adapters/social-toutiao-adb-cookies-extension.test.js +0 -193
  81. package/__tests__/adapters/social-toutiao-adb-snapshot-builder.test.js +0 -196
  82. package/__tests__/adapters/social-toutiao-kuaishou-scaffold.test.js +0 -311
  83. package/__tests__/adapters/social-weibo-adb-api-client.test.js +0 -362
  84. package/__tests__/adapters/social-weibo-adb-collector.test.js +0 -201
  85. package/__tests__/adapters/social-weibo-adb-cookies-extension.test.js +0 -167
  86. package/__tests__/adapters/social-weibo-adb-snapshot-builder.test.js +0 -189
  87. package/__tests__/adapters/social-xiaohongshu-adb-api-client.test.js +0 -431
  88. package/__tests__/adapters/social-xiaohongshu-adb-collector.test.js +0 -207
  89. package/__tests__/adapters/social-xiaohongshu-adb-cookies-extension.test.js +0 -0
  90. package/__tests__/adapters/social-xiaohongshu-adb-sign-provider-injection.test.js +0 -351
  91. package/__tests__/adapters/social-xiaohongshu-adb-sign.test.js +0 -130
  92. package/__tests__/adapters/social-xiaohongshu-adb-snapshot-builder.test.js +0 -200
  93. package/__tests__/adapters/social-zhihu.test.js +0 -246
  94. package/__tests__/adapters/system-data-adapter.test.js +0 -443
  95. package/__tests__/adapters/system-data-android-ingest.test.js +0 -144
  96. package/__tests__/adapters/system-data-android.test.js +0 -519
  97. package/__tests__/adapters/system-data-disclosure.test.js +0 -153
  98. package/__tests__/adapters/travel-12306.test.js +0 -512
  99. package/__tests__/adapters/travel-amap.test.js +0 -219
  100. package/__tests__/adapters/travel-baidu-map.test.js +0 -305
  101. package/__tests__/adapters/travel-base.test.js +0 -205
  102. package/__tests__/adapters/travel-ctrip.test.js +0 -377
  103. package/__tests__/adapters/travel-didi-consumer.test.js +0 -66
  104. package/__tests__/adapters/travel-didi.test.js +0 -204
  105. package/__tests__/adapters/travel-tencent-map.test.js +0 -207
  106. package/__tests__/adapters/travel-tongcheng.test.js +0 -289
  107. package/__tests__/adapters/video-platforms.test.js +0 -152
  108. package/__tests__/adapters/video-xigua.test.js +0 -106
  109. package/__tests__/adapters/vscode.test.js +0 -299
  110. package/__tests__/adapters/wechat-bootstrap.test.js +0 -240
  111. package/__tests__/adapters/wechat-env-probe.test.js +0 -162
  112. package/__tests__/adapters/wechat-frida-agent.test.js +0 -322
  113. package/__tests__/adapters/wechat-frida-integration.test.js +0 -149
  114. package/__tests__/adapters/wechat-frida-key-provider.test.js +0 -188
  115. package/__tests__/adapters/wechat-md5-key-provider.test.js +0 -101
  116. package/__tests__/adapters/wechat-pc-direct-read.test.js +0 -365
  117. package/__tests__/adapters/wechat-pc-group-topic.test.js +0 -63
  118. package/__tests__/adapters/wechat-pc-v4-sidecar.test.js +0 -72
  119. package/__tests__/adapters/weread.test.js +0 -123
  120. package/__tests__/adapters/wework-pc.test.js +0 -124
  121. package/__tests__/adapters/win-recent.test.js +0 -192
  122. package/__tests__/analysis-skills.test.js +0 -754
  123. package/__tests__/analysis.test.js +0 -1845
  124. package/__tests__/audio-ximalaya-snapshot.test.js +0 -279
  125. package/__tests__/batch.test.js +0 -133
  126. package/__tests__/bridges-cc-kg.test.js +0 -231
  127. package/__tests__/bridges-cc-llm.test.js +0 -191
  128. package/__tests__/bridges-cc-rag.test.js +0 -162
  129. package/__tests__/categories.test.js +0 -92
  130. package/__tests__/e2e/ai-chat-cross-source-journey.test.js +0 -213
  131. package/__tests__/e2e/full-user-journey.test.js +0 -188
  132. package/__tests__/e2e/local-data-adapters-cli.e2e.test.js +0 -146
  133. package/__tests__/entity-resolver-ingest-hook.test.js +0 -177
  134. package/__tests__/entity-resolver-stages.test.js +0 -411
  135. package/__tests__/entity-resolver-vault.test.js +0 -249
  136. package/__tests__/entity-resolver.test.js +0 -526
  137. package/__tests__/fitness-keep-snapshot.test.js +0 -224
  138. package/__tests__/fixtures/entity-resolver-200-mock.json +0 -96
  139. package/__tests__/ids.test.js +0 -45
  140. package/__tests__/integration/ai-chat-history-registry.test.js +0 -228
  141. package/__tests__/integration/aichat-wizard-end-to-end.test.js +0 -282
  142. package/__tests__/integration/cross-adapter-pipelines.test.js +0 -396
  143. package/__tests__/integration/local-data-adapters-pipeline.test.js +0 -373
  144. package/__tests__/integration/social-bilibili-pipeline.test.js +0 -261
  145. package/__tests__/integration/wechat-bootstrap-end-to-end.test.js +0 -390
  146. package/__tests__/key-providers.test.js +0 -126
  147. package/__tests__/kg-derive.test.js +0 -219
  148. package/__tests__/llm-client.test.js +0 -122
  149. package/__tests__/longtail-adapters.test.js +0 -281
  150. package/__tests__/messaging-qq-snapshot.test.js +0 -294
  151. package/__tests__/mobile-extractor-encrypted.test.js +0 -460
  152. package/__tests__/mobile-extractor.test.js +0 -288
  153. package/__tests__/mock-adapter.test.js +0 -93
  154. package/__tests__/prompt-builder.test.js +0 -249
  155. package/__tests__/query-parser.test.js +0 -365
  156. package/__tests__/rag-derive.test.js +0 -169
  157. package/__tests__/registry-readiness.test.js +0 -292
  158. package/__tests__/registry.test.js +0 -420
  159. package/__tests__/salvage-ingest.test.js +0 -97
  160. package/__tests__/schemas.test.js +0 -331
  161. package/__tests__/shopping-adapters.test.js +0 -392
  162. package/__tests__/shopping-eleme-snapshot.test.js +0 -454
  163. package/__tests__/shopping-pinduoduo-snapshot.test.js +0 -484
  164. package/__tests__/shopping-snapshot.test.js +0 -438
  165. package/__tests__/shopping-vipshop-snapshot.test.js +0 -425
  166. package/__tests__/shopping-xianyu-snapshot.test.js +0 -451
  167. package/__tests__/sidecar-contacts-cross-validate.test.js +0 -186
  168. package/__tests__/sidecar-supervisor.test.js +0 -128
  169. package/__tests__/sign-providers.test.js +0 -62
  170. package/__tests__/social-adapters.test.js +0 -280
  171. package/__tests__/social-bilibili-snapshot.test.js +0 -278
  172. package/__tests__/social-douban-snapshot.test.js +0 -351
  173. package/__tests__/social-douyin-im-direct-read.test.js +0 -377
  174. package/__tests__/social-douyin-salvage-collector.test.js +0 -98
  175. package/__tests__/social-douyin-salvage-mapper.test.js +0 -90
  176. package/__tests__/social-douyin-snapshot.test.js +0 -256
  177. package/__tests__/social-kuaishou-snapshot.test.js +0 -362
  178. package/__tests__/social-toutiao-snapshot.test.js +0 -366
  179. package/__tests__/social-weibo-snapshot.test.js +0 -234
  180. package/__tests__/social-weibo-sqlite-device.test.js +0 -174
  181. package/__tests__/social-xiaohongshu-snapshot.test.js +0 -232
  182. package/__tests__/sqlite-leaf-salvage.test.js +0 -97
  183. package/__tests__/travel-adapters.test.js +0 -483
  184. package/__tests__/travel-maps-snapshot.test.js +0 -426
  185. package/__tests__/vault-driver-error.test.js +0 -74
  186. package/__tests__/vault-search-helpers.test.js +0 -104
  187. package/__tests__/vault-search.test.js +0 -423
  188. package/__tests__/vault.test.js +0 -767
  189. package/__tests__/wechat-adapter.test.js +0 -594
  190. package/__tests__/whatsapp-adapter.test.js +0 -138
  191. package/scripts/_make-fixture-all.js +0 -126
  192. package/scripts/_make-fixture-contacts.js +0 -84
  193. package/scripts/evaluate-entity-resolver.js +0 -213
  194. package/scripts/run-native-tests-sandbox.sh +0 -55
  195. package/scripts/smoke-phase-5-5.js +0 -196
  196. package/scripts/smoke-phase-5-7.js +0 -181
  197. package/scripts/smoke-system-data-contacts.js +0 -309
  198. package/scripts/smoke-system-data.js +0 -312
  199. package/vitest.config.js +0 -88
@@ -1,196 +0,0 @@
1
- #!/usr/bin/env node
2
- /**
3
- * Phase 5.5 smoke — drives EmailAdapter end-to-end with a mock encrypted
4
- * PDF attachment, exercising:
5
- * - password-trial loop (3 wrong → 1 right)
6
- * - text extraction
7
- * - transactions regex (3 rows from a 招行-style statement)
8
- * - merging transactions[] into bill template fields
9
- * - per-attachment pdfExtraction summary
10
- * - attachment buffer stripping before raw-event emission
11
- *
12
- * Uses an INJECTED pdfExtractor so the smoke runs without pulling the
13
- * heavy pdfjs dep. The shape of the injected output matches what
14
- * `extractPdfText` from pdf-extractor.js would return.
15
- */
16
-
17
- "use strict";
18
-
19
- const { EmailAdapter } = require("../lib/adapters/email-imap/email-adapter");
20
- const { extractTransactions } = require("../lib/adapters/email-imap/transactions");
21
-
22
- const PDF_TEXT = [
23
- "招商银行信用卡 11 月对账单",
24
- "持卡人: 张三 尾号 1234",
25
- "账单周期: 2026-10-26 至 2026-11-25",
26
- "最后还款日: 2026-12-05 应还金额: ¥3,000.00",
27
- "",
28
- "交易明细:",
29
- "2026-10-30 星巴克 上海中山公园店 -39.00 2,961.00",
30
- "2026-11-05 京东自营 -899.00 2,062.00",
31
- "2026-11-12 退款 淘宝 +50.00 2,112.00",
32
- "2026-11-18 美团外卖 -85.00 2,027.00",
33
- "",
34
- "第 1 页 共 1 页",
35
- ].join("\n");
36
-
37
- const PDF_PASSWORDS = ["wrong1", "wrong2", "wrong3", "987654"];
38
-
39
- function makeSession() {
40
- const env = {
41
- uid: 1,
42
- internalDate: new Date("2026-11-26T10:00:00Z"),
43
- flags: ["\\Seen"],
44
- messageId: "<bill-cmb-11@x>",
45
- subject: "招商银行信用卡 11 月对账单",
46
- from: [{ name: "招商银行", address: "ebank@cmbchina.com" }],
47
- to: [{ address: "me@example.com" }],
48
- cc: [],
49
- date: new Date("2026-11-26T10:00:00Z"),
50
- size: 8192,
51
- source: Buffer.from("RAW", "utf8"),
52
- };
53
- return () => ({
54
- async connect() {},
55
- async openMailbox(_name) {
56
- return { uidValidity: 1, uidNext: 9999, exists: 1 };
57
- },
58
- async *fetchFullSince(sinceUid = 0) {
59
- if (env.uid > sinceUid) yield env;
60
- },
61
- async close() {},
62
- });
63
- }
64
-
65
- let trialCount = 0;
66
- async function mockPdfExtractor(buffer, opts) {
67
- trialCount = 0;
68
- for (const pw of ["", ...(opts.passwords || [])]) {
69
- trialCount += 1;
70
- if (pw === "987654") {
71
- return {
72
- decrypted: true,
73
- text: PDF_TEXT,
74
- password: pw,
75
- attempted: trialCount,
76
- wasEncrypted: true,
77
- pageCount: 1,
78
- };
79
- }
80
- }
81
- return {
82
- decrypted: false,
83
- text: "",
84
- attempted: trialCount,
85
- wasEncrypted: true,
86
- pageCount: 0,
87
- error: "all passwords failed",
88
- };
89
- }
90
-
91
- async function main() {
92
- console.log("== Phase 5.5 smoke ==");
93
-
94
- // First validate the standalone transactions parser on the fixture text
95
- console.log("\n— Standalone transactions regex —");
96
- const standaloneTxns = extractTransactions(PDF_TEXT);
97
- console.log(`extracted ${standaloneTxns.length} transactions:`);
98
- for (const t of standaloneTxns) {
99
- const dir = t.amount.direction || "?";
100
- const date = new Date(t.occurredAtMs).toISOString().slice(0, 10);
101
- console.log(` ${date} ${dir.padEnd(3)} ¥${t.amount.value.toFixed(2).padStart(8)} ${t.description}`);
102
- }
103
- if (standaloneTxns.length !== 4) {
104
- console.log(`FAIL: expected 4 transactions, got ${standaloneTxns.length}`);
105
- process.exitCode = 1;
106
- return;
107
- }
108
-
109
- // Now full pipeline
110
- console.log("\n— Full adapter pipeline —");
111
-
112
- const a = new EmailAdapter({
113
- account: { provider: "qq", email: "me@qq.com", authCode: "x", folders: ["INBOX"] },
114
- sessionFactory: makeSession(),
115
- parser: async () => ({
116
- textBody: "您的招商银行信用卡 11 月对账单已生成,详情见附件 PDF。",
117
- attachments: [{
118
- filename: "招行账单_11月.pdf",
119
- contentType: "application/pdf",
120
- contentDisposition: "attachment",
121
- size: 78_456,
122
- sha256: "abc123sha256deadbeef",
123
- isInline: false,
124
- isEncrypted: true,
125
- buffer: Buffer.from("FAKE-PDF-BYTES-DO-NOT-LEAK"),
126
- }],
127
- }),
128
- pdfExtractor: mockPdfExtractor,
129
- pdfPasswordHints: { idCardLast6: "987654", phoneLast6: "555000" },
130
- pdfPasswords: ["wrong1", "wrong2", "wrong3"], // tried before hints
131
- });
132
-
133
- console.log("adapter.version =", a.version);
134
- console.log("adapter.capabilities =", a.capabilities.join(", "));
135
- console.log("pdfPasswords (merged) =", a._pdfPasswords);
136
-
137
- let count = 0;
138
- for await (const raw of a.sync()) {
139
- count += 1;
140
- const ext = raw.payload.extraction;
141
- console.log(`\nemail #${count} subject: ${raw.payload.subject}`);
142
- console.log(" classification.category:", raw.payload.classification.category);
143
- console.log(" extraction.template :", ext.template);
144
- console.log(" extraction.confidence :", ext.confidence);
145
- console.log(" extraction.fields keys :", Object.keys(ext.fields || {}).join(", "));
146
- if (ext.fields.transactions) {
147
- console.log(` transactions[] count : ${ext.fields.transactions.length}`);
148
- for (const t of ext.fields.transactions) {
149
- const date = new Date(t.occurredAtMs).toISOString().slice(0, 10);
150
- console.log(` ${date} ¥${t.amount.value} ${t.amount.direction} ${t.description}`);
151
- }
152
- }
153
- console.log(" pdfExtraction[] :");
154
- for (const p of ext.pdfExtraction || []) {
155
- console.log(` ${p.filename}: decrypted=${p.decrypted} attempted=${p.attempted} txns=${p.transactionsExtracted ?? "-"}`);
156
- }
157
-
158
- // Buffer-leakage check
159
- const serialized = JSON.stringify(raw);
160
- if (serialized.includes("FAKE-PDF-BYTES-DO-NOT-LEAK")) {
161
- console.log("\nBUFFER LEAK ✗ — raw PDF bytes survived into payload!");
162
- process.exitCode = 1;
163
- } else {
164
- console.log(" buffer stripping : ✓ no PDF bytes in payload");
165
- }
166
- // Password-leakage check
167
- if (serialized.match(/987654/)) {
168
- console.log(" PASSWORD LEAK ✗ — real password survived into payload");
169
- process.exitCode = 1;
170
- } else {
171
- console.log(" password redaction : ✓ real password not in payload");
172
- }
173
-
174
- // Normalize and confirm transactions land in extra.fields
175
- const batch = a.normalize(raw);
176
- const ev = batch.events[0];
177
- if (ev.extra.fields && Array.isArray(ev.extra.fields.transactions)) {
178
- console.log(` normalize → extra.fields.transactions: ${ev.extra.fields.transactions.length} rows ✓`);
179
- } else {
180
- console.log(" normalize MISSING transactions in extra.fields ✗");
181
- process.exitCode = 1;
182
- }
183
- }
184
-
185
- if (count === 1 && !process.exitCode) {
186
- console.log("\n== Phase 5.5 smoke PASSED ==");
187
- } else if (!process.exitCode) {
188
- console.log(`expected 1 email, got ${count}`);
189
- process.exitCode = 1;
190
- }
191
- }
192
-
193
- main().catch((err) => {
194
- console.error("smoke failed:", err);
195
- process.exitCode = 1;
196
- });
@@ -1,181 +0,0 @@
1
- #!/usr/bin/env node
2
- /**
3
- * Phase 5.7 smoke — exercises retry-with-backoff + onProgress through a
4
- * flaky session. No real IMAP needed.
5
- *
6
- * 1. First 2 connect attempts throw ECONNRESET (transient)
7
- * 2. 3rd attempt succeeds, yields 5 envelopes
8
- * 3. Verify progress events fire in order: connecting → error → connecting
9
- * → error → connecting → connected → mailbox-opened → fetching × 5 → done
10
- * 4. Verify retry was capped (3 attempts total)
11
- * 5. Verify AUTH_FAILED does NOT retry (separate run)
12
- */
13
-
14
- "use strict";
15
-
16
- const { EmailAdapter } = require("../lib/adapters/email-imap/email-adapter");
17
- const { ImapAuthFailedError } = require("../lib/adapters/email-imap/imap-session");
18
-
19
- function makeFlakyFactory(failuresFirst, envelopes) {
20
- const failures = failuresFirst.slice();
21
- const recorder = { attempts: 0 };
22
- const factory = () => {
23
- let openMb = null;
24
- return {
25
- async connect() {
26
- recorder.attempts += 1;
27
- if (failures.length > 0) {
28
- throw failures.shift();
29
- }
30
- },
31
- async openMailbox(name) {
32
- openMb = name;
33
- return { uidValidity: 1, uidNext: 9999, exists: envelopes.length };
34
- },
35
- async *fetchFullSince(sinceUid = 0) {
36
- for (const env of envelopes) {
37
- if (env.uid > sinceUid) yield { ...env, source: env.source || Buffer.alloc(0) };
38
- }
39
- },
40
- async close() {},
41
- };
42
- };
43
- return { factory, recorder };
44
- }
45
-
46
- function makeEnv(uid) {
47
- return {
48
- uid,
49
- internalDate: new Date(`2026-05-${String(uid).padStart(2, "0")}T10:00:00Z`),
50
- flags: ["\\Seen"],
51
- messageId: `<m-${uid}@x>`,
52
- subject: `Subject ${uid}`,
53
- from: [{ address: `s${uid}@example.com` }],
54
- to: [{ address: "me@example.com" }],
55
- cc: [],
56
- date: new Date(`2026-05-${String(uid).padStart(2, "0")}T10:00:00Z`),
57
- size: 1024,
58
- };
59
- }
60
-
61
- async function scenarioRetrySucceeds() {
62
- console.log("\n=== Scenario A — 2 fails then success, 5 envelopes ===");
63
- const transient = Object.assign(new Error("ECONNRESET"), { code: "ECONNRESET" });
64
- const envs = [1, 2, 3, 4, 5].map(makeEnv);
65
- const { factory, recorder } = makeFlakyFactory([transient, transient], envs);
66
-
67
- const events = [];
68
- const a = new EmailAdapter({
69
- account: { provider: "qq", email: "me@qq.com", authCode: "x", folders: ["INBOX"] },
70
- sessionFactory: factory,
71
- parser: async () => ({ textBody: "", attachments: [] }),
72
- maxConnectRetries: 3,
73
- retryBaseDelayMs: 5,
74
- onProgress: (e) => events.push(e),
75
- });
76
-
77
- console.log("adapter.version =", a.version);
78
- console.log("adapter.capabilities (Phase 5.7) =", a.capabilities.filter((c) => c.startsWith("sync:")).join(", "));
79
-
80
- const raws = [];
81
- for await (const r of a.sync()) raws.push(r);
82
- console.log(`emitted ${raws.length} raws (expected 5)`);
83
- console.log(`connect attempts: ${recorder.attempts} (expected 3)`);
84
-
85
- const phaseSeq = events.map((e) => `${e.phase}${e.attempt ? "(" + e.attempt + ")" : ""}`);
86
- console.log("phase sequence:");
87
- for (const p of phaseSeq) console.log(" -", p);
88
-
89
- // Verify expected phase order
90
- const errs = events.filter((e) => e.phase === "error");
91
- if (errs.length !== 2) {
92
- console.error(`FAIL: expected 2 error events, got ${errs.length}`);
93
- process.exitCode = 1;
94
- } else if (!errs.every((e) => e.retriable === true)) {
95
- console.error("FAIL: error events should be retriable=true during first 2 attempts");
96
- process.exitCode = 1;
97
- } else {
98
- console.log("error events: ✓ both marked retriable");
99
- }
100
-
101
- const done = events.find((e) => e.phase === "done");
102
- if (!done || done.emitted !== 5) {
103
- console.error(`FAIL: expected done event with emitted=5, got ${JSON.stringify(done)}`);
104
- process.exitCode = 1;
105
- } else {
106
- console.log(`done event: ✓ emitted=${done.emitted} durationMs=${done.durationMs}`);
107
- }
108
-
109
- const fetches = events.filter((e) => e.phase === "fetching");
110
- if (fetches.length !== 5) {
111
- console.error(`FAIL: expected 5 fetching events, got ${fetches.length}`);
112
- process.exitCode = 1;
113
- } else if (fetches[0].total !== 5 || fetches[4].current !== 5) {
114
- console.error(`FAIL: fetching events should run 1..5 of 5`);
115
- process.exitCode = 1;
116
- } else {
117
- console.log("fetching events: ✓ 5 events with current/total");
118
- }
119
- }
120
-
121
- async function scenarioAuthFailedNoRetry() {
122
- console.log("\n=== Scenario B — AUTH_FAILED never retries ===");
123
- const authErr = new ImapAuthFailedError("bad creds");
124
- const { factory, recorder } = makeFlakyFactory(
125
- [authErr, authErr, authErr], // shouldn't matter — first one stops us
126
- [],
127
- );
128
-
129
- const events = [];
130
- const a = new EmailAdapter({
131
- account: { provider: "qq", email: "me@qq.com", authCode: "x", folders: ["INBOX"] },
132
- sessionFactory: factory,
133
- parser: async () => ({}),
134
- maxConnectRetries: 3,
135
- retryBaseDelayMs: 1,
136
- onProgress: (e) => events.push(e),
137
- });
138
-
139
- let caught = null;
140
- try {
141
- for await (const _r of a.sync()) { /* drain */ }
142
- } catch (err) {
143
- caught = err;
144
- }
145
- console.log(`connect attempts: ${recorder.attempts} (expected 1)`);
146
- console.log(`caught.code: ${caught && caught.code}`);
147
- if (recorder.attempts !== 1) {
148
- console.error("FAIL: AUTH_FAILED should not retry");
149
- process.exitCode = 1;
150
- } else if (!caught || caught.code !== "AUTH_FAILED") {
151
- console.error("FAIL: error should propagate as AUTH_FAILED");
152
- process.exitCode = 1;
153
- } else {
154
- console.log("AUTH short-circuit: ✓");
155
- }
156
-
157
- const errEvent = events.find((e) => e.phase === "error");
158
- if (!errEvent) {
159
- console.error("FAIL: error progress event missing");
160
- process.exitCode = 1;
161
- } else if (errEvent.retriable !== false) {
162
- console.error("FAIL: AUTH_FAILED error event should have retriable=false");
163
- process.exitCode = 1;
164
- } else {
165
- console.log("AUTH error event: ✓ retriable=false");
166
- }
167
- }
168
-
169
- async function main() {
170
- console.log("== Phase 5.7 smoke (retry + progress) ==");
171
- await scenarioRetrySucceeds();
172
- await scenarioAuthFailedNoRetry();
173
- if (!process.exitCode) {
174
- console.log("\n== Phase 5.7 smoke PASSED ==");
175
- }
176
- }
177
-
178
- main().catch((err) => {
179
- console.error("smoke crashed:", err);
180
- process.exitCode = 1;
181
- });
@@ -1,309 +0,0 @@
1
- #!/usr/bin/env node
2
- /**
3
- * Smoke / real-device runner for Phase 4.5.2 — Contacts extraction.
4
- *
5
- * Drives the full vertical:
6
- *
7
- * ┌──────────────────────────────────────────────────────────────┐
8
- * │ 1. (optional) android.list_devices │
9
- * │ 2. (optional) android.pull_file /data/.../contacts2.db │
10
- * │ 3. system.parse_contacts → Persons │
11
- * │ 4. hub-side UnifiedSchema validatePerson() on every row │
12
- * │ 5. write NormalizedBatch JSON to ./out/<timestamp>/ │
13
- * └──────────────────────────────────────────────────────────────┘
14
- *
15
- * Three modes:
16
- *
17
- * --db <path> Skip ADB entirely; parse a contacts2.db already
18
- * on disk. Best for first-run sanity on the dev box.
19
- *
20
- * --serial <serial> Run `adb pull` first. Requires `adb root` (most
21
- * retail builds reject this) OR a userdebug build.
22
- * On a stock Redmi 24115RA8EC use --workaround.
23
- *
24
- * --workaround sdcard Look for a contacts2.db copy at
25
- * /sdcard/Download/contacts2.db (you copied it
26
- * out via Termux + tsu, or via Mi cloud export,
27
- * per docs/design/Adapter_System_Data.md §2.1).
28
- *
29
- * Usage examples:
30
- *
31
- * # Local fixture
32
- * node scripts/smoke-system-data-contacts.js --db ./fixtures/contacts2.db
33
- *
34
- * # List devices, then prompt for serial
35
- * node scripts/smoke-system-data-contacts.js --list
36
- *
37
- * # Real device with /sdcard workaround
38
- * node scripts/smoke-system-data-contacts.js \
39
- * --serial 24115RA8ECabc123 --workaround sdcard
40
- *
41
- * Exits non-zero on any sidecar error or schema validation failure.
42
- */
43
-
44
- "use strict";
45
-
46
- const path = require("node:path");
47
- const fs = require("node:fs");
48
- const os = require("node:os");
49
-
50
- const { SidecarSupervisor } = require("../lib/sidecar");
51
- const { validatePerson } = require("../lib/schemas");
52
-
53
- const SIDECAR_ROOT = path.resolve(__dirname, "..", "..", "personal-data-hub-bridge");
54
- const PYTHON = process.env.FORENSICS_BRIDGE_PYTHON || "python";
55
-
56
- const SDCARD_WORKAROUND_PATH = "/sdcard/Download/contacts2.db";
57
- const SYSTEM_PROVIDER_PATH =
58
- "/data/data/com.android.providers.contacts/databases/contacts2.db";
59
-
60
- // ---------------------------------------------------------------------------
61
- // CLI parsing — kept dependency-free
62
- // ---------------------------------------------------------------------------
63
-
64
- function parseArgs(argv) {
65
- const out = {
66
- db: null,
67
- serial: null,
68
- workaround: null,
69
- list: false,
70
- outDir: null,
71
- help: false,
72
- };
73
- for (let i = 0; i < argv.length; i += 1) {
74
- const a = argv[i];
75
- switch (a) {
76
- case "--db":
77
- out.db = argv[++i];
78
- break;
79
- case "--serial":
80
- out.serial = argv[++i];
81
- break;
82
- case "--workaround":
83
- out.workaround = argv[++i];
84
- break;
85
- case "--list":
86
- out.list = true;
87
- break;
88
- case "--out":
89
- out.outDir = argv[++i];
90
- break;
91
- case "-h":
92
- case "--help":
93
- out.help = true;
94
- break;
95
- default:
96
- if (a.startsWith("--")) {
97
- throw new Error(`unknown flag: ${a}`);
98
- }
99
- }
100
- }
101
- return out;
102
- }
103
-
104
- function printHelp() {
105
- process.stdout.write(`
106
- smoke-system-data-contacts — drive sidecar end-to-end for contacts.
107
-
108
- --db <path> Parse a contacts2.db already on disk (skip ADB).
109
- --serial <serial> Target this ADB device for the pull step.
110
- --workaround sdcard Pull from ${SDCARD_WORKAROUND_PATH} instead of /data/data.
111
- Required on stock Android (no adb root).
112
- --list Just list ADB devices and exit.
113
- --out <dir> Write NormalizedBatch JSON here. Default: ./out/<ts>.
114
- -h, --help Show this help.
115
-
116
- Env:
117
- FORENSICS_BRIDGE_PYTHON override Python interpreter (default: python).
118
-
119
- Exit codes:
120
- 0 success
121
- 1 sidecar / hub error
122
- 2 invalid Persons (schema validation failed)
123
- `);
124
- }
125
-
126
- // ---------------------------------------------------------------------------
127
- // Helpers
128
- // ---------------------------------------------------------------------------
129
-
130
- function timestampSlug() {
131
- const d = new Date();
132
- const z = (n) => String(n).padStart(2, "0");
133
- return (
134
- `${d.getFullYear()}${z(d.getMonth() + 1)}${z(d.getDate())}-` +
135
- `${z(d.getHours())}${z(d.getMinutes())}${z(d.getSeconds())}`
136
- );
137
- }
138
-
139
- function makeSupervisor() {
140
- return new SidecarSupervisor({
141
- command: PYTHON,
142
- args: ["-u", "-m", "forensics_bridge.ipc_server"],
143
- cwd: SIDECAR_ROOT,
144
- healthCheckIntervalMs: 0,
145
- env: { PYTHONPATH: SIDECAR_ROOT },
146
- });
147
- }
148
-
149
- function log(level, msg, extra = {}) {
150
- const line = JSON.stringify({
151
- ts: new Date().toISOString(),
152
- level,
153
- msg,
154
- ...extra,
155
- });
156
- if (level === "error") process.stderr.write(line + "\n");
157
- else process.stdout.write(line + "\n");
158
- }
159
-
160
- // ---------------------------------------------------------------------------
161
- // Main
162
- // ---------------------------------------------------------------------------
163
-
164
- async function main(rawArgs) {
165
- let args;
166
- try {
167
- args = parseArgs(rawArgs);
168
- } catch (err) {
169
- console.error(err.message);
170
- printHelp();
171
- process.exit(2);
172
- }
173
- if (args.help) {
174
- printHelp();
175
- return;
176
- }
177
-
178
- const outDir = path.resolve(
179
- args.outDir || path.join(process.cwd(), "out", timestampSlug()),
180
- );
181
- fs.mkdirSync(outDir, { recursive: true });
182
- log("info", "output directory ready", { outDir });
183
-
184
- const sup = makeSupervisor();
185
- // Stream sidecar pino-style logs out as ndjson so the user sees timing.
186
- sup.on("log", (line) => process.stderr.write(`[sidecar] ${line}\n`));
187
-
188
- await sup.start({ readyTimeoutMs: 10_000 });
189
- log("info", "sidecar ready");
190
-
191
- try {
192
- // ---------- list-only path ----------
193
- if (args.list) {
194
- const devices = await sup.invoke("android.list_devices");
195
- log("info", "adb devices", devices);
196
- console.log(JSON.stringify(devices, null, 2));
197
- return;
198
- }
199
-
200
- // ---------- choose source for contacts2.db ----------
201
- let dbPath = args.db ? path.resolve(args.db) : null;
202
-
203
- if (!dbPath) {
204
- if (!args.serial) {
205
- throw new Error(
206
- "neither --db nor --serial provided; nothing to extract",
207
- );
208
- }
209
- const remotePath =
210
- args.workaround === "sdcard" ? SDCARD_WORKAROUND_PATH : SYSTEM_PROVIDER_PATH;
211
- log("info", "pulling from device", { serial: args.serial, remotePath });
212
- const pulled = await sup.invoke(
213
- "android.pull_file",
214
- {
215
- serial: args.serial,
216
- remote_path: remotePath,
217
- local_dir: outDir,
218
- },
219
- { timeoutMs: 60_000 },
220
- );
221
- log("info", "pull completed", pulled);
222
- dbPath = pulled.local;
223
- }
224
-
225
- if (!fs.existsSync(dbPath)) {
226
- throw new Error(`contacts db not found at ${dbPath}`);
227
- }
228
-
229
- // ---------- parse + validate ----------
230
- const persons = [];
231
- let chunks = 0;
232
- const t0 = Date.now();
233
- const parseResult = await sup.invoke(
234
- "system.parse_contacts",
235
- {
236
- data_path: dbPath,
237
- device_serial: args.serial || null,
238
- },
239
- {
240
- timeoutMs: 120_000,
241
- onProgress: (p) => log("info", "progress", p),
242
- onChunk: (batch) => {
243
- chunks += 1;
244
- for (const person of batch.persons || []) persons.push(person);
245
- },
246
- },
247
- );
248
- const wallMs = Date.now() - t0;
249
- log("info", "parse completed", {
250
- ...parseResult,
251
- chunks,
252
- wallMs,
253
- personsCollected: persons.length,
254
- });
255
-
256
- // ---------- hub-side schema check ----------
257
- const invalid = [];
258
- for (const p of persons) {
259
- const v = validatePerson(p);
260
- if (!v.valid) invalid.push({ id: p.id, errors: v.errors });
261
- }
262
- if (invalid.length) {
263
- log("error", "validation failed", { count: invalid.length });
264
- fs.writeFileSync(
265
- path.join(outDir, "validation-errors.json"),
266
- JSON.stringify(invalid, null, 2),
267
- );
268
- process.exitCode = 2;
269
- } else {
270
- log("info", "all persons passed UnifiedSchema validation");
271
- }
272
-
273
- // ---------- persist for inspection ----------
274
- const dump = {
275
- schemaVersion: "0.1.0",
276
- generatedAt: new Date().toISOString(),
277
- sidecar: { pythonRoot: SIDECAR_ROOT },
278
- input: {
279
- dbPath,
280
- serial: args.serial || null,
281
- workaround: args.workaround || null,
282
- },
283
- parseResult,
284
- wallMs,
285
- persons,
286
- };
287
- const dumpPath = path.join(outDir, "contacts-normalized-batch.json");
288
- fs.writeFileSync(dumpPath, JSON.stringify(dump, null, 2));
289
- log("info", "wrote dump", { dumpPath, bytes: fs.statSync(dumpPath).size });
290
-
291
- // ---------- compact summary ----------
292
- log("info", "summary", {
293
- totalPersons: parseResult.totalPersons,
294
- withPhone: parseResult.stats?.with_phone,
295
- withEmail: parseResult.stats?.with_email,
296
- starred: parseResult.stats?.starred,
297
- invalidPersons: invalid.length,
298
- outDir,
299
- });
300
- } finally {
301
- await sup.stop({ graceMs: 2000 });
302
- }
303
- }
304
-
305
- main(process.argv.slice(2)).catch((err) => {
306
- log("error", "fatal", { name: err.name, message: err.message, code: err.code });
307
- if (err.stack) process.stderr.write(err.stack + "\n");
308
- process.exit(1);
309
- });