@chainlesschain/personal-data-hub 0.3.9 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +45 -25
- package/__tests__/adapters/apple-health.test.js +95 -0
- package/__tests__/adapters/email-templates.test.js +123 -0
- package/__tests__/adapters/family-23-collectors-scaffold.test.js +178 -0
- package/__tests__/adapters/game-genshin-scaffold.test.js +107 -0
- package/__tests__/adapters/git-activity.test.js +7 -1
- package/__tests__/adapters/local-im-pc.test.js +149 -0
- package/__tests__/adapters/netease-music.test.js +74 -0
- package/__tests__/adapters/qq-pc-direct-read.test.js +186 -0
- package/__tests__/adapters/system-data-adapter.test.js +4 -1
- package/__tests__/adapters/wechat-pc-direct-read.test.js +207 -0
- package/__tests__/adapters/weread.test.js +123 -0
- package/__tests__/analysis.test.js +120 -15
- package/__tests__/mobile-extractor-encrypted.test.js +460 -0
- package/__tests__/prompt-builder.test.js +25 -0
- package/__tests__/registry-readiness.test.js +233 -0
- package/__tests__/social-douyin-im-direct-read.test.js +311 -0
- package/__tests__/social-douyin-snapshot.test.js +5 -2
- package/__tests__/vault.test.js +99 -0
- package/lib/adapter-guide.js +520 -0
- package/lib/adapter-readiness.js +257 -0
- package/lib/adapters/_local-im-db-reader.js +218 -0
- package/lib/adapters/_local-im-pc-adapter.js +162 -0
- package/lib/adapters/apple-health/index.js +329 -0
- package/lib/adapters/dingtalk-pc/index.js +29 -0
- package/lib/adapters/edu-huawei-learning/api-client.js +47 -0
- package/lib/adapters/edu-huawei-learning/index.js +255 -0
- package/lib/adapters/edu-zuoyebang/api-client.js +48 -0
- package/lib/adapters/edu-zuoyebang/index.js +259 -0
- package/lib/adapters/email-imap/email-adapter.js +16 -0
- package/lib/adapters/email-imap/templates/bill.js +174 -18
- package/lib/adapters/feishu-pc/index.js +29 -0
- package/lib/adapters/finance-alipay/api-client.js +48 -0
- package/lib/adapters/finance-alipay/index.js +257 -0
- package/lib/adapters/game-genshin/api-client.js +59 -0
- package/lib/adapters/game-genshin/index.js +274 -0
- package/lib/adapters/game-honor-of-kings/api-client.js +54 -0
- package/lib/adapters/game-honor-of-kings/index.js +259 -0
- package/lib/adapters/netease-music/index.js +227 -0
- package/lib/adapters/qq-pc/index.js +200 -0
- package/lib/adapters/qq-pc/nt-db-reader.js +210 -0
- package/lib/adapters/social-douyin/index.js +194 -1
- package/lib/adapters/wechat/wechat-adapter.js +7 -1
- package/lib/adapters/wechat-pc/index.js +335 -0
- package/lib/adapters/wechat-pc/pc-db-reader.js +327 -0
- package/lib/adapters/weread/api-client.js +128 -0
- package/lib/adapters/weread/index.js +337 -0
- package/lib/analysis.js +65 -0
- package/lib/index.js +39 -0
- package/lib/mobile-extractor/bplist.js +233 -0
- package/lib/mobile-extractor/ios-backup-crypto.js +315 -0
- package/lib/mobile-extractor/ios.js +131 -16
- package/lib/prompt-builder.js +11 -1
- package/lib/registry.js +170 -0
- package/lib/vault.js +105 -0
- package/package.json +1 -1
- package/scripts/run-native-tests-sandbox.sh +2 -0
- package/vitest.config.js +79 -1
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* AdapterRegistry.readiness() — the "why can't I collect" surface.
|
|
5
|
+
*
|
|
6
|
+
* Uses a STUB vault (readiness only calls vault.getWatermark, defensively)
|
|
7
|
+
* so this file does NOT depend on the native SQLCipher driver and runs on
|
|
8
|
+
* every host — unlike registry.test.js which opens a real LocalVault and is
|
|
9
|
+
* auto-skipped when bs3mc's ABI doesn't match the host Node. See
|
|
10
|
+
* vitest.config.js NATIVE_DEPENDENT_TESTS.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { describe, it, expect } from "vitest";
|
|
14
|
+
|
|
15
|
+
const fs = require("node:fs");
|
|
16
|
+
const os = require("node:os");
|
|
17
|
+
const path = require("node:path");
|
|
18
|
+
|
|
19
|
+
const { AdapterRegistry } = require("../lib/registry");
|
|
20
|
+
const {
|
|
21
|
+
READINESS_CATEGORY,
|
|
22
|
+
READINESS_STATUS,
|
|
23
|
+
} = require("../lib/adapter-readiness");
|
|
24
|
+
const { BilibiliAdapter } = require("../lib/adapters/social-bilibili");
|
|
25
|
+
const { TelegramAdapter } = require("../lib/adapters/messaging-telegram");
|
|
26
|
+
const { Train12306Adapter } = require("../lib/adapters/travel-12306");
|
|
27
|
+
const { EmailAdapter } = require("../lib/adapters/email-imap");
|
|
28
|
+
const { WechatAdapter } = require("../lib/adapters/wechat");
|
|
29
|
+
|
|
30
|
+
// ─── Stub vault — readiness() only needs getWatermark ─────────────────────
|
|
31
|
+
|
|
32
|
+
function stubVault(watermarks = {}) {
|
|
33
|
+
return {
|
|
34
|
+
_wm: watermarks,
|
|
35
|
+
getWatermark(adapter /*, scope */) {
|
|
36
|
+
return this._wm[adapter] || null;
|
|
37
|
+
},
|
|
38
|
+
audit() {},
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function byName(reports, name) {
|
|
43
|
+
return reports.find((r) => r.name === name);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
describe("AdapterRegistry.readiness()", () => {
|
|
47
|
+
it("snapshot adapter with no input → needs_setup / NO_INPUT", async () => {
|
|
48
|
+
const reg = new AdapterRegistry({ vault: stubVault() });
|
|
49
|
+
reg.register(new BilibiliAdapter());
|
|
50
|
+
const [r] = await reg.readiness();
|
|
51
|
+
expect(r.name).toBe("social-bilibili");
|
|
52
|
+
expect(r.ready).toBe(false);
|
|
53
|
+
expect(r.status).toBe(READINESS_STATUS.NEEDS_SETUP);
|
|
54
|
+
expect(r.reason).toBe("NO_INPUT");
|
|
55
|
+
expect(r.category).toBe(READINESS_CATEGORY.SNAPSHOT);
|
|
56
|
+
expect(typeof r.message).toBe("string");
|
|
57
|
+
expect(r.message.length).toBeGreaterThan(0);
|
|
58
|
+
expect(r.actionHint).toBeTruthy();
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
it("device-pull adapter (telegram) → needs_setup / DB_NOT_PULLED / device", async () => {
|
|
62
|
+
const reg = new AdapterRegistry({ vault: stubVault() });
|
|
63
|
+
reg.register(new TelegramAdapter());
|
|
64
|
+
const [r] = await reg.readiness();
|
|
65
|
+
expect(r.ready).toBe(false);
|
|
66
|
+
expect(r.reason).toBe("DB_NOT_PULLED");
|
|
67
|
+
expect(r.category).toBe(READINESS_CATEGORY.DEVICE);
|
|
68
|
+
expect(r.extractMode).toBe("device-pull");
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
it("12306 snapshot adapter → needs_setup", async () => {
|
|
72
|
+
const reg = new AdapterRegistry({ vault: stubVault() });
|
|
73
|
+
reg.register(new Train12306Adapter());
|
|
74
|
+
const [r] = await reg.readiness();
|
|
75
|
+
expect(r.ready).toBe(false);
|
|
76
|
+
expect(r.reason).toBe("NO_INPUT");
|
|
77
|
+
});
|
|
78
|
+
|
|
79
|
+
it("email snapshot stub → NO_INPUT (no live IMAP login)", async () => {
|
|
80
|
+
const reg = new AdapterRegistry({ vault: stubVault() });
|
|
81
|
+
reg.register(new EmailAdapter({ snapshotMode: true }));
|
|
82
|
+
const [r] = await reg.readiness();
|
|
83
|
+
expect(r.ready).toBe(false);
|
|
84
|
+
expect(r.reason).toBe("NO_INPUT");
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
it("email per-account → ready=configured WITHOUT opening an IMAP session", async () => {
|
|
88
|
+
let sessionFactoryCalled = false;
|
|
89
|
+
const adapter = new EmailAdapter({
|
|
90
|
+
account: { email: "user@gmail.com", authCode: "secret", provider: "gmail" },
|
|
91
|
+
// If readiness wrongly performed a live login it would call this.
|
|
92
|
+
sessionFactory: () => {
|
|
93
|
+
sessionFactoryCalled = true;
|
|
94
|
+
return { connect: async () => {}, close: async () => {} };
|
|
95
|
+
},
|
|
96
|
+
});
|
|
97
|
+
const reg = new AdapterRegistry({ vault: stubVault() });
|
|
98
|
+
reg.register(adapter);
|
|
99
|
+
const [r] = await reg.readiness();
|
|
100
|
+
expect(r.ready).toBe(true);
|
|
101
|
+
expect(r.status).toBe(READINESS_STATUS.READY);
|
|
102
|
+
expect(r.mode).toBe("configured");
|
|
103
|
+
expect(sessionFactoryCalled).toBe(false);
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
it("wechat readiness with db+keyProvider present → configured WITHOUT calling getKey", async () => {
|
|
107
|
+
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "pdh-rd-wx-"));
|
|
108
|
+
const dbPath = path.join(tmp, "EnMicroMsg.db");
|
|
109
|
+
fs.writeFileSync(dbPath, "x");
|
|
110
|
+
let getKeyCalled = false;
|
|
111
|
+
const adapter = new WechatAdapter({
|
|
112
|
+
account: { uin: "12345" },
|
|
113
|
+
dbPath,
|
|
114
|
+
keyProvider: {
|
|
115
|
+
getKey: async () => {
|
|
116
|
+
getKeyCalled = true;
|
|
117
|
+
return "deadbeef";
|
|
118
|
+
},
|
|
119
|
+
},
|
|
120
|
+
});
|
|
121
|
+
const reg = new AdapterRegistry({ vault: stubVault() });
|
|
122
|
+
reg.register(adapter);
|
|
123
|
+
const [r] = await reg.readiness();
|
|
124
|
+
expect(r.ready).toBe(true);
|
|
125
|
+
expect(r.mode).toBe("configured");
|
|
126
|
+
// The whole point of readinessOnly: don't invoke the (frida) key provider.
|
|
127
|
+
expect(getKeyCalled).toBe(false);
|
|
128
|
+
fs.rmSync(tmp, { recursive: true, force: true });
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
it("wechat with no db → DB_NOT_PULLED", async () => {
|
|
132
|
+
const adapter = new WechatAdapter({ account: { uin: "1" } });
|
|
133
|
+
const reg = new AdapterRegistry({ vault: stubVault() });
|
|
134
|
+
reg.register(adapter);
|
|
135
|
+
const [r] = await reg.readiness();
|
|
136
|
+
expect(r.ready).toBe(false);
|
|
137
|
+
expect(r.reason).toBe("DB_NOT_PULLED");
|
|
138
|
+
expect(r.category).toBe(READINESS_CATEGORY.DEVICE);
|
|
139
|
+
});
|
|
140
|
+
|
|
141
|
+
it("a hanging authenticate() hits the per-adapter timeout → PROBE_TIMEOUT", async () => {
|
|
142
|
+
const reg = new AdapterRegistry({ vault: stubVault() });
|
|
143
|
+
reg.register({
|
|
144
|
+
name: "hang-test",
|
|
145
|
+
version: "1.0.0",
|
|
146
|
+
capabilities: [],
|
|
147
|
+
dataDisclosure: { fields: [], sensitivity: "low" },
|
|
148
|
+
authenticate: () => new Promise(() => {}), // never resolves
|
|
149
|
+
healthCheck: async () => ({ ok: true }),
|
|
150
|
+
normalize: (r) => r,
|
|
151
|
+
// eslint-disable-next-line require-yield
|
|
152
|
+
sync: async function* () {},
|
|
153
|
+
});
|
|
154
|
+
const [r] = await reg.readiness({ timeoutMs: 200 });
|
|
155
|
+
expect(r.ready).toBe(false);
|
|
156
|
+
expect(r.reason).toBe("PROBE_TIMEOUT");
|
|
157
|
+
expect(r.status).toBe(READINESS_STATUS.ERROR);
|
|
158
|
+
});
|
|
159
|
+
|
|
160
|
+
it("an unknown reason code falls back to UNKNOWN (never crashes)", async () => {
|
|
161
|
+
const reg = new AdapterRegistry({ vault: stubVault() });
|
|
162
|
+
reg.register({
|
|
163
|
+
name: "weird",
|
|
164
|
+
version: "1.0.0",
|
|
165
|
+
capabilities: [],
|
|
166
|
+
dataDisclosure: { fields: [], sensitivity: "low" },
|
|
167
|
+
authenticate: async () => ({ ok: false, reason: "TOTALLY_NEW_CODE_42" }),
|
|
168
|
+
healthCheck: async () => ({ ok: true }),
|
|
169
|
+
normalize: (r) => r,
|
|
170
|
+
// eslint-disable-next-line require-yield
|
|
171
|
+
sync: async function* () {},
|
|
172
|
+
});
|
|
173
|
+
const [r] = await reg.readiness();
|
|
174
|
+
expect(r.ready).toBe(false);
|
|
175
|
+
expect(r.reason).toBe("TOTALLY_NEW_CODE_42");
|
|
176
|
+
expect(r.message).toBeTruthy(); // mapped via UNKNOWN fallback
|
|
177
|
+
});
|
|
178
|
+
|
|
179
|
+
it("folds last sync outcome from the watermark into the report", async () => {
|
|
180
|
+
const reg = new AdapterRegistry({
|
|
181
|
+
vault: stubVault({
|
|
182
|
+
"social-bilibili": {
|
|
183
|
+
last_synced_at: 1700000000000,
|
|
184
|
+
last_status: "error",
|
|
185
|
+
last_error: "boom from last run",
|
|
186
|
+
},
|
|
187
|
+
}),
|
|
188
|
+
});
|
|
189
|
+
reg.register(new BilibiliAdapter());
|
|
190
|
+
const [r] = await reg.readiness();
|
|
191
|
+
expect(r.lastSyncedAt).toBe(1700000000000);
|
|
192
|
+
expect(r.lastStatus).toBe("error");
|
|
193
|
+
expect(r.lastError).toBe("boom from last run");
|
|
194
|
+
});
|
|
195
|
+
|
|
196
|
+
it("attaches a step-by-step import guide to each report", async () => {
|
|
197
|
+
const reg = new AdapterRegistry({ vault: stubVault() });
|
|
198
|
+
reg.register(new BilibiliAdapter());
|
|
199
|
+
reg.register(new WechatAdapter({ account: { uin: "1" } }));
|
|
200
|
+
const reports = await reg.readiness();
|
|
201
|
+
const bili = byName(reports, "social-bilibili");
|
|
202
|
+
expect(bili.guide).toBeTruthy();
|
|
203
|
+
expect(bili.guide.displayName).toBe("哔哩哔哩");
|
|
204
|
+
expect(Array.isArray(bili.guide.methods)).toBe(true);
|
|
205
|
+
expect(bili.guide.methods.length).toBeGreaterThan(0);
|
|
206
|
+
expect(bili.guide.methods[0].steps.length).toBeGreaterThan(0);
|
|
207
|
+
// wechat gets the bespoke device override, not the generic category guide
|
|
208
|
+
const wx = byName(reports, "wechat");
|
|
209
|
+
expect(wx.guide.displayName).toBe("微信(手机)");
|
|
210
|
+
expect(wx.guide.methods[0].label).toMatch(/frida|root/);
|
|
211
|
+
});
|
|
212
|
+
|
|
213
|
+
it("reports every registered adapter in registration order", async () => {
|
|
214
|
+
const reg = new AdapterRegistry({ vault: stubVault() });
|
|
215
|
+
reg.register(new BilibiliAdapter());
|
|
216
|
+
reg.register(new TelegramAdapter());
|
|
217
|
+
reg.register(new Train12306Adapter());
|
|
218
|
+
const reports = await reg.readiness();
|
|
219
|
+
expect(reports.map((r) => r.name)).toEqual([
|
|
220
|
+
"social-bilibili",
|
|
221
|
+
"messaging-telegram",
|
|
222
|
+
"travel-12306",
|
|
223
|
+
]);
|
|
224
|
+
// every report carries the required UI fields
|
|
225
|
+
for (const r of reports) {
|
|
226
|
+
expect(r).toHaveProperty("ready");
|
|
227
|
+
expect(r).toHaveProperty("status");
|
|
228
|
+
expect(r).toHaveProperty("category");
|
|
229
|
+
expect(r).toHaveProperty("message");
|
|
230
|
+
}
|
|
231
|
+
expect(byName(reports, "messaging-telegram").reason).toBe("DB_NOT_PULLED");
|
|
232
|
+
});
|
|
233
|
+
});
|
|
@@ -0,0 +1,311 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
import { describe, it, expect, beforeEach, afterEach } from "vitest";
|
|
4
|
+
|
|
5
|
+
const fs = require("node:fs");
|
|
6
|
+
const path = require("node:path");
|
|
7
|
+
const os = require("node:os");
|
|
8
|
+
|
|
9
|
+
const { DouyinAdapter } = require("../lib/adapters/social-douyin");
|
|
10
|
+
const { partitionBatch } = require("../lib/batch");
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* 本地直读样板 (Douyin <uid>_im.db local direct-read) + the normalize
|
|
14
|
+
* message/contact gap fix.
|
|
15
|
+
*
|
|
16
|
+
* Two things this covers that nothing else did:
|
|
17
|
+
*
|
|
18
|
+
* 1. REGRESSION: DouyinAdapter.normalize() used to throw "unknown kind
|
|
19
|
+
* message/contact" for IM events — so every 私信 + 联系人 silently
|
|
20
|
+
* dropped (registry catches the throw → invalidCount++ → 0 rows in the
|
|
21
|
+
* vault) even though the snapshot/ADB path "succeeded". The old snapshot
|
|
22
|
+
* test only round-tripped `profile`, so it never caught this.
|
|
23
|
+
*
|
|
24
|
+
* 2. NEW direct-read mode: `sync({ imDbPath })` / `--input <uid>_im.db`
|
|
25
|
+
* opens the plaintext SQLite directly (no ADB, no snapshot JSON) and
|
|
26
|
+
* emits message/contact raws whose originalIds match the snapshot path
|
|
27
|
+
* (idempotent across both routes).
|
|
28
|
+
*
|
|
29
|
+
* No native SQLite needed — a fake Database driver is injected via
|
|
30
|
+
* `_deps.dbDriverFactory` (the parser accepts it as `_databaseClass`).
|
|
31
|
+
*/
|
|
32
|
+
|
|
33
|
+
// Fake better-sqlite3-style driver answering the parser's PRAGMA + SELECTs.
|
|
34
|
+
function makeFakeDb({ msgRows, userRows, msgCols, userCols }) {
|
|
35
|
+
class FakeStmt {
|
|
36
|
+
constructor(sql) {
|
|
37
|
+
this.sql = sql;
|
|
38
|
+
}
|
|
39
|
+
all() {
|
|
40
|
+
const s = this.sql;
|
|
41
|
+
if (/PRAGMA table_info\(msg\)/.test(s)) return msgCols;
|
|
42
|
+
if (/FROM msg/.test(s)) return msgRows;
|
|
43
|
+
if (/PRAGMA table_info\(SIMPLE_USER\)/.test(s)) return userCols;
|
|
44
|
+
if (/FROM SIMPLE_USER/.test(s)) return userRows;
|
|
45
|
+
return [];
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
return class FakeDb {
|
|
49
|
+
// eslint-disable-next-line no-unused-vars
|
|
50
|
+
constructor(_path, _opts) {}
|
|
51
|
+
prepare(sql) {
|
|
52
|
+
return new FakeStmt(sql);
|
|
53
|
+
}
|
|
54
|
+
close() {}
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
const DEFAULT_FAKE = {
|
|
59
|
+
msgCols: [
|
|
60
|
+
{ name: "sender" },
|
|
61
|
+
{ name: "created_time" },
|
|
62
|
+
{ name: "content" },
|
|
63
|
+
{ name: "conversation_id" },
|
|
64
|
+
{ name: "read_status" },
|
|
65
|
+
],
|
|
66
|
+
msgRows: [
|
|
67
|
+
{
|
|
68
|
+
sender: 111,
|
|
69
|
+
createdTime: 1700000000000,
|
|
70
|
+
content: JSON.stringify({ text: "你好呀" }),
|
|
71
|
+
conversationId: "conv-1",
|
|
72
|
+
readStatus: 1,
|
|
73
|
+
},
|
|
74
|
+
{
|
|
75
|
+
sender: 222,
|
|
76
|
+
createdTime: 1700000001000,
|
|
77
|
+
content: JSON.stringify({ text: "在吗" }),
|
|
78
|
+
conversationId: "conv-1",
|
|
79
|
+
readStatus: 0,
|
|
80
|
+
},
|
|
81
|
+
],
|
|
82
|
+
userCols: [
|
|
83
|
+
{ name: "UID" },
|
|
84
|
+
{ name: "short_id" },
|
|
85
|
+
{ name: "name" },
|
|
86
|
+
{ name: "avatar_url" },
|
|
87
|
+
{ name: "follow_status" },
|
|
88
|
+
],
|
|
89
|
+
userRows: [
|
|
90
|
+
{
|
|
91
|
+
uid: 222,
|
|
92
|
+
shortId: 888,
|
|
93
|
+
name: "小明",
|
|
94
|
+
avatarUrl: "http://x/a.jpg",
|
|
95
|
+
followStatus: 2,
|
|
96
|
+
},
|
|
97
|
+
],
|
|
98
|
+
};
|
|
99
|
+
|
|
100
|
+
function freshAdapter(fakeSpec = DEFAULT_FAKE, fsOverride) {
|
|
101
|
+
const a = new DouyinAdapter();
|
|
102
|
+
a._deps.fs = fsOverride || { existsSync: () => true };
|
|
103
|
+
a._deps.dbDriverFactory = () => makeFakeDb(fakeSpec);
|
|
104
|
+
return a;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
async function collect(iter) {
|
|
108
|
+
const out = [];
|
|
109
|
+
for await (const r of iter) out.push(r);
|
|
110
|
+
return out;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
describe("DouyinAdapter — normalize message/contact (regression)", () => {
|
|
114
|
+
it("normalizes a message raw into one MESSAGE event (no throw)", () => {
|
|
115
|
+
const a = new DouyinAdapter();
|
|
116
|
+
const raw = {
|
|
117
|
+
adapter: "social-douyin",
|
|
118
|
+
kind: "message",
|
|
119
|
+
originalId: "douyin:message:msg-conv-1-1700000000000",
|
|
120
|
+
capturedAt: 1700000000000,
|
|
121
|
+
payload: {
|
|
122
|
+
kind: "message",
|
|
123
|
+
text: "你好",
|
|
124
|
+
senderUid: "111",
|
|
125
|
+
conversationId: "conv-1",
|
|
126
|
+
readStatus: 1,
|
|
127
|
+
contentBlob: '{"text":"你好"}',
|
|
128
|
+
},
|
|
129
|
+
};
|
|
130
|
+
const n = a.normalize(raw);
|
|
131
|
+
expect(n.events).toHaveLength(1);
|
|
132
|
+
expect(n.persons).toHaveLength(0);
|
|
133
|
+
const ev = n.events[0];
|
|
134
|
+
expect(ev.subtype).toBe("message");
|
|
135
|
+
expect(ev.content.text).toBe("你好");
|
|
136
|
+
expect(ev.extra.senderUid).toBe("111");
|
|
137
|
+
expect(ev.extra.conversationId).toBe("conv-1");
|
|
138
|
+
expect(ev.extra.platform).toBe("douyin");
|
|
139
|
+
});
|
|
140
|
+
|
|
141
|
+
it("normalizes a contact raw into one CONTACT person", () => {
|
|
142
|
+
const a = new DouyinAdapter();
|
|
143
|
+
const raw = {
|
|
144
|
+
adapter: "social-douyin",
|
|
145
|
+
kind: "contact",
|
|
146
|
+
originalId: "douyin:contact:contact-222",
|
|
147
|
+
capturedAt: 1700000000000,
|
|
148
|
+
payload: {
|
|
149
|
+
kind: "contact",
|
|
150
|
+
uid: "222",
|
|
151
|
+
shortId: "888",
|
|
152
|
+
name: "小明",
|
|
153
|
+
avatarUrl: "http://x/a.jpg",
|
|
154
|
+
followStatus: 2,
|
|
155
|
+
},
|
|
156
|
+
};
|
|
157
|
+
const n = a.normalize(raw);
|
|
158
|
+
expect(n.persons).toHaveLength(1);
|
|
159
|
+
expect(n.events).toHaveLength(0);
|
|
160
|
+
const per = n.persons[0];
|
|
161
|
+
expect(per.subtype).toBe("contact");
|
|
162
|
+
expect(per.id).toBe("person-douyin-222");
|
|
163
|
+
expect(per.names).toEqual(["小明"]);
|
|
164
|
+
expect(per.identifiers["douyin-uid"]).toEqual(["222"]);
|
|
165
|
+
expect(per.extra.followStatus).toBe(2);
|
|
166
|
+
});
|
|
167
|
+
|
|
168
|
+
it("an empty-text (non-text) message still produces a valid event", () => {
|
|
169
|
+
const a = new DouyinAdapter();
|
|
170
|
+
const raw = {
|
|
171
|
+
adapter: "social-douyin",
|
|
172
|
+
kind: "message",
|
|
173
|
+
originalId: "douyin:message:x",
|
|
174
|
+
capturedAt: 1700000000000,
|
|
175
|
+
payload: { kind: "message", text: null, senderUid: "111" },
|
|
176
|
+
};
|
|
177
|
+
const n = a.normalize(raw);
|
|
178
|
+
const { valid, invalidReasons } = partitionBatch({
|
|
179
|
+
events: n.events,
|
|
180
|
+
persons: [],
|
|
181
|
+
places: [],
|
|
182
|
+
items: [],
|
|
183
|
+
topics: [],
|
|
184
|
+
});
|
|
185
|
+
expect(invalidReasons).toHaveLength(0);
|
|
186
|
+
expect(valid.events).toHaveLength(1);
|
|
187
|
+
});
|
|
188
|
+
});
|
|
189
|
+
|
|
190
|
+
describe("DouyinAdapter — 本地直读 <uid>_im.db", () => {
|
|
191
|
+
let tmpDir;
|
|
192
|
+
beforeEach(() => {
|
|
193
|
+
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "douyin-imdb-"));
|
|
194
|
+
});
|
|
195
|
+
afterEach(() => {
|
|
196
|
+
if (tmpDir && fs.existsSync(tmpDir)) {
|
|
197
|
+
fs.rmSync(tmpDir, { recursive: true, force: true });
|
|
198
|
+
}
|
|
199
|
+
});
|
|
200
|
+
|
|
201
|
+
it("sync({ imDbPath }) yields message + contact raws", async () => {
|
|
202
|
+
const a = freshAdapter();
|
|
203
|
+
const raws = await collect(a.sync({ imDbPath: "/fake/123_im.db" }));
|
|
204
|
+
expect(raws.map((r) => r.kind)).toEqual(["message", "message", "contact"]);
|
|
205
|
+
});
|
|
206
|
+
|
|
207
|
+
it("direct-read events normalize to a fully valid batch (no silent drop)", async () => {
|
|
208
|
+
const a = freshAdapter();
|
|
209
|
+
const raws = await collect(a.sync({ imDbPath: "/fake/123_im.db" }));
|
|
210
|
+
const merged = { events: [], persons: [], places: [], items: [], topics: [] };
|
|
211
|
+
for (const r of raws) {
|
|
212
|
+
const n = a.normalize(r);
|
|
213
|
+
for (const k of Object.keys(merged)) merged[k].push(...n[k]);
|
|
214
|
+
}
|
|
215
|
+
const { valid, invalidReasons } = partitionBatch(merged);
|
|
216
|
+
expect(invalidReasons).toHaveLength(0);
|
|
217
|
+
expect(valid.events).toHaveLength(2); // two messages
|
|
218
|
+
expect(valid.persons).toHaveLength(1); // one contact
|
|
219
|
+
});
|
|
220
|
+
|
|
221
|
+
it("originalIds match the snapshot composite strategy (idempotent across routes)", async () => {
|
|
222
|
+
const a = freshAdapter();
|
|
223
|
+
const raws = await collect(a.sync({ imDbPath: "/fake/123_im.db" }));
|
|
224
|
+
expect(raws.map((r) => r.originalId)).toEqual([
|
|
225
|
+
"douyin:message:msg-conv-1-1700000000000",
|
|
226
|
+
"douyin:message:msg-conv-1-1700000001000",
|
|
227
|
+
"douyin:contact:contact-222",
|
|
228
|
+
]);
|
|
229
|
+
});
|
|
230
|
+
|
|
231
|
+
it("respects include={message:false} / limit", async () => {
|
|
232
|
+
const a = freshAdapter();
|
|
233
|
+
const onlyContacts = await collect(
|
|
234
|
+
a.sync({ imDbPath: "/fake/123_im.db", include: { message: false } }),
|
|
235
|
+
);
|
|
236
|
+
expect(onlyContacts.every((r) => r.kind === "contact")).toBe(true);
|
|
237
|
+
|
|
238
|
+
const capped = await collect(a.sync({ imDbPath: "/fake/123_im.db", limit: 1 }));
|
|
239
|
+
expect(capped).toHaveLength(1);
|
|
240
|
+
});
|
|
241
|
+
|
|
242
|
+
it("emits an im-db-parsed progress event with the diagnostic", async () => {
|
|
243
|
+
const a = freshAdapter();
|
|
244
|
+
const events = [];
|
|
245
|
+
await collect(
|
|
246
|
+
a.sync({
|
|
247
|
+
imDbPath: "/fake/123_im.db",
|
|
248
|
+
onProgress: (e) => events.push(e),
|
|
249
|
+
}),
|
|
250
|
+
);
|
|
251
|
+
const parsed = events.find((e) => e.phase === "im-db-parsed");
|
|
252
|
+
expect(parsed).toBeTruthy();
|
|
253
|
+
expect(parsed.hadMsgTable).toBe(true);
|
|
254
|
+
expect(parsed.hadSimpleUserTable).toBe(true);
|
|
255
|
+
expect(parsed.messageCount).toBe(2);
|
|
256
|
+
expect(parsed.contactCount).toBe(1);
|
|
257
|
+
});
|
|
258
|
+
|
|
259
|
+
it("missing db file yields nothing (no throw)", async () => {
|
|
260
|
+
const a = freshAdapter(DEFAULT_FAKE, { existsSync: () => false });
|
|
261
|
+
const raws = await collect(a.sync({ imDbPath: "/does/not/exist_im.db" }));
|
|
262
|
+
expect(raws).toHaveLength(0);
|
|
263
|
+
});
|
|
264
|
+
});
|
|
265
|
+
|
|
266
|
+
describe("DouyinAdapter — sync() input routing (sniff)", () => {
|
|
267
|
+
let tmpDir;
|
|
268
|
+
beforeEach(() => {
|
|
269
|
+
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "douyin-route-"));
|
|
270
|
+
});
|
|
271
|
+
afterEach(() => {
|
|
272
|
+
if (tmpDir && fs.existsSync(tmpDir)) {
|
|
273
|
+
fs.rmSync(tmpDir, { recursive: true, force: true });
|
|
274
|
+
}
|
|
275
|
+
});
|
|
276
|
+
|
|
277
|
+
it("--input <file with SQLite magic header> routes to direct IM read", async () => {
|
|
278
|
+
// Real file with the 16-byte SQLite magic header so _looksLikeSqlite
|
|
279
|
+
// (which uses real fs) returns true; the fake driver supplies the rows.
|
|
280
|
+
const dbFile = path.join(tmpDir, "123_im.db");
|
|
281
|
+
const header = Buffer.alloc(100);
|
|
282
|
+
header.write("SQLite format 3", 0, "latin1");
|
|
283
|
+
fs.writeFileSync(dbFile, header);
|
|
284
|
+
|
|
285
|
+
const a = new DouyinAdapter();
|
|
286
|
+
a._deps.dbDriverFactory = () => makeFakeDb(DEFAULT_FAKE);
|
|
287
|
+
const raws = [];
|
|
288
|
+
for await (const r of a.sync({ inputPath: dbFile })) raws.push(r);
|
|
289
|
+
expect(raws.map((r) => r.kind)).toEqual(["message", "message", "contact"]);
|
|
290
|
+
});
|
|
291
|
+
|
|
292
|
+
it("--input <JSON snapshot> routes to snapshot mode (not IM)", async () => {
|
|
293
|
+
const snapFile = path.join(tmpDir, "social-douyin.json");
|
|
294
|
+
fs.writeFileSync(
|
|
295
|
+
snapFile,
|
|
296
|
+
JSON.stringify({
|
|
297
|
+
schemaVersion: 1,
|
|
298
|
+
snapshottedAt: 1700000000000,
|
|
299
|
+
account: { secUid: "MS4abc", shortId: "9", displayName: "me" },
|
|
300
|
+
events: [
|
|
301
|
+
{ kind: "profile", id: "profile-MS4abc", capturedAt: 1700000000000, secUid: "MS4abc", nickname: "me" },
|
|
302
|
+
],
|
|
303
|
+
}),
|
|
304
|
+
);
|
|
305
|
+
const a = new DouyinAdapter();
|
|
306
|
+
const raws = [];
|
|
307
|
+
for await (const r of a.sync({ inputPath: snapFile })) raws.push(r);
|
|
308
|
+
expect(raws).toHaveLength(1);
|
|
309
|
+
expect(raws[0].kind).toBe("profile");
|
|
310
|
+
});
|
|
311
|
+
});
|
|
@@ -33,14 +33,17 @@ describe("DouyinAdapter snapshot mode", () => {
|
|
|
33
33
|
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "douyin-snap-"));
|
|
34
34
|
});
|
|
35
35
|
|
|
36
|
-
it("exports SNAPSHOT_SCHEMA_VERSION = 1 +
|
|
36
|
+
it("exports SNAPSHOT_SCHEMA_VERSION = 1 + 6 VALID_SNAPSHOT_KINDS", () => {
|
|
37
37
|
expect(SNAPSHOT_SCHEMA_VERSION).toBe(1);
|
|
38
|
-
//
|
|
38
|
+
// Forward-compat list (lib index.js): profile/history/favourite/like from
|
|
39
|
+
// v0.2/v0.3, plus message/contact added in Phase 2a (3c5126401, _im.db pull).
|
|
39
40
|
expect(VALID_SNAPSHOT_KINDS).toEqual([
|
|
40
41
|
"profile",
|
|
41
42
|
"history",
|
|
42
43
|
"favourite",
|
|
43
44
|
"like",
|
|
45
|
+
"message",
|
|
46
|
+
"contact",
|
|
44
47
|
]);
|
|
45
48
|
});
|
|
46
49
|
|
package/__tests__/vault.test.js
CHANGED
|
@@ -666,3 +666,102 @@ describe("LocalVault.stats", () => {
|
|
|
666
666
|
expect(s.auditLog).toBeGreaterThanOrEqual(1);
|
|
667
667
|
});
|
|
668
668
|
});
|
|
669
|
+
|
|
670
|
+
describe("LocalVault.sumEventAmount", () => {
|
|
671
|
+
// shopping/travel shape: content.amount = { value, currency, direction }
|
|
672
|
+
const shopEvent = (amount, over = {}) =>
|
|
673
|
+
eventOk({
|
|
674
|
+
subtype: "order",
|
|
675
|
+
source: source({ adapter: "shopping-jd" }),
|
|
676
|
+
content: { title: "订单", amount },
|
|
677
|
+
...over,
|
|
678
|
+
});
|
|
679
|
+
// alipay shape: extra.amountFen (cents) + extra.direction
|
|
680
|
+
const alipayEvent = (amountFen, direction, over = {}) =>
|
|
681
|
+
eventOk({
|
|
682
|
+
subtype: "payment",
|
|
683
|
+
source: source({ adapter: "finance-alipay" }),
|
|
684
|
+
content: { title: "支付" },
|
|
685
|
+
extra: { amountFen, direction },
|
|
686
|
+
...over,
|
|
687
|
+
});
|
|
688
|
+
|
|
689
|
+
it("sums content.amount (shopping/travel) split by direction", () => {
|
|
690
|
+
freshVault();
|
|
691
|
+
vault.putEvent(shopEvent({ value: 100, currency: "CNY", direction: "out" }));
|
|
692
|
+
vault.putEvent(shopEvent({ value: 30, currency: "CNY", direction: "out" }));
|
|
693
|
+
vault.putEvent(shopEvent({ value: 50, currency: "CNY", direction: "in" }));
|
|
694
|
+
const r = vault.sumEventAmount();
|
|
695
|
+
expect(r.count).toBe(3);
|
|
696
|
+
expect(r.byDirection.out).toBe(130);
|
|
697
|
+
expect(r.byDirection.in).toBe(50);
|
|
698
|
+
expect(r.total).toBe(180);
|
|
699
|
+
expect(r.currency).toBe("CNY");
|
|
700
|
+
});
|
|
701
|
+
|
|
702
|
+
it("sums extra.amountFen (alipay), converting cents → yuan", () => {
|
|
703
|
+
freshVault();
|
|
704
|
+
vault.putEvent(alipayEvent(12345, "out"));
|
|
705
|
+
vault.putEvent(alipayEvent(5500, "in"));
|
|
706
|
+
const r = vault.sumEventAmount();
|
|
707
|
+
expect(r.count).toBe(2);
|
|
708
|
+
expect(r.byDirection.out).toBe(123.45);
|
|
709
|
+
expect(r.byDirection.in).toBe(55);
|
|
710
|
+
});
|
|
711
|
+
|
|
712
|
+
it("excludes events with no extractable amount (messages/visits)", () => {
|
|
713
|
+
freshVault();
|
|
714
|
+
vault.putEvent(eventOk({ subtype: "message", content: { text: "hi" }, source: source({ adapter: "wechat" }) }));
|
|
715
|
+
vault.putEvent(shopEvent({ value: 10, currency: "CNY", direction: "out" }));
|
|
716
|
+
const r = vault.sumEventAmount();
|
|
717
|
+
expect(r.count).toBe(1);
|
|
718
|
+
expect(r.total).toBe(10);
|
|
719
|
+
});
|
|
720
|
+
|
|
721
|
+
it("filters by adapter and by time window", () => {
|
|
722
|
+
freshVault();
|
|
723
|
+
vault.putEvent(shopEvent({ value: 10, currency: "CNY", direction: "out" }, { occurredAt: 1000, source: source({ adapter: "shopping-jd" }) }));
|
|
724
|
+
vault.putEvent(shopEvent({ value: 20, currency: "CNY", direction: "out" }, { occurredAt: 5000, source: source({ adapter: "shopping-taobao" }) }));
|
|
725
|
+
expect(vault.sumEventAmount({ adapter: "shopping-jd" }).total).toBe(10);
|
|
726
|
+
expect(vault.sumEventAmount({ since: 2000 }).total).toBe(20);
|
|
727
|
+
expect(vault.sumEventAmount({ until: 2000 }).total).toBe(10);
|
|
728
|
+
});
|
|
729
|
+
|
|
730
|
+
it("mixed shapes coexist; per-currency breakdown, NO cross-currency sum", () => {
|
|
731
|
+
freshVault();
|
|
732
|
+
vault.putEvent(shopEvent({ value: 100, currency: "CNY", direction: "out" }));
|
|
733
|
+
vault.putEvent(alipayEvent(20000, "out")); // 200 元 (CNY, alipay shape)
|
|
734
|
+
vault.putEvent(shopEvent({ value: 5, currency: "USD", direction: "out" }));
|
|
735
|
+
const r = vault.sumEventAmount();
|
|
736
|
+
expect(r.count).toBe(3);
|
|
737
|
+
// CNY has 2 events → primary; top-level reports CNY only (NOT 305 cross-sum).
|
|
738
|
+
expect(r.currency).toBe("CNY");
|
|
739
|
+
expect(r.total).toBe(300);
|
|
740
|
+
expect(r.byDirection.out).toBe(300);
|
|
741
|
+
// Full breakdown per currency.
|
|
742
|
+
expect(r.byCurrency.CNY).toEqual({ total: 300, count: 2, byDirection: { out: 300, in: 0 } });
|
|
743
|
+
expect(r.byCurrency.USD).toEqual({ total: 5, count: 1, byDirection: { out: 5, in: 0 } });
|
|
744
|
+
});
|
|
745
|
+
|
|
746
|
+
it("single currency → byCurrency has one entry matching top-level", () => {
|
|
747
|
+
freshVault();
|
|
748
|
+
vault.putEvent(shopEvent({ value: 40, currency: "CNY", direction: "out" }));
|
|
749
|
+
vault.putEvent(shopEvent({ value: 10, currency: "CNY", direction: "in" }));
|
|
750
|
+
const r = vault.sumEventAmount();
|
|
751
|
+
expect(Object.keys(r.byCurrency)).toEqual(["CNY"]);
|
|
752
|
+
expect(r.byCurrency.CNY).toEqual({ total: 50, count: 2, byDirection: { out: 40, in: 10 } });
|
|
753
|
+
expect(r.total).toBe(50);
|
|
754
|
+
expect(r.currency).toBe("CNY");
|
|
755
|
+
});
|
|
756
|
+
|
|
757
|
+
it("empty vault → zeros, CNY, count 0, empty byCurrency", () => {
|
|
758
|
+
freshVault();
|
|
759
|
+
expect(vault.sumEventAmount()).toEqual({
|
|
760
|
+
total: 0,
|
|
761
|
+
currency: "CNY",
|
|
762
|
+
byCurrency: {},
|
|
763
|
+
count: 0,
|
|
764
|
+
byDirection: { out: 0, in: 0 },
|
|
765
|
+
});
|
|
766
|
+
});
|
|
767
|
+
});
|