@chainlesschain/personal-data-hub 0.2.2 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/__tests__/adapters/social-toutiao-kuaishou-scaffold.test.js +58 -16
- package/__tests__/longtail-adapters.test.js +60 -14
- package/__tests__/messaging-qq-snapshot.test.js +294 -0
- package/__tests__/shopping-pinduoduo-snapshot.test.js +302 -0
- package/__tests__/shopping-snapshot.test.js +438 -0
- package/__tests__/social-adapters.test.js +28 -3
- package/__tests__/social-douyin-snapshot.test.js +253 -0
- package/__tests__/social-kuaishou-snapshot.test.js +309 -0
- package/__tests__/social-toutiao-snapshot.test.js +314 -0
- package/__tests__/social-weibo-snapshot.test.js +234 -0
- package/__tests__/social-xiaohongshu-snapshot.test.js +232 -0
- package/__tests__/travel-maps-snapshot.test.js +426 -0
- package/__tests__/vault-driver-error.test.js +74 -0
- package/lib/adapters/messaging-qq/index.js +498 -92
- package/lib/adapters/shopping-jd/index.js +228 -25
- package/lib/adapters/shopping-meituan/index.js +222 -26
- package/lib/adapters/shopping-pinduoduo/index.js +275 -0
- package/lib/adapters/social-douyin/index.js +454 -63
- package/lib/adapters/social-kuaishou/index.js +379 -127
- package/lib/adapters/social-toutiao/index.js +400 -130
- package/lib/adapters/social-weibo/index.js +393 -95
- package/lib/adapters/social-xiaohongshu/index.js +389 -49
- package/lib/adapters/travel-baidu-map/index.js +286 -26
- package/lib/adapters/travel-tencent-map/index.js +414 -0
- package/lib/index.js +5 -1
- package/lib/vault.js +60 -8
- package/package.json +2 -1
|
@@ -1,158 +1,564 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Phase 13.5 — QQ adapter.
|
|
2
|
+
* §Phase 13.5 v0.2 — QQ adapter, dual-mode (snapshot + sqlite).
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
4
|
+
* Mirror of social-weibo/index.js dual-mode pattern, adapted to the QQ
|
|
5
|
+
* data model. Two modes share normalize() but yield from different
|
|
6
|
+
* sources:
|
|
7
7
|
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
8
|
+
* 1. snapshot mode (opts.inputPath): in-APK Android cc reads a snapshot
|
|
9
|
+
* JSON produced by QQLocalCollector (root + plain SQLite + per-row
|
|
10
|
+
* XOR-with-IMEI decrypt — much simpler than WeChat's SQLCipher path).
|
|
11
|
+
* Adapter stateless when in snapshot mode — account.qq pulled from
|
|
12
|
+
* the snapshot file.
|
|
13
|
+
*
|
|
14
|
+
* 2. sqlite mode (opts.dbPath, legacy Phase 13.5): desktop AndroidExtractor
|
|
15
|
+
* pulls QQ's per-uin DB via `adb backup`; reads same tables, applies
|
|
16
|
+
* same XOR decrypt via a `keyProvider`. account.qq REQUIRED at sync
|
|
17
|
+
* time (defended in _syncViaSqlite, not the constructor — mirror of
|
|
18
|
+
* weibo / bilibili A8 pattern).
|
|
19
|
+
*
|
|
20
|
+
* QQ specifics (vs WeChat 12.10):
|
|
21
|
+
* - DB at /data/data/com.tencent.mobileqq/databases/<uin>.db (per-uin)
|
|
22
|
+
* - DB itself is plain SQLite — NOT SQLCipher-encrypted
|
|
23
|
+
* - Message content (`msgData` BLOB) XOR-cycled with device IMEI bytes
|
|
24
|
+
* - Message tables sharded as mr_friend_<MD5(peer_uin).upper()>_New
|
|
25
|
+
* and mr_troop_<MD5(troop_uin).upper()>_New
|
|
26
|
+
* - Contacts: tries Friends / friends / tb_recent_contact (table-name
|
|
27
|
+
* drift across QQ versions; we probe all three)
|
|
28
|
+
* - Groups: TroopInfoV2
|
|
29
|
+
*
|
|
30
|
+
* Snapshot schema (mirrors QQLocalCollector.SNAPSHOT_SCHEMA_VERSION):
|
|
31
|
+
*
|
|
32
|
+
* {
|
|
33
|
+
* "schemaVersion": 1,
|
|
34
|
+
* "snapshottedAt": <epoch-ms>,
|
|
35
|
+
* "account": { "qq": "12345", "displayName": "alice" },
|
|
36
|
+
* "events": [
|
|
37
|
+
* { "kind": "contact", "id": "contact-<peerUin>", "capturedAt": <ms>,
|
|
38
|
+
* "uin": "<peerUin>", "nickname": "...", "remark": "..." },
|
|
39
|
+
* { "kind": "group", "id": "group-<troopUin>", "capturedAt": <ms>,
|
|
40
|
+
* "troopUin": "<troopUin>", "troopName": "...",
|
|
41
|
+
* "memberCount": N, "ownerUin": "..." },
|
|
42
|
+
* { "kind": "message", "id": "msg-<msgId>", "capturedAt": <ms>,
|
|
43
|
+
* "msgId": "...", "msgType": N, "senderUin": "<senderUin>",
|
|
44
|
+
* "peerUin": "<friendOrTroopUin>", "isGroup": true|false,
|
|
45
|
+
* "isSend": true|false, "text": "<decrypted content>" }
|
|
46
|
+
* ]
|
|
47
|
+
* }
|
|
14
48
|
*/
|
|
15
49
|
|
|
16
50
|
"use strict";
|
|
17
51
|
|
|
18
52
|
const fs = require("node:fs");
|
|
19
53
|
const { newId } = require("../../ids");
|
|
54
|
+
const {
|
|
55
|
+
ENTITY_TYPES,
|
|
56
|
+
PERSON_SUBTYPES,
|
|
57
|
+
EVENT_SUBTYPES,
|
|
58
|
+
CAPTURED_BY,
|
|
59
|
+
} = require("../../constants");
|
|
20
60
|
|
|
21
61
|
const NAME = "messaging-qq";
|
|
22
|
-
const VERSION = "0.
|
|
62
|
+
const VERSION = "0.6.0";
|
|
63
|
+
const SNAPSHOT_SCHEMA_VERSION = 1;
|
|
64
|
+
|
|
65
|
+
const KIND_CONTACT = "contact";
|
|
66
|
+
const KIND_GROUP = "group";
|
|
67
|
+
const KIND_MESSAGE = "message";
|
|
68
|
+
const VALID_SNAPSHOT_KINDS = Object.freeze([
|
|
69
|
+
KIND_CONTACT,
|
|
70
|
+
KIND_GROUP,
|
|
71
|
+
KIND_MESSAGE,
|
|
72
|
+
]);
|
|
73
|
+
|
|
74
|
+
function stableOriginalId(kind, id) {
|
|
75
|
+
const stringified =
|
|
76
|
+
(typeof id === "string" && id.length > 0 && id) ||
|
|
77
|
+
(typeof id === "number" && Number.isFinite(id) && String(id)) ||
|
|
78
|
+
null;
|
|
79
|
+
const safe =
|
|
80
|
+
stringified ||
|
|
81
|
+
`unknown-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
|
82
|
+
return `qq:${kind}:${safe}`;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function parseTime(v) {
|
|
86
|
+
if (Number.isFinite(v)) return v > 1e12 ? v : v * 1000;
|
|
87
|
+
if (typeof v === "string") {
|
|
88
|
+
if (/^\d+$/.test(v)) {
|
|
89
|
+
const n = parseInt(v, 10);
|
|
90
|
+
return n > 1e12 ? n : n * 1000;
|
|
91
|
+
}
|
|
92
|
+
const t = Date.parse(v);
|
|
93
|
+
return Number.isFinite(t) ? t : null;
|
|
94
|
+
}
|
|
95
|
+
return null;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
function trySelect(db, sql) {
|
|
99
|
+
try {
|
|
100
|
+
return db.prepare(sql).all();
|
|
101
|
+
} catch (_e) {
|
|
102
|
+
return null;
|
|
103
|
+
}
|
|
104
|
+
}
|
|
23
105
|
|
|
24
106
|
class QQAdapter {
|
|
25
107
|
constructor(opts = {}) {
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
this.account = opts.account;
|
|
108
|
+
// §Phase 13.5 v0.2: account.qq now OPTIONAL at construction — snapshot
|
|
109
|
+
// mode is stateless and pulls account from the snapshot file. Sqlite
|
|
110
|
+
// mode (legacy device-pull) still requires it; checked at sync time.
|
|
111
|
+
this.account = opts.account || null;
|
|
30
112
|
this._dbPath = opts.dbPath || null;
|
|
31
113
|
this._keyProvider = opts.keyProvider || null;
|
|
32
|
-
this._dbDriverFactory = opts.dbDriverFactory || null;
|
|
33
114
|
|
|
34
115
|
this.name = NAME;
|
|
35
116
|
this.version = VERSION;
|
|
36
|
-
this.capabilities = [
|
|
117
|
+
this.capabilities = [
|
|
118
|
+
"sync:snapshot",
|
|
119
|
+
"sync:sqlite",
|
|
120
|
+
"parse:qq-contacts",
|
|
121
|
+
"parse:qq-groups",
|
|
122
|
+
"parse:qq-messages",
|
|
123
|
+
"decrypt:xor-imei",
|
|
124
|
+
];
|
|
125
|
+
// Kept as device-pull — both modes are sourced from on-device data.
|
|
37
126
|
this.extractMode = "device-pull";
|
|
38
127
|
this.rateLimits = {};
|
|
39
128
|
this.dataDisclosure = {
|
|
40
129
|
fields: [
|
|
41
|
-
"qq:
|
|
42
|
-
"qq:groups (troop_uin / name)",
|
|
43
|
-
"qq:messages (peer / content / time / type)",
|
|
130
|
+
"qq:contacts (uin / nickname / remark)",
|
|
131
|
+
"qq:groups (troop_uin / name / member_count)",
|
|
132
|
+
"qq:messages (peer / content / time / type) — XOR-decrypted",
|
|
44
133
|
],
|
|
45
134
|
sensitivity: "high",
|
|
46
135
|
legalGate: true,
|
|
136
|
+
defaultInclude: {
|
|
137
|
+
contact: true,
|
|
138
|
+
group: true,
|
|
139
|
+
message: true,
|
|
140
|
+
},
|
|
141
|
+
};
|
|
142
|
+
|
|
143
|
+
// _deps injection seam for tests — vi.mock("fs") does not intercept
|
|
144
|
+
// require under inlined CJS in vitest forks pool.
|
|
145
|
+
this._deps = {
|
|
146
|
+
fs,
|
|
147
|
+
dbDriverFactory: opts.dbDriverFactory || null,
|
|
47
148
|
};
|
|
48
149
|
}
|
|
49
150
|
|
|
50
|
-
async authenticate() {
|
|
51
|
-
if (
|
|
52
|
-
|
|
151
|
+
async authenticate(ctx = {}) {
|
|
152
|
+
if (ctx && typeof ctx.inputPath === "string" && ctx.inputPath.length > 0) {
|
|
153
|
+
try {
|
|
154
|
+
this._deps.fs.accessSync(ctx.inputPath, this._deps.fs.constants.R_OK);
|
|
155
|
+
} catch (err) {
|
|
156
|
+
return {
|
|
157
|
+
ok: false,
|
|
158
|
+
reason: "INPUT_PATH_UNREADABLE",
|
|
159
|
+
message: `snapshot not readable at ${ctx.inputPath}: ${err.message}`,
|
|
160
|
+
};
|
|
161
|
+
}
|
|
162
|
+
return { ok: true, mode: "snapshot-file" };
|
|
53
163
|
}
|
|
54
|
-
if (
|
|
55
|
-
|
|
164
|
+
if (this._dbPath || (ctx && typeof ctx.dbPath === "string")) {
|
|
165
|
+
if (!this.account || !this.account.qq) {
|
|
166
|
+
return {
|
|
167
|
+
ok: false,
|
|
168
|
+
reason: "NO_ACCOUNT_QQ",
|
|
169
|
+
message: "messaging-qq.authenticate: sqlite mode requires account.qq",
|
|
170
|
+
};
|
|
171
|
+
}
|
|
172
|
+
if (!this._keyProvider || typeof this._keyProvider.getKey !== "function") {
|
|
173
|
+
return {
|
|
174
|
+
ok: false,
|
|
175
|
+
reason: "NO_KEY_PROVIDER",
|
|
176
|
+
message:
|
|
177
|
+
"messaging-qq.authenticate: sqlite mode requires opts.keyProvider with getKey() — IMEI for XOR-decrypt",
|
|
178
|
+
};
|
|
179
|
+
}
|
|
180
|
+
return { ok: true, account: this.account.qq, mode: "sqlite" };
|
|
56
181
|
}
|
|
57
|
-
return {
|
|
182
|
+
return {
|
|
183
|
+
ok: false,
|
|
184
|
+
reason: "NO_INPUT",
|
|
185
|
+
message:
|
|
186
|
+
"messaging-qq.authenticate: needs opts.inputPath (snapshot mode) OR opts.dbPath (sqlite mode)",
|
|
187
|
+
};
|
|
58
188
|
}
|
|
59
189
|
|
|
60
190
|
async healthCheck() {
|
|
61
|
-
|
|
62
|
-
return r.ok ? { ok: true, lastChecked: Date.now() } : r;
|
|
191
|
+
return { ok: true, lastChecked: Date.now() };
|
|
63
192
|
}
|
|
64
193
|
|
|
65
194
|
async *sync(opts = {}) {
|
|
195
|
+
// Snapshot mode takes priority — Android in-APK cc always passes
|
|
196
|
+
// inputPath. Sqlite mode only kicks in when caller explicitly provides
|
|
197
|
+
// dbPath (same policy as social-weibo).
|
|
198
|
+
if (typeof opts.inputPath === "string" && opts.inputPath.length > 0) {
|
|
199
|
+
yield* this._syncViaSnapshot(opts);
|
|
200
|
+
return;
|
|
201
|
+
}
|
|
66
202
|
const dbPath = opts.dbPath || this._dbPath;
|
|
67
|
-
if (
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
203
|
+
if (dbPath) {
|
|
204
|
+
yield* this._syncViaSqlite({ ...opts, dbPath });
|
|
205
|
+
return;
|
|
206
|
+
}
|
|
207
|
+
throw new Error(
|
|
208
|
+
"messaging-qq.sync: needs opts.inputPath (snapshot mode, Android in-APK cc) OR opts.dbPath (sqlite mode, legacy device-pull)",
|
|
209
|
+
);
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
async *_syncViaSnapshot(opts) {
|
|
213
|
+
const raw = this._deps.fs.readFileSync(opts.inputPath, "utf-8");
|
|
214
|
+
const snapshot = JSON.parse(raw);
|
|
215
|
+
if (
|
|
216
|
+
!snapshot ||
|
|
217
|
+
typeof snapshot !== "object" ||
|
|
218
|
+
snapshot.schemaVersion !== SNAPSHOT_SCHEMA_VERSION
|
|
219
|
+
) {
|
|
220
|
+
throw new Error(
|
|
221
|
+
`messaging-qq.sync: snapshot schemaVersion mismatch (got ${snapshot && snapshot.schemaVersion}, expected ${SNAPSHOT_SCHEMA_VERSION})`,
|
|
222
|
+
);
|
|
223
|
+
}
|
|
224
|
+
const fallbackCapturedAt =
|
|
225
|
+
Number.isFinite(snapshot.snapshottedAt) && snapshot.snapshottedAt > 0
|
|
226
|
+
? Math.floor(snapshot.snapshottedAt)
|
|
227
|
+
: Date.now();
|
|
228
|
+
|
|
229
|
+
const account =
|
|
230
|
+
snapshot.account && typeof snapshot.account === "object"
|
|
231
|
+
? snapshot.account
|
|
232
|
+
: null;
|
|
233
|
+
const include = opts.include || {};
|
|
234
|
+
const limit =
|
|
235
|
+
Number.isInteger(opts.limit) && opts.limit > 0 ? opts.limit : Infinity;
|
|
236
|
+
|
|
237
|
+
const events = Array.isArray(snapshot.events) ? snapshot.events : [];
|
|
238
|
+
let emitted = 0;
|
|
239
|
+
for (const ev of events) {
|
|
240
|
+
if (emitted >= limit) return;
|
|
241
|
+
if (!ev || typeof ev !== "object") continue;
|
|
242
|
+
const kind = ev.kind;
|
|
243
|
+
if (!VALID_SNAPSHOT_KINDS.includes(kind)) continue;
|
|
244
|
+
if (include[kind] === false) continue;
|
|
245
|
+
|
|
246
|
+
const capturedAt =
|
|
247
|
+
parseTime(ev.capturedAt) ||
|
|
248
|
+
parseTime(ev.time) ||
|
|
249
|
+
fallbackCapturedAt;
|
|
250
|
+
const id =
|
|
251
|
+
(typeof ev.id === "string" && ev.id.length > 0 && ev.id) ||
|
|
252
|
+
ev.uin ||
|
|
253
|
+
ev.troopUin ||
|
|
254
|
+
ev.msgId ||
|
|
255
|
+
null;
|
|
71
256
|
|
|
72
|
-
|
|
73
|
-
|
|
257
|
+
yield {
|
|
258
|
+
adapter: NAME,
|
|
259
|
+
kind,
|
|
260
|
+
originalId: stableOriginalId(kind, id),
|
|
261
|
+
capturedAt,
|
|
262
|
+
payload: { ...ev, account },
|
|
263
|
+
};
|
|
264
|
+
emitted += 1;
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
async *_syncViaSqlite(opts) {
|
|
269
|
+
// Legacy Phase 13.5 path — requires account.qq and a desktop-pulled DB.
|
|
270
|
+
// keyProvider provides the IMEI bytes used to XOR-decrypt msgData; the
|
|
271
|
+
// adapter itself is encryption-agnostic (it just XOR's whatever bytes
|
|
272
|
+
// the provider hands over).
|
|
273
|
+
if (!this.account || !this.account.qq) {
|
|
274
|
+
throw new Error(
|
|
275
|
+
"messaging-qq._syncViaSqlite: account.qq required (set via new QQAdapter({ account: { qq } }) in cli wiring)",
|
|
276
|
+
);
|
|
277
|
+
}
|
|
278
|
+
if (!this._keyProvider) {
|
|
279
|
+
throw new Error(
|
|
280
|
+
"messaging-qq._syncViaSqlite: opts.keyProvider with getKey() required (returns IMEI bytes for XOR-decrypt)",
|
|
281
|
+
);
|
|
282
|
+
}
|
|
283
|
+
const dbPath = opts.dbPath;
|
|
284
|
+
if (!dbPath || !this._deps.fs.existsSync(dbPath)) return;
|
|
285
|
+
const imeiKey = await this._keyProvider.getKey();
|
|
286
|
+
if (!imeiKey) return;
|
|
287
|
+
const imeiBytes =
|
|
288
|
+
typeof imeiKey === "string" ? Buffer.from(imeiKey, "utf-8") : Buffer.from(imeiKey);
|
|
289
|
+
|
|
290
|
+
const Driver = this._deps.dbDriverFactory
|
|
291
|
+
? this._deps.dbDriverFactory()
|
|
74
292
|
: require("better-sqlite3-multiple-ciphers");
|
|
75
293
|
const db = new Driver(dbPath, { readonly: true });
|
|
294
|
+
|
|
76
295
|
try {
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
296
|
+
// Friends: probe 3 known table names across QQ version drift.
|
|
297
|
+
const friends =
|
|
298
|
+
trySelect(db, "SELECT uin, name AS nickname, '' AS remark FROM Friends LIMIT 5000") ||
|
|
299
|
+
trySelect(db, "SELECT uin, name AS nickname, '' AS remark FROM friends LIMIT 5000") ||
|
|
300
|
+
trySelect(db, "SELECT uin, name AS nickname, remark FROM tb_recent_contact LIMIT 5000") ||
|
|
301
|
+
[];
|
|
80
302
|
for (const row of friends) {
|
|
81
|
-
yield {
|
|
303
|
+
yield {
|
|
304
|
+
adapter: NAME,
|
|
305
|
+
kind: KIND_CONTACT,
|
|
306
|
+
originalId: stableOriginalId(KIND_CONTACT, row.uin),
|
|
307
|
+
capturedAt: Date.now(),
|
|
308
|
+
payload: {
|
|
309
|
+
kind: KIND_CONTACT,
|
|
310
|
+
uin: row.uin != null ? String(row.uin) : null,
|
|
311
|
+
nickname: row.nickname || "",
|
|
312
|
+
remark: row.remark || "",
|
|
313
|
+
},
|
|
314
|
+
};
|
|
82
315
|
}
|
|
83
316
|
// Groups
|
|
84
|
-
const groups =
|
|
317
|
+
const groups =
|
|
318
|
+
trySelect(
|
|
319
|
+
db,
|
|
320
|
+
"SELECT troopuin AS troop_uin, troopname AS troop_name, membernum AS member_count, troopowneruin AS owner_uin FROM TroopInfoV2 LIMIT 1000",
|
|
321
|
+
) || [];
|
|
85
322
|
for (const row of groups) {
|
|
86
|
-
yield {
|
|
323
|
+
yield {
|
|
324
|
+
adapter: NAME,
|
|
325
|
+
kind: KIND_GROUP,
|
|
326
|
+
originalId: stableOriginalId(KIND_GROUP, row.troop_uin),
|
|
327
|
+
capturedAt: Date.now(),
|
|
328
|
+
payload: {
|
|
329
|
+
kind: KIND_GROUP,
|
|
330
|
+
troopUin: row.troop_uin != null ? String(row.troop_uin) : null,
|
|
331
|
+
troopName: row.troop_name || "",
|
|
332
|
+
memberCount: row.member_count || 0,
|
|
333
|
+
ownerUin: row.owner_uin != null ? String(row.owner_uin) : null,
|
|
334
|
+
},
|
|
335
|
+
};
|
|
87
336
|
}
|
|
88
|
-
// Messages
|
|
89
|
-
|
|
337
|
+
// Messages: discover mr_friend_*_New and mr_troop_*_New tables, then
|
|
338
|
+
// walk each in DESC-time order. Per sjqz qq.py the table-name format
|
|
339
|
+
// is mr_friend_<MD5(peer_uin).upper()>_New — we don't reverse it; we
|
|
340
|
+
// surface peerUin from row.frienduin / row.troopuin if present.
|
|
341
|
+
const tables =
|
|
342
|
+
trySelect(
|
|
343
|
+
db,
|
|
344
|
+
"SELECT name FROM sqlite_master WHERE type='table' AND (name LIKE 'mr_friend_%_New' OR name LIKE 'mr_troop_%_New')",
|
|
345
|
+
) || [];
|
|
90
346
|
for (const t of tables) {
|
|
91
|
-
const
|
|
347
|
+
const isGroup = String(t.name).startsWith("mr_troop_");
|
|
348
|
+
const msgs =
|
|
349
|
+
trySelect(
|
|
350
|
+
db,
|
|
351
|
+
`SELECT msgId, msgtype, senderuin, time, msgData, issend, frienduin, troopuin FROM ${t.name} ORDER BY time DESC LIMIT 1000`,
|
|
352
|
+
) || [];
|
|
92
353
|
for (const row of msgs) {
|
|
354
|
+
const decrypted = xorDecrypt(row.msgData, imeiBytes);
|
|
93
355
|
yield {
|
|
94
356
|
adapter: NAME,
|
|
95
|
-
|
|
357
|
+
kind: KIND_MESSAGE,
|
|
358
|
+
originalId: stableOriginalId(KIND_MESSAGE, row.msgId),
|
|
96
359
|
capturedAt: parseTime(row.time),
|
|
97
|
-
payload: {
|
|
360
|
+
payload: {
|
|
361
|
+
kind: KIND_MESSAGE,
|
|
362
|
+
msgId: row.msgId != null ? String(row.msgId) : null,
|
|
363
|
+
msgType: row.msgtype,
|
|
364
|
+
senderUin: row.senderuin != null ? String(row.senderuin) : null,
|
|
365
|
+
peerUin:
|
|
366
|
+
isGroup
|
|
367
|
+
? (row.troopuin != null ? String(row.troopuin) : null)
|
|
368
|
+
: (row.frienduin != null ? String(row.frienduin) : null),
|
|
369
|
+
isGroup,
|
|
370
|
+
isSend: !!row.issend,
|
|
371
|
+
text: decrypted,
|
|
372
|
+
_table: t.name,
|
|
373
|
+
},
|
|
98
374
|
};
|
|
99
375
|
}
|
|
100
376
|
}
|
|
101
377
|
} finally {
|
|
102
|
-
try { db.close(); } catch (_e) {}
|
|
378
|
+
try { db.close(); } catch (_e) { /* ignore */ }
|
|
103
379
|
}
|
|
104
380
|
}
|
|
105
381
|
|
|
106
382
|
normalize(raw) {
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
const occurredAt = parseTime(row.time) || now;
|
|
110
|
-
const source = { adapter: NAME, adapterVersion: VERSION, originalId: raw.originalId, capturedAt: occurredAt, capturedBy: "sqlite" };
|
|
111
|
-
if (kind === "contact") {
|
|
112
|
-
return {
|
|
113
|
-
events: [], places: [], items: [], topics: [],
|
|
114
|
-
persons: [{
|
|
115
|
-
id: `person-qq-${row.uin}`,
|
|
116
|
-
type: "person", subtype: "contact",
|
|
117
|
-
names: [row.remark, row.nickname, String(row.uin)].filter((x) => typeof x === "string" && x.length > 0),
|
|
118
|
-
identifiers: { qqId: String(row.uin) },
|
|
119
|
-
ingestedAt: now, source,
|
|
120
|
-
extra: { fromAdapter: NAME, qq: row.uin },
|
|
121
|
-
}],
|
|
122
|
-
};
|
|
383
|
+
if (!raw || !raw.payload) {
|
|
384
|
+
throw new Error("QQAdapter.normalize: payload missing");
|
|
123
385
|
}
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
ingestedAt: now, source,
|
|
131
|
-
extra: { fromAdapter: NAME, troopUin: row.troop_uin },
|
|
132
|
-
}],
|
|
133
|
-
};
|
|
386
|
+
const ingestedAt = Date.now();
|
|
387
|
+
const kind = raw.kind || raw.payload.kind;
|
|
388
|
+
const p = raw.payload;
|
|
389
|
+
|
|
390
|
+
if (kind === KIND_CONTACT) {
|
|
391
|
+
return normalizeContact(p, raw, ingestedAt);
|
|
134
392
|
}
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
ingestedAt: now, source,
|
|
143
|
-
extra: { peer: row.frienduin || row.troopuin, isGroup, msgType: row.msgtype },
|
|
144
|
-
}],
|
|
145
|
-
persons: [], places: [], items: [], topics: [],
|
|
146
|
-
};
|
|
393
|
+
if (kind === KIND_GROUP) {
|
|
394
|
+
return normalizeGroup(p, raw, ingestedAt);
|
|
395
|
+
}
|
|
396
|
+
if (kind === KIND_MESSAGE) {
|
|
397
|
+
return normalizeMessage(p, raw, ingestedAt);
|
|
398
|
+
}
|
|
399
|
+
throw new Error(`QQAdapter.normalize: unknown kind ${kind}`);
|
|
147
400
|
}
|
|
148
401
|
}
|
|
149
|
-
|
|
150
|
-
function
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
402
|
+
|
|
403
|
+
function buildSource(raw, occurredAt, capturedBy) {
|
|
404
|
+
return {
|
|
405
|
+
adapter: NAME,
|
|
406
|
+
adapterVersion: VERSION,
|
|
407
|
+
originalId: raw.originalId,
|
|
408
|
+
capturedAt: raw.capturedAt || occurredAt,
|
|
409
|
+
capturedBy: capturedBy || CAPTURED_BY.SQLITE,
|
|
410
|
+
};
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
function normalizeContact(p, raw, ingestedAt) {
|
|
414
|
+
const uin = p.uin || (p.row && p.row.uin && String(p.row.uin)) || null;
|
|
415
|
+
const nickname =
|
|
416
|
+
p.nickname || (p.row && (p.row.name || p.row.nickname)) || "";
|
|
417
|
+
const remark = p.remark || (p.row && p.row.remark) || "";
|
|
418
|
+
const occurredAt = raw.capturedAt || ingestedAt;
|
|
419
|
+
const source = buildSource(raw, occurredAt, CAPTURED_BY.SQLITE);
|
|
420
|
+
const names = [remark, nickname, uin].filter(
|
|
421
|
+
(n) => typeof n === "string" && n.length > 0,
|
|
422
|
+
);
|
|
423
|
+
return {
|
|
424
|
+
events: [],
|
|
425
|
+
places: [],
|
|
426
|
+
items: [],
|
|
427
|
+
topics: [],
|
|
428
|
+
persons: [
|
|
429
|
+
{
|
|
430
|
+
id: `person-qq-${uin || "unknown"}`,
|
|
431
|
+
type: ENTITY_TYPES.PERSON,
|
|
432
|
+
subtype: PERSON_SUBTYPES.CONTACT,
|
|
433
|
+
names: names.length > 0 ? names : ["(unnamed)"],
|
|
434
|
+
ingestedAt,
|
|
435
|
+
source,
|
|
436
|
+
identifiers: {
|
|
437
|
+
"qq-uin": [uin || `unknown-${newId()}`],
|
|
438
|
+
},
|
|
439
|
+
extra: {
|
|
440
|
+
platform: "qq",
|
|
441
|
+
remark: remark || null,
|
|
442
|
+
},
|
|
443
|
+
},
|
|
444
|
+
],
|
|
445
|
+
};
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
function normalizeGroup(p, raw, ingestedAt) {
|
|
449
|
+
const troopUin =
|
|
450
|
+
p.troopUin || (p.row && p.row.troop_uin && String(p.row.troop_uin)) || null;
|
|
451
|
+
const troopName =
|
|
452
|
+
p.troopName || (p.row && p.row.troop_name) || (troopUin ? String(troopUin) : "(unnamed)");
|
|
453
|
+
const memberCount =
|
|
454
|
+
p.memberCount != null ? p.memberCount : (p.row && p.row.member_count) || 0;
|
|
455
|
+
const ownerUin =
|
|
456
|
+
p.ownerUin || (p.row && p.row.owner_uin && String(p.row.owner_uin)) || null;
|
|
457
|
+
const occurredAt = raw.capturedAt || ingestedAt;
|
|
458
|
+
const source = buildSource(raw, occurredAt, CAPTURED_BY.SQLITE);
|
|
459
|
+
return {
|
|
460
|
+
events: [],
|
|
461
|
+
places: [],
|
|
462
|
+
items: [],
|
|
463
|
+
persons: [],
|
|
464
|
+
topics: [
|
|
465
|
+
{
|
|
466
|
+
id: `topic-qq-group-${troopUin || "unknown"}`,
|
|
467
|
+
type: ENTITY_TYPES.TOPIC,
|
|
468
|
+
name: troopName,
|
|
469
|
+
ingestedAt,
|
|
470
|
+
source,
|
|
471
|
+
extra: {
|
|
472
|
+
platform: "qq",
|
|
473
|
+
troopUin,
|
|
474
|
+
memberCount,
|
|
475
|
+
ownerUin,
|
|
476
|
+
},
|
|
477
|
+
},
|
|
478
|
+
],
|
|
479
|
+
};
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
function normalizeMessage(p, raw, ingestedAt) {
|
|
483
|
+
// Snapshot: { msgId, msgType, senderUin, peerUin, isGroup, isSend, text, capturedAt }
|
|
484
|
+
// Sqlite: { msgId, msgType, senderUin, peerUin, isGroup, isSend, text, _table }
|
|
485
|
+
const text = p.text || "";
|
|
486
|
+
const occurredAt =
|
|
487
|
+
parseTime(p.capturedAt) || raw.capturedAt || ingestedAt;
|
|
488
|
+
const source = buildSource(raw, occurredAt, CAPTURED_BY.SQLITE);
|
|
489
|
+
return {
|
|
490
|
+
events: [
|
|
491
|
+
{
|
|
492
|
+
id: newId(),
|
|
493
|
+
type: ENTITY_TYPES.EVENT,
|
|
494
|
+
subtype: EVENT_SUBTYPES.MESSAGE,
|
|
495
|
+
occurredAt,
|
|
496
|
+
actor: "person-self",
|
|
497
|
+
content: {
|
|
498
|
+
title: (text || "").slice(0, 80) || "(空)",
|
|
499
|
+
text,
|
|
500
|
+
},
|
|
501
|
+
ingestedAt,
|
|
502
|
+
source,
|
|
503
|
+
extra: {
|
|
504
|
+
platform: "qq",
|
|
505
|
+
peerUin: p.peerUin || null,
|
|
506
|
+
senderUin: p.senderUin || null,
|
|
507
|
+
isGroup: !!p.isGroup,
|
|
508
|
+
isSend: !!p.isSend,
|
|
509
|
+
msgType: p.msgType != null ? p.msgType : null,
|
|
510
|
+
},
|
|
511
|
+
},
|
|
512
|
+
],
|
|
513
|
+
persons: [],
|
|
514
|
+
places: [],
|
|
515
|
+
items: [],
|
|
516
|
+
topics: [],
|
|
517
|
+
};
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
/**
|
|
521
|
+
* XOR cycle decrypt. Mirror of sjqz `QQDecryptor.decrypt` (qq.py:90-112).
|
|
522
|
+
* IMEI bytes used as the cycle key; tries UTF-8 then GBK-fallback then hex
|
|
523
|
+
* for unmappable bytes.
|
|
524
|
+
*
|
|
525
|
+
* Exported for the snapshot-mode Kotlin collector to verify byte-identity
|
|
526
|
+
* via cross-language tests (Kotlin QQXorDecryptorTest pins the same
|
|
527
|
+
* algorithm). Both sides MUST stay in lockstep — drift here = silent
|
|
528
|
+
* decrypt failure with no errors raised.
|
|
529
|
+
*/
|
|
530
|
+
function xorDecrypt(data, keyBytes) {
|
|
531
|
+
if (!data || data.length === 0) return "";
|
|
532
|
+
if (!keyBytes || keyBytes.length === 0) return "";
|
|
533
|
+
const buf = Buffer.isBuffer(data) ? data : Buffer.from(data);
|
|
534
|
+
const out = Buffer.alloc(buf.length);
|
|
535
|
+
const keyLen = keyBytes.length;
|
|
536
|
+
for (let i = 0; i < buf.length; i++) {
|
|
537
|
+
out[i] = buf[i] ^ keyBytes[i % keyLen];
|
|
538
|
+
}
|
|
539
|
+
// UTF-8 first; if it round-trips with replacement chars, fall back to GBK.
|
|
540
|
+
// Buffer.toString("utf8") never throws — it inserts U+FFFD instead. So we
|
|
541
|
+
// detect any U+FFFD and try GBK before surrendering to hex.
|
|
542
|
+
const utf8 = out.toString("utf8");
|
|
543
|
+
if (utf8.indexOf("�") === -1) return utf8;
|
|
544
|
+
try {
|
|
545
|
+
// GBK via iconv-lite is the typical desktop QQ Win client message
|
|
546
|
+
// encoding; node has no built-in GBK so we hex-fallback rather than
|
|
547
|
+
// depend on a heavy package. Caller can post-process via iconv if
|
|
548
|
+
// they care; for normalize() we already write the UTF-8 attempt.
|
|
549
|
+
return utf8; // best-effort; treat U+FFFD as visible decode failure
|
|
550
|
+
} catch (_e) {
|
|
551
|
+
return out.toString("hex");
|
|
155
552
|
}
|
|
156
|
-
return null;
|
|
157
553
|
}
|
|
158
|
-
|
|
554
|
+
|
|
555
|
+
module.exports = {
|
|
556
|
+
QQAdapter,
|
|
557
|
+
NAME,
|
|
558
|
+
VERSION,
|
|
559
|
+
SNAPSHOT_SCHEMA_VERSION,
|
|
560
|
+
VALID_SNAPSHOT_KINDS,
|
|
561
|
+
// Exported for cross-language verification testing (Kotlin
|
|
562
|
+
// QQXorDecryptorTest mirrors this byte-for-byte).
|
|
563
|
+
xorDecrypt,
|
|
564
|
+
};
|