@chainlesschain/personal-data-hub 0.2.2 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/__tests__/adapters/social-toutiao-kuaishou-scaffold.test.js +58 -16
- package/__tests__/longtail-adapters.test.js +60 -14
- package/__tests__/messaging-qq-snapshot.test.js +294 -0
- package/__tests__/shopping-pinduoduo-snapshot.test.js +302 -0
- package/__tests__/shopping-snapshot.test.js +438 -0
- package/__tests__/social-adapters.test.js +28 -3
- package/__tests__/social-douyin-snapshot.test.js +253 -0
- package/__tests__/social-kuaishou-snapshot.test.js +309 -0
- package/__tests__/social-toutiao-snapshot.test.js +314 -0
- package/__tests__/social-weibo-snapshot.test.js +234 -0
- package/__tests__/social-xiaohongshu-snapshot.test.js +232 -0
- package/__tests__/travel-maps-snapshot.test.js +426 -0
- package/__tests__/vault-driver-error.test.js +74 -0
- package/lib/adapters/messaging-qq/index.js +498 -92
- package/lib/adapters/shopping-jd/index.js +228 -25
- package/lib/adapters/shopping-meituan/index.js +222 -26
- package/lib/adapters/shopping-pinduoduo/index.js +275 -0
- package/lib/adapters/social-douyin/index.js +454 -63
- package/lib/adapters/social-kuaishou/index.js +379 -127
- package/lib/adapters/social-toutiao/index.js +400 -130
- package/lib/adapters/social-weibo/index.js +393 -95
- package/lib/adapters/social-xiaohongshu/index.js +389 -49
- package/lib/adapters/travel-baidu-map/index.js +286 -26
- package/lib/adapters/travel-tencent-map/index.js +414 -0
- package/lib/index.js +5 -1
- package/lib/vault.js +60 -8
- package/package.json +2 -1
|
@@ -1,12 +1,40 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
2
|
+
* §2.5b 地图三联 v0.2 — Baidu Map (百度地图) adapter, dual-mode.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
4
|
+
* Mirror of social-weibo / social-bilibili two-mode pattern:
|
|
5
|
+
*
|
|
6
|
+
* 1. snapshot mode (opts.inputPath): in-APK Android cc reads a snapshot
|
|
7
|
+
* JSON produced by BaiduMapLocalCollector (WebView cookie scrape).
|
|
8
|
+
* Desktop-independent path. Adapter is stateless in this mode —
|
|
9
|
+
* account.deviceId is OPTIONAL at construction; account meta carried
|
|
10
|
+
* in payload.
|
|
11
|
+
*
|
|
12
|
+
* 2. sqlite mode (opts.dbPath, legacy Phase 9.4b): device-pull path —
|
|
13
|
+
* reads Baidu Map Android app's SQLite (search_history / route_history /
|
|
14
|
+
* my_favourite). Preserved for backward compat. account.deviceId
|
|
15
|
+
* REQUIRED in this mode (checked at sync, not construction).
|
|
16
|
+
*
|
|
17
|
+
* Snapshot schema (mirrors BaiduMapLocalCollector.SNAPSHOT_SCHEMA_VERSION):
|
|
18
|
+
*
|
|
19
|
+
* {
|
|
20
|
+
* "schemaVersion": 1,
|
|
21
|
+
* "snapshottedAt": <epoch-ms>,
|
|
22
|
+
* "vendor": "baidu-map",
|
|
23
|
+
* "account": { "uid": "...", "displayName": "..." },
|
|
24
|
+
* "events": [
|
|
25
|
+
* { "kind": "favourite", "id": "fav-<rid>", "capturedAt": <ms>,
|
|
26
|
+
* "name": "...", "address": "...", "lat": .., "lng": .., "category": "home|company|other" },
|
|
27
|
+
* { "kind": "search", "id": "search-<sid>","capturedAt": <ms>,
|
|
28
|
+
* "query": "...", "city": "..." },
|
|
29
|
+
* { "kind": "route", "id": "route-<rid>", "capturedAt": <ms>,
|
|
30
|
+
* "from": {...}, "to": {...}, "mode": "drive|walk|bus|bike|trip" }
|
|
31
|
+
* ]
|
|
32
|
+
* }
|
|
33
|
+
*
|
|
34
|
+
* Per sjqz/parsers/baidumap.py the key SQLite tables (sqlite mode) are:
|
|
6
35
|
* - search_history (queries)
|
|
7
36
|
* - route_history (planned routes)
|
|
8
37
|
* - my_favourite (saved places)
|
|
9
|
-
* - offline_map (downloaded offline maps; v2)
|
|
10
38
|
*/
|
|
11
39
|
|
|
12
40
|
"use strict";
|
|
@@ -15,35 +43,87 @@ const fs = require("node:fs");
|
|
|
15
43
|
const { normalizeTravelRecord, parseChineseDateTime } = require("../travel-base");
|
|
16
44
|
|
|
17
45
|
const NAME = "travel-baidu-map";
|
|
18
|
-
const VERSION = "0.
|
|
46
|
+
const VERSION = "0.6.0";
|
|
47
|
+
const SNAPSHOT_SCHEMA_VERSION = 1;
|
|
48
|
+
|
|
49
|
+
const KIND_FAVOURITE = "favourite";
|
|
50
|
+
const KIND_SEARCH = "search";
|
|
51
|
+
const KIND_ROUTE = "route";
|
|
52
|
+
const VALID_SNAPSHOT_KINDS = Object.freeze([KIND_FAVOURITE, KIND_SEARCH, KIND_ROUTE]);
|
|
19
53
|
|
|
20
54
|
class BaiduMapAdapter {
|
|
21
55
|
constructor(opts = {}) {
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
this.account = opts.account;
|
|
56
|
+
// §2.5b v0.2: account.deviceId now OPTIONAL — snapshot mode is stateless
|
|
57
|
+
// and pulls account from the JSON file. Sqlite mode still requires it;
|
|
58
|
+
// checked at sync time, not construction.
|
|
59
|
+
this.account = opts.account || null;
|
|
26
60
|
this._dbPath = opts.dbPath || null;
|
|
27
|
-
this._dbDriverFactory = opts.dbDriverFactory || null;
|
|
28
61
|
|
|
29
62
|
this.name = NAME;
|
|
30
63
|
this.version = VERSION;
|
|
31
|
-
this.capabilities = [
|
|
64
|
+
this.capabilities = [
|
|
65
|
+
"sync:snapshot",
|
|
66
|
+
"sync:sqlite",
|
|
67
|
+
"parse:baidu-map-favourite",
|
|
68
|
+
"parse:baidu-map-history",
|
|
69
|
+
];
|
|
70
|
+
// Existing desktop wiring may key off this — sqlite mode is the desktop-
|
|
71
|
+
// side, snapshot mode is in-APK Android. Reported value stays compatible.
|
|
32
72
|
this.extractMode = "device-pull";
|
|
33
73
|
this.rateLimits = {};
|
|
34
74
|
this.dataDisclosure = {
|
|
35
75
|
fields: [
|
|
36
|
-
"baidu:
|
|
37
|
-
"baidu:
|
|
38
|
-
"baidu:
|
|
76
|
+
"baidu:account (uid / displayName, cookie scrape)",
|
|
77
|
+
"baidu:my_favourite (saved places — home / company / other)",
|
|
78
|
+
"baidu:search_history (queries, legacy sqlite mode)",
|
|
79
|
+
"baidu:route_history (planned routes, legacy sqlite mode)",
|
|
39
80
|
],
|
|
40
81
|
sensitivity: "medium",
|
|
41
82
|
legalGate: false,
|
|
83
|
+
defaultInclude: {
|
|
84
|
+
favourite: true,
|
|
85
|
+
search: true,
|
|
86
|
+
route: true,
|
|
87
|
+
},
|
|
88
|
+
};
|
|
89
|
+
|
|
90
|
+
// _deps injection seam — vi.mock fs doesn't intercept inlined CJS require
|
|
91
|
+
// (see .claude/rules/testing.md).
|
|
92
|
+
this._deps = {
|
|
93
|
+
fs,
|
|
94
|
+
dbDriverFactory: opts.dbDriverFactory || null,
|
|
42
95
|
};
|
|
43
96
|
}
|
|
44
97
|
|
|
45
|
-
async authenticate() {
|
|
46
|
-
|
|
98
|
+
async authenticate(ctx = {}) {
|
|
99
|
+
if (ctx && typeof ctx.inputPath === "string" && ctx.inputPath.length > 0) {
|
|
100
|
+
try {
|
|
101
|
+
this._deps.fs.accessSync(ctx.inputPath, this._deps.fs.constants.R_OK);
|
|
102
|
+
} catch (err) {
|
|
103
|
+
return {
|
|
104
|
+
ok: false,
|
|
105
|
+
reason: "INPUT_PATH_UNREADABLE",
|
|
106
|
+
message: `snapshot not readable at ${ctx.inputPath}: ${err.message}`,
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
return { ok: true, mode: "snapshot-file" };
|
|
110
|
+
}
|
|
111
|
+
if (this._dbPath || (ctx && typeof ctx.dbPath === "string")) {
|
|
112
|
+
if (!this.account || !this.account.deviceId) {
|
|
113
|
+
return {
|
|
114
|
+
ok: false,
|
|
115
|
+
reason: "NO_ACCOUNT_DEVICE_ID",
|
|
116
|
+
message: "travel-baidu-map.authenticate: sqlite mode requires account.deviceId",
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
return { ok: true, account: this.account.deviceId, mode: "sqlite" };
|
|
120
|
+
}
|
|
121
|
+
return {
|
|
122
|
+
ok: false,
|
|
123
|
+
reason: "NO_INPUT",
|
|
124
|
+
message:
|
|
125
|
+
"travel-baidu-map.authenticate: needs opts.inputPath (snapshot mode) OR opts.dbPath (sqlite mode)",
|
|
126
|
+
};
|
|
47
127
|
}
|
|
48
128
|
|
|
49
129
|
async healthCheck() {
|
|
@@ -51,10 +131,85 @@ class BaiduMapAdapter {
|
|
|
51
131
|
}
|
|
52
132
|
|
|
53
133
|
async *sync(opts = {}) {
|
|
134
|
+
if (typeof opts.inputPath === "string" && opts.inputPath.length > 0) {
|
|
135
|
+
yield* this._syncViaSnapshot(opts);
|
|
136
|
+
return;
|
|
137
|
+
}
|
|
54
138
|
const dbPath = opts.dbPath || this._dbPath;
|
|
55
|
-
if (
|
|
56
|
-
|
|
57
|
-
|
|
139
|
+
if (dbPath) {
|
|
140
|
+
yield* this._syncViaSqlite({ ...opts, dbPath });
|
|
141
|
+
return;
|
|
142
|
+
}
|
|
143
|
+
throw new Error(
|
|
144
|
+
"travel-baidu-map.sync: needs opts.inputPath (snapshot mode, Android in-APK cc) OR opts.dbPath (sqlite mode, legacy device-pull)",
|
|
145
|
+
);
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
async *_syncViaSnapshot(opts) {
|
|
149
|
+
const raw = this._deps.fs.readFileSync(opts.inputPath, "utf-8");
|
|
150
|
+
const snapshot = JSON.parse(raw);
|
|
151
|
+
if (
|
|
152
|
+
!snapshot ||
|
|
153
|
+
typeof snapshot !== "object" ||
|
|
154
|
+
snapshot.schemaVersion !== SNAPSHOT_SCHEMA_VERSION
|
|
155
|
+
) {
|
|
156
|
+
throw new Error(
|
|
157
|
+
`travel-baidu-map.sync: snapshot schemaVersion mismatch (got ${snapshot && snapshot.schemaVersion}, expected ${SNAPSHOT_SCHEMA_VERSION})`,
|
|
158
|
+
);
|
|
159
|
+
}
|
|
160
|
+
const fallbackCapturedAt =
|
|
161
|
+
Number.isFinite(snapshot.snapshottedAt) && snapshot.snapshottedAt > 0
|
|
162
|
+
? Math.floor(snapshot.snapshottedAt)
|
|
163
|
+
: Date.now();
|
|
164
|
+
const account =
|
|
165
|
+
snapshot.account && typeof snapshot.account === "object"
|
|
166
|
+
? snapshot.account
|
|
167
|
+
: null;
|
|
168
|
+
const include = opts.include || {};
|
|
169
|
+
const limit =
|
|
170
|
+
Number.isInteger(opts.limit) && opts.limit > 0 ? opts.limit : Infinity;
|
|
171
|
+
|
|
172
|
+
const events = Array.isArray(snapshot.events) ? snapshot.events : [];
|
|
173
|
+
let emitted = 0;
|
|
174
|
+
for (const ev of events) {
|
|
175
|
+
if (emitted >= limit) return;
|
|
176
|
+
if (!ev || typeof ev !== "object") continue;
|
|
177
|
+
const kind = ev.kind;
|
|
178
|
+
if (!VALID_SNAPSHOT_KINDS.includes(kind)) continue;
|
|
179
|
+
if (include[kind] === false) continue;
|
|
180
|
+
|
|
181
|
+
const capturedAt =
|
|
182
|
+
parseTime(ev.capturedAt) ||
|
|
183
|
+
parseTime(ev.time) ||
|
|
184
|
+
fallbackCapturedAt;
|
|
185
|
+
const id =
|
|
186
|
+
(typeof ev.id === "string" && ev.id.length > 0 && ev.id) ||
|
|
187
|
+
ev.rid ||
|
|
188
|
+
null;
|
|
189
|
+
|
|
190
|
+
yield {
|
|
191
|
+
adapter: NAME,
|
|
192
|
+
kind,
|
|
193
|
+
originalId: stableOriginalId(kind, id),
|
|
194
|
+
capturedAt,
|
|
195
|
+
payload: { ...ev, account },
|
|
196
|
+
};
|
|
197
|
+
emitted += 1;
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
async *_syncViaSqlite(opts) {
|
|
202
|
+
// Legacy Phase 9.4b path — requires account.deviceId in constructor.
|
|
203
|
+
if (!this.account || !this.account.deviceId) {
|
|
204
|
+
throw new Error(
|
|
205
|
+
"travel-baidu-map._syncViaSqlite: account.deviceId required (set via new BaiduMapAdapter({ account: { deviceId } }))",
|
|
206
|
+
);
|
|
207
|
+
}
|
|
208
|
+
const dbPath = opts.dbPath;
|
|
209
|
+
if (!dbPath || !this._deps.fs.existsSync(dbPath)) return;
|
|
210
|
+
const Driver = this._deps.dbDriverFactory
|
|
211
|
+
? this._deps.dbDriverFactory()
|
|
212
|
+
: require("better-sqlite3-multiple-ciphers");
|
|
58
213
|
const db = new Driver(dbPath, { readonly: true });
|
|
59
214
|
|
|
60
215
|
try {
|
|
@@ -67,7 +222,7 @@ class BaiduMapAdapter {
|
|
|
67
222
|
adapter: NAME,
|
|
68
223
|
originalId: rec.recordId,
|
|
69
224
|
capturedAt: rec.bookedAt || Date.now(),
|
|
70
|
-
payload: { record: rec, kind:
|
|
225
|
+
payload: { record: rec, kind: KIND_ROUTE },
|
|
71
226
|
};
|
|
72
227
|
}
|
|
73
228
|
}
|
|
@@ -79,26 +234,110 @@ class BaiduMapAdapter {
|
|
|
79
234
|
adapter: NAME,
|
|
80
235
|
originalId: rec.recordId,
|
|
81
236
|
capturedAt: rec.bookedAt || Date.now(),
|
|
82
|
-
payload: { record: rec, kind:
|
|
237
|
+
payload: { record: rec, kind: KIND_SEARCH },
|
|
83
238
|
};
|
|
84
239
|
}
|
|
85
240
|
}
|
|
86
241
|
} finally {
|
|
87
|
-
try { db.close(); } catch (_e) {}
|
|
242
|
+
try { db.close(); } catch (_e) { /* ignore */ }
|
|
88
243
|
}
|
|
89
244
|
}
|
|
90
245
|
|
|
91
246
|
normalize(raw) {
|
|
92
|
-
if (!raw || !raw.payload
|
|
93
|
-
throw new Error("BaiduMapAdapter.normalize:
|
|
247
|
+
if (!raw || !raw.payload) {
|
|
248
|
+
throw new Error("BaiduMapAdapter.normalize: payload missing");
|
|
249
|
+
}
|
|
250
|
+
const kind = raw.kind || raw.payload.kind;
|
|
251
|
+
const p = raw.payload;
|
|
252
|
+
|
|
253
|
+
// Sqlite-mode payload carries `record`; snapshot-mode payload carries fields
|
|
254
|
+
// directly (favourite / search / route).
|
|
255
|
+
if (p.record) {
|
|
256
|
+
return normalizeTravelRecord(p.record, {
|
|
257
|
+
adapterName: NAME,
|
|
258
|
+
adapterVersion: VERSION,
|
|
259
|
+
});
|
|
94
260
|
}
|
|
95
|
-
|
|
261
|
+
// Snapshot-mode normalize: build a TravelRecord on-the-fly so we share
|
|
262
|
+
// the travel-base normalizer (1 event + place(s) + carrier merchant).
|
|
263
|
+
const rec = snapshotEventToRecord(kind, p, raw.originalId);
|
|
264
|
+
return normalizeTravelRecord(rec, {
|
|
96
265
|
adapterName: NAME,
|
|
97
266
|
adapterVersion: VERSION,
|
|
98
267
|
});
|
|
99
268
|
}
|
|
100
269
|
}
|
|
101
270
|
|
|
271
|
+
function stableOriginalId(kind, id) {
|
|
272
|
+
const stringified =
|
|
273
|
+
(typeof id === "string" && id.length > 0 && id) ||
|
|
274
|
+
(typeof id === "number" && Number.isFinite(id) && String(id)) ||
|
|
275
|
+
null;
|
|
276
|
+
const safe =
|
|
277
|
+
stringified ||
|
|
278
|
+
`unknown-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
|
279
|
+
return `baidu-map:${kind}:${safe}`;
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
function snapshotEventToRecord(kind, p, originalId) {
|
|
283
|
+
if (kind === KIND_FAVOURITE) {
|
|
284
|
+
return {
|
|
285
|
+
vendorId: "baidumap",
|
|
286
|
+
recordId: originalId,
|
|
287
|
+
vehicleType: "visit",
|
|
288
|
+
to: {
|
|
289
|
+
name: p.name || p.address || null,
|
|
290
|
+
lat: numberOrNull(p.lat),
|
|
291
|
+
lng: numberOrNull(p.lng),
|
|
292
|
+
city: p.city || null,
|
|
293
|
+
},
|
|
294
|
+
departureMs: parseTime(p.capturedAt),
|
|
295
|
+
carrier: "百度地图",
|
|
296
|
+
extras: { category: p.category || null, kind: KIND_FAVOURITE },
|
|
297
|
+
};
|
|
298
|
+
}
|
|
299
|
+
if (kind === KIND_SEARCH) {
|
|
300
|
+
return {
|
|
301
|
+
vendorId: "baidumap",
|
|
302
|
+
recordId: originalId,
|
|
303
|
+
vehicleType: "visit",
|
|
304
|
+
to: {
|
|
305
|
+
name: p.query || null,
|
|
306
|
+
lat: numberOrNull(p.lat),
|
|
307
|
+
lng: numberOrNull(p.lng),
|
|
308
|
+
city: p.city || null,
|
|
309
|
+
},
|
|
310
|
+
departureMs: parseTime(p.capturedAt),
|
|
311
|
+
carrier: "百度地图",
|
|
312
|
+
extras: { query: p.query || null, kind: KIND_SEARCH },
|
|
313
|
+
};
|
|
314
|
+
}
|
|
315
|
+
if (kind === KIND_ROUTE) {
|
|
316
|
+
return {
|
|
317
|
+
vendorId: "baidumap",
|
|
318
|
+
recordId: originalId,
|
|
319
|
+
vehicleType: detectVehicle(p.mode),
|
|
320
|
+
from: p.from
|
|
321
|
+
? { name: p.from.name || null, lat: numberOrNull(p.from.lat), lng: numberOrNull(p.from.lng) }
|
|
322
|
+
: undefined,
|
|
323
|
+
to: p.to
|
|
324
|
+
? { name: p.to.name || null, lat: numberOrNull(p.to.lat), lng: numberOrNull(p.to.lng) }
|
|
325
|
+
: undefined,
|
|
326
|
+
departureMs: parseTime(p.capturedAt),
|
|
327
|
+
carrier: "百度地图",
|
|
328
|
+
extras: { mode: p.mode || null, kind: KIND_ROUTE },
|
|
329
|
+
};
|
|
330
|
+
}
|
|
331
|
+
// Fallback (shouldn't reach — VALID_SNAPSHOT_KINDS filters earlier)
|
|
332
|
+
return {
|
|
333
|
+
vendorId: "baidumap",
|
|
334
|
+
recordId: originalId,
|
|
335
|
+
vehicleType: "visit",
|
|
336
|
+
carrier: "百度地图",
|
|
337
|
+
extras: { kind, raw: p },
|
|
338
|
+
};
|
|
339
|
+
}
|
|
340
|
+
|
|
102
341
|
function trySelect(db, sql) {
|
|
103
342
|
try {
|
|
104
343
|
return db.prepare(sql).all();
|
|
@@ -147,6 +386,27 @@ function detectVehicle(v) {
|
|
|
147
386
|
return "trip";
|
|
148
387
|
}
|
|
149
388
|
|
|
389
|
+
function numberOrNull(v) {
|
|
390
|
+
if (Number.isFinite(v)) return v;
|
|
391
|
+
if (typeof v === "string" && /^-?\d+(\.\d+)?$/.test(v)) return parseFloat(v);
|
|
392
|
+
return null;
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
function parseTime(v) {
|
|
396
|
+
if (Number.isFinite(v)) return v > 1e12 ? v : v * 1000;
|
|
397
|
+
if (typeof v === "string") {
|
|
398
|
+
if (/^\d+$/.test(v)) {
|
|
399
|
+
const n = parseInt(v, 10);
|
|
400
|
+
return n > 1e12 ? n : n * 1000;
|
|
401
|
+
}
|
|
402
|
+
const parsed = parseChineseDateTime(v);
|
|
403
|
+
if (Number.isFinite(parsed)) return parsed;
|
|
404
|
+
const t = Date.parse(v);
|
|
405
|
+
return Number.isFinite(t) ? t : null;
|
|
406
|
+
}
|
|
407
|
+
return null;
|
|
408
|
+
}
|
|
409
|
+
|
|
150
410
|
function numberOrParse(v) {
|
|
151
411
|
if (Number.isFinite(v)) return v > 1e12 ? v : v * 1000;
|
|
152
412
|
if (typeof v === "string") {
|
|
@@ -159,4 +419,4 @@ function numberOrParse(v) {
|
|
|
159
419
|
return null;
|
|
160
420
|
}
|
|
161
421
|
|
|
162
|
-
module.exports = { BaiduMapAdapter, NAME, VERSION };
|
|
422
|
+
module.exports = { BaiduMapAdapter, NAME, VERSION, SNAPSHOT_SCHEMA_VERSION };
|