@chainlesschain/personal-data-hub 0.3.1 → 0.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/__tests__/adapters/email-adapter-snapshot.test.js +237 -0
- package/__tests__/adapters/email-adapter.test.js +1 -1
- package/__tests__/adapters/email-pdf-extractor.test.js +1 -1
- package/__tests__/adapters/email-retry-progress.test.js +1 -1
- package/__tests__/adapters/email-templates.test.js +1 -1
- package/__tests__/adapters/social-bilibili-adb-api-client.test.js +721 -0
- package/__tests__/adapters/social-bilibili-adb-chromium-cookies-reader.test.js +346 -0
- package/__tests__/adapters/social-bilibili-adb-collector.test.js +284 -0
- package/__tests__/adapters/social-bilibili-adb-cookies-extension.test.js +343 -0
- package/__tests__/adapters/social-bilibili-adb-snapshot-builder.test.js +296 -0
- package/__tests__/adapters/social-douyin-adb-collector.test.js +254 -0
- package/__tests__/adapters/social-douyin-adb-im-db-parser.test.js +304 -0
- package/__tests__/adapters/social-douyin-adb-snapshot-builder.test.js +216 -0
- package/__tests__/adapters/social-weibo-adb-api-client.test.js +362 -0
- package/__tests__/adapters/social-weibo-adb-collector.test.js +201 -0
- package/__tests__/adapters/social-weibo-adb-snapshot-builder.test.js +189 -0
- package/__tests__/adapters/social-xiaohongshu-adb-collector.test.js +207 -0
- package/__tests__/adapters/social-xiaohongshu-adb-sign.test.js +130 -0
- package/__tests__/adapters/system-data-android.test.js +32 -1
- package/__tests__/longtail-adapters.test.js +15 -2
- package/__tests__/shopping-adapters.test.js +96 -0
- package/__tests__/sign-providers.test.js +62 -0
- package/__tests__/travel-adapters.test.js +66 -0
- package/__tests__/whatsapp-adapter.test.js +5 -2
- package/lib/adapters/browser-history-chrome/chrome-db-reader.js +11 -1
- package/lib/adapters/email-imap/email-adapter.js +224 -17
- package/lib/adapters/messaging-telegram/index.js +15 -12
- package/lib/adapters/messaging-whatsapp/index.js +15 -12
- package/lib/adapters/shopping-taobao/index.js +161 -21
- package/lib/adapters/social-bilibili-adb/api-client.js +555 -0
- package/lib/adapters/social-bilibili-adb/chromium-cookies-reader.js +296 -0
- package/lib/adapters/social-bilibili-adb/collector.js +190 -0
- package/lib/adapters/social-bilibili-adb/cookies-extension.js +250 -0
- package/lib/adapters/social-bilibili-adb/index.js +51 -0
- package/lib/adapters/social-bilibili-adb/snapshot-builder.js +197 -0
- package/lib/adapters/social-douyin/index.js +4 -0
- package/lib/adapters/social-douyin-adb/collector.js +165 -0
- package/lib/adapters/social-douyin-adb/db-extension.js +281 -0
- package/lib/adapters/social-douyin-adb/im-db-parser.js +287 -0
- package/lib/adapters/social-douyin-adb/index.js +57 -0
- package/lib/adapters/social-douyin-adb/snapshot-builder.js +174 -0
- package/lib/adapters/social-weibo-adb/api-client.js +281 -0
- package/lib/adapters/social-weibo-adb/collector.js +169 -0
- package/lib/adapters/social-weibo-adb/cookies-extension.js +251 -0
- package/lib/adapters/social-weibo-adb/index.js +55 -0
- package/lib/adapters/social-weibo-adb/snapshot-builder.js +145 -0
- package/lib/adapters/social-xiaohongshu-adb/api-client.js +278 -0
- package/lib/adapters/social-xiaohongshu-adb/collector.js +158 -0
- package/lib/adapters/social-xiaohongshu-adb/cookies-extension.js +211 -0
- package/lib/adapters/social-xiaohongshu-adb/index.js +50 -0
- package/lib/adapters/social-xiaohongshu-adb/sign.js +90 -0
- package/lib/adapters/social-xiaohongshu-adb/snapshot-builder.js +126 -0
- package/lib/adapters/system-data-android/adapter.js +77 -3
- package/lib/adapters/travel-amap/index.js +16 -10
- package/lib/adapters/travel-ctrip/index.js +25 -9
- package/lib/adapters/vscode/vscode-reader.js +7 -1
- package/lib/sign-providers/index.js +20 -0
- package/lib/sign-providers/interface.js +82 -0
- package/lib/sign-providers/null-sign-provider.js +30 -0
- package/package.json +6 -1
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Phase 3c (Xhs C 路径 — 2026-05-25): X-S signature generator (Node port).
|
|
5
|
+
*
|
|
6
|
+
* Byte-parity port of
|
|
7
|
+
* `android-app/.../pdh/social/xiaohongshu/XhsApiClient.kt`:computeXsXt.
|
|
8
|
+
*
|
|
9
|
+
* **Real xhs.js algorithm (open-source reverse-engineered, best-effort)**:
|
|
10
|
+
* 1. payload = "url=" + url_path_with_query + ("" or body_json)
|
|
11
|
+
* 2. raw = ts_ms + payload + a1_cookie
|
|
12
|
+
* 3. md5_hex = MD5(raw).hex() — hex STRING (not bytes)
|
|
13
|
+
* 4. X-S = "XYW_" + base64(utf8_bytes(md5_hex))
|
|
14
|
+
* Critical: base64 encodes the UTF-8 bytes of the hex STRING, not
|
|
15
|
+
* the raw 16 MD5 bytes. This is what xhs.js does — it stringifies
|
|
16
|
+
* the digest before base64-ing it.
|
|
17
|
+
* 5. X-T = ts_ms (as decimal string)
|
|
18
|
+
*
|
|
19
|
+
* **Real xhs.js does one more step after step 3** — XOR-rotate with a
|
|
20
|
+
* key derived from b1 cookie, then base64 with `=` padding. v0.2 we
|
|
21
|
+
* skip that step → ~60% GET hit rate, <30% POST hit rate. UI banner
|
|
22
|
+
* surfaces lastErrorCode=461 when xhs rejects our X-S; collector
|
|
23
|
+
* gracefully degrades to emptyList() per endpoint.
|
|
24
|
+
*
|
|
25
|
+
* Future Phase 3c-v0.3: a WebView-based bridge (see Android-side
|
|
26
|
+
* XhsSignBridge.kt — runs xhs's own JS in a hidden Electron BrowserView)
|
|
27
|
+
* would push the hit rate to ~100%. Out of scope for v0.2 Node port.
|
|
28
|
+
*/
|
|
29
|
+
|
|
30
|
+
const crypto = require("node:crypto");
|
|
31
|
+
|
|
32
|
+
/** "XYW_" prefix — matches xhs.js output. */
|
|
33
|
+
const XS_PREFIX = "XYW_";
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Compute X-S + X-T headers for a GET request.
|
|
37
|
+
*
|
|
38
|
+
* @param {string} urlPathWithQuery url.pathname + url.search (path + "?" + query, encoded)
|
|
39
|
+
* @param {string|null} body POST body as a JSON string, or null/empty for GET
|
|
40
|
+
* @param {string} a1 a1 cookie value (anti-bot fingerprint)
|
|
41
|
+
* @param {{now?: () => number}} [opts] test seam — inject `now: () => 1716383021000`
|
|
42
|
+
* @returns {{xs: string, xt: string}}
|
|
43
|
+
*/
|
|
44
|
+
function computeXsXt(urlPathWithQuery, body, a1, opts = {}) {
|
|
45
|
+
if (typeof urlPathWithQuery !== "string" || urlPathWithQuery.length === 0) {
|
|
46
|
+
throw new TypeError("computeXsXt: urlPathWithQuery must be non-empty string");
|
|
47
|
+
}
|
|
48
|
+
if (typeof a1 !== "string" || a1.length === 0) {
|
|
49
|
+
throw new TypeError("computeXsXt: a1 must be non-empty string");
|
|
50
|
+
}
|
|
51
|
+
const ts = (opts.now || Date.now)();
|
|
52
|
+
const bodyStr = typeof body === "string" ? body : "";
|
|
53
|
+
const payload = "url=" + urlPathWithQuery + bodyStr;
|
|
54
|
+
const raw = `${ts}${payload}${a1}`;
|
|
55
|
+
const md5Hex = crypto.createHash("md5").update(raw, "utf8").digest("hex");
|
|
56
|
+
// base64 encode the UTF-8 bytes of the hex STRING (32 chars → 32 bytes
|
|
57
|
+
// → 44-char base64 with padding). xhs.js NO_WRAP NO_PADDING flags
|
|
58
|
+
// mirror: replace = padding with "", remove newlines (default in
|
|
59
|
+
// Buffer.toString("base64") already no-newlines, only padding to strip).
|
|
60
|
+
const b64NoPad = Buffer.from(md5Hex, "utf8").toString("base64").replace(/=+$/, "");
|
|
61
|
+
return {
|
|
62
|
+
xs: XS_PREFIX + b64NoPad,
|
|
63
|
+
xt: String(ts),
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Extract a1 cookie value from a Cookie header string.
|
|
69
|
+
*
|
|
70
|
+
* "web_session=abc; a1=18d6e123abc; xsec_token=xxx" → "18d6e123abc"
|
|
71
|
+
*
|
|
72
|
+
* Returns null when a1 not present.
|
|
73
|
+
*/
|
|
74
|
+
function extractA1(cookie) {
|
|
75
|
+
if (typeof cookie !== "string") return null;
|
|
76
|
+
for (const part of cookie.split(";")) {
|
|
77
|
+
const trimmed = part.trim();
|
|
78
|
+
if (trimmed.startsWith("a1=")) {
|
|
79
|
+
const v = trimmed.substring(3);
|
|
80
|
+
return v.length > 0 ? v : null;
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
return null;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
module.exports = {
|
|
87
|
+
computeXsXt,
|
|
88
|
+
extractA1,
|
|
89
|
+
XS_PREFIX,
|
|
90
|
+
};
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Phase 3c (Xhs C 路径 — 2026-05-25): API responses → snapshot JSON.
|
|
5
|
+
*
|
|
6
|
+
* Matches the existing `social-xiaohongshu` adapter's snapshot mode
|
|
7
|
+
* schema (schemaVersion=1). Kinds: note / liked / follow.
|
|
8
|
+
*
|
|
9
|
+
* Note: xhs userId is a Base64-ish string (e.g. "5e8c8f7e..."), not a
|
|
10
|
+
* numeric Long. The account.uid in the snapshot is set to userId
|
|
11
|
+
* verbatim (string passthrough); consumers shouldn't expect numeric uid.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
const fs = require("node:fs");
|
|
15
|
+
const path = require("node:path");
|
|
16
|
+
const os = require("node:os");
|
|
17
|
+
const crypto = require("node:crypto");
|
|
18
|
+
|
|
19
|
+
const SNAPSHOT_SCHEMA_VERSION = 1;
|
|
20
|
+
|
|
21
|
+
function buildSnapshot(input) {
|
|
22
|
+
if (!input || typeof input !== "object") {
|
|
23
|
+
throw new TypeError("buildSnapshot: input must be an object");
|
|
24
|
+
}
|
|
25
|
+
const userId = input.userId;
|
|
26
|
+
if (typeof userId !== "string" || userId.length === 0) {
|
|
27
|
+
throw new TypeError("buildSnapshot: input.userId must be a non-empty string");
|
|
28
|
+
}
|
|
29
|
+
const snapshottedAt =
|
|
30
|
+
Number.isFinite(input.snapshottedAt) && input.snapshottedAt > 0
|
|
31
|
+
? input.snapshottedAt
|
|
32
|
+
: Date.now();
|
|
33
|
+
const account = {
|
|
34
|
+
userId, // xhs userId is a string, not numeric
|
|
35
|
+
nickname: typeof input.nickname === "string" ? input.nickname : "",
|
|
36
|
+
};
|
|
37
|
+
const events = [];
|
|
38
|
+
|
|
39
|
+
// notes
|
|
40
|
+
const notes = Array.isArray(input.notes) ? input.notes : [];
|
|
41
|
+
notes.forEach((n, idx) => {
|
|
42
|
+
if (!n || typeof n !== "object") return;
|
|
43
|
+
events.push({
|
|
44
|
+
kind: "note",
|
|
45
|
+
id: n.noteId ? `note-${n.noteId}` : `note-${idx}`,
|
|
46
|
+
capturedAt:
|
|
47
|
+
typeof n.createdAt === "number" && n.createdAt > 0
|
|
48
|
+
? n.createdAt
|
|
49
|
+
: snapshottedAt,
|
|
50
|
+
noteId: n.noteId || null,
|
|
51
|
+
title: n.title || null,
|
|
52
|
+
desc: n.desc || null,
|
|
53
|
+
type: n.type || "normal",
|
|
54
|
+
likedCount: typeof n.likedCount === "number" ? n.likedCount : 0,
|
|
55
|
+
collectedCount:
|
|
56
|
+
typeof n.collectedCount === "number" ? n.collectedCount : 0,
|
|
57
|
+
commentCount: typeof n.commentCount === "number" ? n.commentCount : 0,
|
|
58
|
+
});
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
// liked
|
|
62
|
+
const liked = Array.isArray(input.liked) ? input.liked : [];
|
|
63
|
+
liked.forEach((l, idx) => {
|
|
64
|
+
if (!l || typeof l !== "object") return;
|
|
65
|
+
events.push({
|
|
66
|
+
kind: "liked",
|
|
67
|
+
id: l.noteId ? `liked-${l.noteId}` : `liked-${idx}`,
|
|
68
|
+
// xhs doesn't return liked_at — use snapshottedAt
|
|
69
|
+
capturedAt: snapshottedAt,
|
|
70
|
+
noteId: l.noteId || null,
|
|
71
|
+
title: l.title || null,
|
|
72
|
+
authorNickname: l.authorNickname || null,
|
|
73
|
+
});
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
// follows
|
|
77
|
+
const follows = Array.isArray(input.follows) ? input.follows : [];
|
|
78
|
+
follows.forEach((f, idx) => {
|
|
79
|
+
if (!f || typeof f !== "object") return;
|
|
80
|
+
events.push({
|
|
81
|
+
kind: "follow",
|
|
82
|
+
id: f.userId ? `follow-${f.userId}` : `follow-${idx}`,
|
|
83
|
+
capturedAt: snapshottedAt,
|
|
84
|
+
userId: f.userId || null,
|
|
85
|
+
nickname: f.nickname || null,
|
|
86
|
+
image: f.image || null,
|
|
87
|
+
});
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
return {
|
|
91
|
+
schemaVersion: SNAPSHOT_SCHEMA_VERSION,
|
|
92
|
+
snapshottedAt,
|
|
93
|
+
account,
|
|
94
|
+
events,
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
function writeSnapshotJson(snapshot, opts = {}) {
|
|
99
|
+
const dir = opts.dir || os.tmpdir();
|
|
100
|
+
const fileName =
|
|
101
|
+
opts.fileName || `cc-xhs-snapshot-${crypto.randomUUID()}.json`;
|
|
102
|
+
if (fileName.includes("/") || fileName.includes("\\")) {
|
|
103
|
+
throw new Error(
|
|
104
|
+
"writeSnapshotJson: opts.fileName must be a basename, not a path",
|
|
105
|
+
);
|
|
106
|
+
}
|
|
107
|
+
const full = path.join(dir, fileName);
|
|
108
|
+
fs.writeFileSync(full, JSON.stringify(snapshot), "utf-8");
|
|
109
|
+
return full;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
function cleanupSnapshotJson(filePath) {
|
|
113
|
+
if (!filePath) return;
|
|
114
|
+
try {
|
|
115
|
+
fs.unlinkSync(filePath);
|
|
116
|
+
} catch (_e) {
|
|
117
|
+
// ignore
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
module.exports = {
|
|
122
|
+
buildSnapshot,
|
|
123
|
+
writeSnapshotJson,
|
|
124
|
+
cleanupSnapshotJson,
|
|
125
|
+
SNAPSHOT_SCHEMA_VERSION,
|
|
126
|
+
};
|
|
@@ -28,13 +28,28 @@ const {
|
|
|
28
28
|
} = require("../../constants");
|
|
29
29
|
|
|
30
30
|
const NAME = "system-data-android";
|
|
31
|
+
// v0.3.2 (2026-05-25): denormalise contact identifiers (phones/emails/
|
|
32
|
+
// organization/starred) and app version/install fields onto
|
|
33
|
+
// event.extra so the Vault Browser tap-to-detail sheet can render
|
|
34
|
+
// human-readable fields without joining back to the persons/items
|
|
35
|
+
// tables. Same content lives on the entity rows; events are now a
|
|
36
|
+
// convenience copy. Adds ~50-200 bytes per event but keeps the detail
|
|
37
|
+
// UI single-table.
|
|
38
|
+
// v0.3.1 (2026-05-25): normalize() now emits a synthetic OTHER event per
|
|
39
|
+
// contact + per app. Snapshot mode previously only wrote persons/items;
|
|
40
|
+
// Vault Browser's `category=system` facet only counts events, so the
|
|
41
|
+
// chip showed (0) forever even after a successful sync. Synthetic event
|
|
42
|
+
// per entity (stable id, idempotent across re-syncs via UPSERT) lights
|
|
43
|
+
// up the chip with `total = #contacts + #apps`. occurredAt = capturedAt
|
|
44
|
+
// of the latest snapshot containing the entity. sms/call/media events
|
|
45
|
+
// were already emitted in v0.2 — unchanged.
|
|
31
46
|
// v0.3.0 (2026-05-24): added kind="media-file" via bridge mode
|
|
32
47
|
// (host-adb-bridge media.list across 5 /sdcard categories). Metadata
|
|
33
48
|
// only — path/size/mtime/ext, no file content.
|
|
34
49
|
// v0.2.0 (2026-05-24): added kind="sms" + kind="call" via bridge mode.
|
|
35
50
|
// Snapshot mode still v1 schema — sms/calls/media only land via
|
|
36
51
|
// bridge path until Android snapshot writer is updated to include them.
|
|
37
|
-
const VERSION = "0.3.
|
|
52
|
+
const VERSION = "0.3.2";
|
|
38
53
|
const SNAPSHOT_SCHEMA_VERSION = 1;
|
|
39
54
|
|
|
40
55
|
// Stable per-source originalId — registry.putRawEvent rejects null originalId
|
|
@@ -391,8 +406,42 @@ class SystemDataAndroidAdapter {
|
|
|
391
406
|
if (typeof p.photoUri === "string" && p.photoUri.length > 0) extra.photoUri = p.photoUri;
|
|
392
407
|
if (Object.keys(extra).length > 0) person.extra = extra;
|
|
393
408
|
|
|
409
|
+
// v0.3.1 — synthesise an OTHER event so the snapshot contact shows up
|
|
410
|
+
// in the Vault Browser's `category=system` facet (which counts events,
|
|
411
|
+
// not persons). Stable id keyed on stableKey makes re-syncs idempotent
|
|
412
|
+
// via UPSERT; occurredAt floats forward to the latest snapshot time
|
|
413
|
+
// ("last time we saw this contact").
|
|
414
|
+
//
|
|
415
|
+
// v0.3.2 — duplicate the contact's identifying fields onto event.extra
|
|
416
|
+
// so the Vault Browser's tap-to-detail sheet can render them inline
|
|
417
|
+
// without joining back to the persons table. Phones/emails/relation/
|
|
418
|
+
// starred — same data shape as person.identifiers + person.relation
|
|
419
|
+
// + person.extra, just denormalised so a single events-table read
|
|
420
|
+
// suffices for the detail UI.
|
|
421
|
+
const eventExtra = { kind: "contact-snapshot" };
|
|
422
|
+
if (identifiers.phone && identifiers.phone.length > 0) {
|
|
423
|
+
eventExtra.phones = identifiers.phone;
|
|
424
|
+
}
|
|
425
|
+
if (identifiers.email && identifiers.email.length > 0) {
|
|
426
|
+
eventExtra.emails = identifiers.email;
|
|
427
|
+
}
|
|
428
|
+
if (typeof p.organization === "string" && p.organization.trim().length > 0) {
|
|
429
|
+
eventExtra.organization = p.organization.trim();
|
|
430
|
+
}
|
|
431
|
+
if (typeof p.starred === "boolean") eventExtra.starred = p.starred;
|
|
432
|
+
const event = {
|
|
433
|
+
id: `event-android-contact-${stableKey}`,
|
|
434
|
+
type: ENTITY_TYPES.EVENT,
|
|
435
|
+
subtype: EVENT_SUBTYPES.OTHER,
|
|
436
|
+
occurredAt: raw.capturedAt,
|
|
437
|
+
ingestedAt,
|
|
438
|
+
source: source(`android-contact:${stableKey}`),
|
|
439
|
+
content: { title: `联系人:${displayName}` },
|
|
440
|
+
extra: eventExtra,
|
|
441
|
+
};
|
|
442
|
+
|
|
394
443
|
return {
|
|
395
|
-
events: [],
|
|
444
|
+
events: [event],
|
|
396
445
|
persons: [person],
|
|
397
446
|
places: [],
|
|
398
447
|
items: [],
|
|
@@ -428,8 +477,33 @@ class SystemDataAndroidAdapter {
|
|
|
428
477
|
},
|
|
429
478
|
};
|
|
430
479
|
|
|
480
|
+
// v0.3.1 — same rationale as the contact branch: emit a synthetic
|
|
481
|
+
// OTHER event so installed apps show up in the system facet count.
|
|
482
|
+
// v0.3.2 — copy versioning/install fields onto event.extra so the
|
|
483
|
+
// detail sheet can render them inline.
|
|
484
|
+
const eventExtra = { kind: "app-snapshot", packageName: pkgName };
|
|
485
|
+
if (typeof a.versionName === "string" && a.versionName.length > 0) {
|
|
486
|
+
eventExtra.versionName = a.versionName;
|
|
487
|
+
}
|
|
488
|
+
if (Number.isInteger(a.versionCode)) eventExtra.versionCode = a.versionCode;
|
|
489
|
+
if (Number.isInteger(a.firstInstallTime)) {
|
|
490
|
+
eventExtra.firstInstallTime = a.firstInstallTime;
|
|
491
|
+
}
|
|
492
|
+
if (Number.isInteger(a.lastUpdateTime)) eventExtra.lastUpdateTime = a.lastUpdateTime;
|
|
493
|
+
if (typeof a.isSystem === "boolean") eventExtra.isSystem = a.isSystem;
|
|
494
|
+
const event = {
|
|
495
|
+
id: `event-android-app-${pkgName}`,
|
|
496
|
+
type: ENTITY_TYPES.EVENT,
|
|
497
|
+
subtype: EVENT_SUBTYPES.OTHER,
|
|
498
|
+
occurredAt: raw.capturedAt,
|
|
499
|
+
ingestedAt,
|
|
500
|
+
source: source(`android-app:${pkgName}`),
|
|
501
|
+
content: { title: `应用:${label}` },
|
|
502
|
+
extra: eventExtra,
|
|
503
|
+
};
|
|
504
|
+
|
|
431
505
|
return {
|
|
432
|
-
events: [],
|
|
506
|
+
events: [event],
|
|
433
507
|
persons: [],
|
|
434
508
|
places: [],
|
|
435
509
|
items: [item],
|
|
@@ -19,20 +19,22 @@ const fs = require("node:fs");
|
|
|
19
19
|
const { normalizeTravelRecord, parseChineseDateTime } = require("../travel-base");
|
|
20
20
|
|
|
21
21
|
const NAME = "travel-amap";
|
|
22
|
-
const VERSION = "0.
|
|
22
|
+
const VERSION = "0.6.0"; // 2026-05-25 — account.deviceId OPTIONAL + inputPath alias
|
|
23
23
|
|
|
24
24
|
class AmapAdapter {
|
|
25
25
|
constructor(opts = {}) {
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
26
|
+
// 2026-05-25 — account.deviceId OPTIONAL (mirror Taobao/Ctrip/Telegram).
|
|
27
|
+
// sqlite-mode adapter still requires user to provide a pulled amap.db
|
|
28
|
+
// (`/data/data/com.autonavi.minimap/databases/amap.db`). Earlier strict
|
|
29
|
+
// ctor blocked auto-register at boot → silent "no adapter travel-amap"
|
|
30
|
+
// when Android collector ships extracted db.
|
|
31
|
+
this.account = opts.account || null;
|
|
32
|
+
this._dbPath = opts.dbPath || opts.inputPath || null;
|
|
31
33
|
this._dbDriverFactory = opts.dbDriverFactory || null;
|
|
32
34
|
|
|
33
35
|
this.name = NAME;
|
|
34
36
|
this.version = VERSION;
|
|
35
|
-
this.capabilities = ["sync:sqlite", "parse:amap-history"];
|
|
37
|
+
this.capabilities = ["sync:sqlite", "sync:snapshot", "parse:amap-history"];
|
|
36
38
|
this.extractMode = "device-pull";
|
|
37
39
|
this.rateLimits = {};
|
|
38
40
|
this.dataDisclosure = {
|
|
@@ -46,8 +48,12 @@ class AmapAdapter {
|
|
|
46
48
|
};
|
|
47
49
|
}
|
|
48
50
|
|
|
49
|
-
async authenticate() {
|
|
50
|
-
|
|
51
|
+
async authenticate(ctx = {}) {
|
|
52
|
+
const dbPath = (ctx && (ctx.inputPath || ctx.dbPath)) || this._dbPath;
|
|
53
|
+
if (!dbPath || !fs.existsSync(dbPath)) {
|
|
54
|
+
return { ok: true, account: this.account ? this.account.deviceId : null, mode: "ready" };
|
|
55
|
+
}
|
|
56
|
+
return { ok: true, account: this.account ? this.account.deviceId : null, mode: "snapshot-file" };
|
|
51
57
|
}
|
|
52
58
|
|
|
53
59
|
async healthCheck() {
|
|
@@ -55,7 +61,7 @@ class AmapAdapter {
|
|
|
55
61
|
}
|
|
56
62
|
|
|
57
63
|
async *sync(opts = {}) {
|
|
58
|
-
const dbPath = opts.dbPath || this._dbPath;
|
|
64
|
+
const dbPath = opts.inputPath || opts.dbPath || this._dbPath;
|
|
59
65
|
if (!dbPath || !fs.existsSync(dbPath)) return;
|
|
60
66
|
const Database = this._dbDriverFactory || (() => require("better-sqlite3-multiple-ciphers"));
|
|
61
67
|
const Driver = typeof Database === "function" ? Database() : Database;
|
|
@@ -16,19 +16,21 @@ const fs = require("node:fs");
|
|
|
16
16
|
const { normalizeTravelRecord, parseChineseDateTime } = require("../travel-base");
|
|
17
17
|
|
|
18
18
|
const NAME = "travel-ctrip";
|
|
19
|
-
const VERSION = "0.
|
|
19
|
+
const VERSION = "0.6.0"; // §9.3b — account.email OPTIONAL + inputPath snapshot alias
|
|
20
20
|
|
|
21
21
|
class CtripAdapter {
|
|
22
22
|
constructor(opts = {}) {
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
23
|
+
// §9.3b 2026-05-25 — account.email OPTIONAL (mirror shopping-jd/taobao
|
|
24
|
+
// dual-mode). file-import mode is stateless; bookkeeping account.email
|
|
25
|
+
// is informational, not gating. Earlier strict ctor blocked
|
|
26
|
+
// auto-register at boot → Android collector ship JSON staging path
|
|
27
|
+
// failed with silent "no adapter travel-ctrip".
|
|
28
|
+
this.account = opts.account || null;
|
|
27
29
|
this._dataPath = opts.dataPath || null;
|
|
28
30
|
|
|
29
31
|
this.name = NAME;
|
|
30
32
|
this.version = VERSION;
|
|
31
|
-
this.capabilities = ["import:json", "parse:ctrip-orders"];
|
|
33
|
+
this.capabilities = ["import:json", "sync:snapshot", "parse:ctrip-orders"];
|
|
32
34
|
this.extractMode = "file-import";
|
|
33
35
|
this.rateLimits = {};
|
|
34
36
|
this.dataDisclosure = {
|
|
@@ -40,8 +42,19 @@ class CtripAdapter {
|
|
|
40
42
|
};
|
|
41
43
|
}
|
|
42
44
|
|
|
43
|
-
async authenticate() {
|
|
44
|
-
|
|
45
|
+
async authenticate(ctx = {}) {
|
|
46
|
+
// Snapshot / file-import path: validate file readable when an inputPath
|
|
47
|
+
// / dataPath is provided. Otherwise return ok with whatever account
|
|
48
|
+
// bookkeeping we have (file path can be supplied later via sync(opts)).
|
|
49
|
+
const filePath = (ctx && ctx.inputPath) || ctx.dataPath || this._dataPath;
|
|
50
|
+
if (filePath) {
|
|
51
|
+
try { fs.accessSync(filePath, fs.constants.R_OK); }
|
|
52
|
+
catch (err) {
|
|
53
|
+
return { ok: false, reason: "INPUT_PATH_UNREADABLE", message: `not readable at ${filePath}: ${err.message}` };
|
|
54
|
+
}
|
|
55
|
+
return { ok: true, mode: "snapshot-file" };
|
|
56
|
+
}
|
|
57
|
+
return { ok: true, account: this.account ? this.account.email : null, mode: "ready" };
|
|
45
58
|
}
|
|
46
59
|
|
|
47
60
|
async healthCheck() {
|
|
@@ -49,7 +62,10 @@ class CtripAdapter {
|
|
|
49
62
|
}
|
|
50
63
|
|
|
51
64
|
async *sync(opts = {}) {
|
|
52
|
-
|
|
65
|
+
// Snapshot mode aliases dataPath → inputPath so Android in-APK cc can
|
|
66
|
+
// call syncAdapter("travel-ctrip", path) with the same shape it uses
|
|
67
|
+
// for the other snapshot-mode adapters (shopping-jd / travel-12306).
|
|
68
|
+
const dataPath = opts.inputPath || opts.dataPath || this._dataPath;
|
|
53
69
|
if (!dataPath || !fs.existsSync(dataPath)) return;
|
|
54
70
|
const text = fs.readFileSync(dataPath, "utf-8");
|
|
55
71
|
let records;
|
|
@@ -23,7 +23,12 @@ const os = require("node:os");
|
|
|
23
23
|
// Dual-load: bs3mc tracks Electron's ABI 140 (runtime path), plain
|
|
24
24
|
// better-sqlite3 tracks Node's ABI 127 (test path). Whichever loads
|
|
25
25
|
// wins. See chrome-db-reader.js for the same pattern + rationale.
|
|
26
|
+
//
|
|
27
|
+
// CRITICAL: must be lazy. Top-level invocation kills main process when
|
|
28
|
+
// both modules absent/ABI-mismatched (v5.0.3.87 startup crash).
|
|
29
|
+
let _cachedDatabaseClass = null;
|
|
26
30
|
function loadDatabase() {
|
|
31
|
+
if (_cachedDatabaseClass) return _cachedDatabaseClass;
|
|
27
32
|
for (const mod of ["better-sqlite3-multiple-ciphers", "better-sqlite3"]) {
|
|
28
33
|
let cls;
|
|
29
34
|
try {
|
|
@@ -35,6 +40,7 @@ function loadDatabase() {
|
|
|
35
40
|
try {
|
|
36
41
|
const probe = new cls(":memory:");
|
|
37
42
|
probe.close();
|
|
43
|
+
_cachedDatabaseClass = cls;
|
|
38
44
|
return cls;
|
|
39
45
|
} catch (_e) {
|
|
40
46
|
/* ABI mismatch, try next */
|
|
@@ -44,7 +50,6 @@ function loadDatabase() {
|
|
|
44
50
|
"vscode-reader: neither better-sqlite3-multiple-ciphers nor better-sqlite3 loaded — both ABI-mismatched",
|
|
45
51
|
);
|
|
46
52
|
}
|
|
47
|
-
const Database = loadDatabase();
|
|
48
53
|
|
|
49
54
|
function defaultVscodeRoot() {
|
|
50
55
|
if (process.platform === "win32") {
|
|
@@ -136,6 +141,7 @@ function readTerminalHistory(vscodeRoot, opts = {}) {
|
|
|
136
141
|
}
|
|
137
142
|
}
|
|
138
143
|
try {
|
|
144
|
+
const Database = loadDatabase();
|
|
139
145
|
const db = new Database(tmp, { readonly: true });
|
|
140
146
|
const get = (k) => {
|
|
141
147
|
try {
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Phase 6a (2026-05-25) — sign-providers entry point.
|
|
5
|
+
*
|
|
6
|
+
* Re-exports the abstract `SignProvider` contract + the default
|
|
7
|
+
* `NullSignProvider` impl. Real implementations live in the desktop
|
|
8
|
+
* Electron main process (`desktop-app-vue/src/main/sign-bridge/`)
|
|
9
|
+
* because they need a WebContentsView. CLI / web-shell / test code
|
|
10
|
+
* uses NullSignProvider unless desktop-side wiring injects a real one.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
const { SignProvider } = require("./interface");
|
|
14
|
+
const { NullSignProvider, NULL_SIGN_PROVIDER } = require("./null-sign-provider");
|
|
15
|
+
|
|
16
|
+
module.exports = {
|
|
17
|
+
SignProvider,
|
|
18
|
+
NullSignProvider,
|
|
19
|
+
NULL_SIGN_PROVIDER,
|
|
20
|
+
};
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Phase 6a (2026-05-25) — SignProvider abstract contract for platforms
|
|
5
|
+
* that require per-request signatures (Toutiao `_signature` / Kuaishou
|
|
6
|
+
* `NS_sig3` / Xhs `X-S` / Douyin `X-Bogus`).
|
|
7
|
+
*
|
|
8
|
+
* Mirror of Android-side `pdh/social/SignProvider.kt`. Same 3-method
|
|
9
|
+
* interface so Node API clients can swap impl without changes:
|
|
10
|
+
*
|
|
11
|
+
* - `signUrl(rawUrl, purpose)` — Some platforms (Toutiao, Kuaishou)
|
|
12
|
+
* append `_signature=...` / `NS_sig3=...` to the URL itself; this
|
|
13
|
+
* returns a NEW URL with sig appended, OR `null` if signing failed.
|
|
14
|
+
*
|
|
15
|
+
* - `signedHeaders(rawUrl, purpose)` — Other platforms (Xhs) leave
|
|
16
|
+
* the URL alone and put `X-S` / `X-T` / `X-S-Common` in HTTP
|
|
17
|
+
* headers; this returns an object of header name → value, possibly
|
|
18
|
+
* empty when signing failed.
|
|
19
|
+
*
|
|
20
|
+
* - `shutdown()` — Release the WebContentsView / WebView held by the
|
|
21
|
+
* bridge implementation. Idempotent.
|
|
22
|
+
*
|
|
23
|
+
* **The two methods are independent** — a bridge for Toutiao implements
|
|
24
|
+
* signUrl (URL mutation), Xhs implements signedHeaders (header set).
|
|
25
|
+
* The base abstract returns null/empty for both so subclasses only
|
|
26
|
+
* implement what they need.
|
|
27
|
+
*
|
|
28
|
+
* **`purpose` string** is platform-defined opaque context the JS in the
|
|
29
|
+
* WebContentsView needs to discriminate which signing function to call.
|
|
30
|
+
* For Xhs we encode as `"<pathWithQuery>|<bodyJsonOrEmpty>"`. For
|
|
31
|
+
* Toutiao we use a string like `"feed"` / `"collection"` to pick the
|
|
32
|
+
* acrawler.js entry point. Subclasses define the schema.
|
|
33
|
+
*/
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Abstract base — direct subclassing in JS uses prototypal extension;
|
|
37
|
+
* the methods here are stubs returning null/empty so subclasses can
|
|
38
|
+
* implement only what their platform needs.
|
|
39
|
+
*/
|
|
40
|
+
class SignProvider {
|
|
41
|
+
/**
|
|
42
|
+
* Sign a URL by appending platform-specific query params (e.g.
|
|
43
|
+
* Toutiao's `_signature`). Returns a new URL string with sig
|
|
44
|
+
* appended, or `null` if signing failed (bridge cold / JS rotated).
|
|
45
|
+
*
|
|
46
|
+
* The caller (api-client) MUST handle `null` by surfacing a
|
|
47
|
+
* lastErrorCode like -99 ("_signature unavailable") and returning
|
|
48
|
+
* empty result for that endpoint, NOT throwing.
|
|
49
|
+
*
|
|
50
|
+
* @param {URL|string} rawUrl the unsigned URL
|
|
51
|
+
* @param {string} purpose opaque context for the bridge's JS
|
|
52
|
+
* @returns {Promise<string|null>} signed URL or null
|
|
53
|
+
*/
|
|
54
|
+
async signUrl(_rawUrl, _purpose) {
|
|
55
|
+
return null;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Sign by returning HTTP headers to merge into the request (e.g.
|
|
60
|
+
* Xhs `X-S` / `X-T` / `X-S-Common`). Returns an object — empty
|
|
61
|
+
* map `{}` when signing failed (NOT null) so callers can spread
|
|
62
|
+
* the result unconditionally.
|
|
63
|
+
*
|
|
64
|
+
* @param {URL|string} rawUrl
|
|
65
|
+
* @param {string} purpose
|
|
66
|
+
* @returns {Promise<{[name: string]: string}>}
|
|
67
|
+
*/
|
|
68
|
+
async signedHeaders(_rawUrl, _purpose) {
|
|
69
|
+
return {};
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Release WebContentsView and any background resources. Idempotent —
|
|
74
|
+
* the api-client calls this in a `finally` so racing shutdowns
|
|
75
|
+
* must not throw.
|
|
76
|
+
*/
|
|
77
|
+
async shutdown() {
|
|
78
|
+
// no-op default
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
module.exports = { SignProvider };
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Phase 6a (2026-05-25) — NullSignProvider: default no-op for callers
|
|
5
|
+
* that don't have a WebContentsView context (e.g. headless `cc serve`
|
|
6
|
+
* without Electron, or unit tests).
|
|
7
|
+
*
|
|
8
|
+
* Mirror of Android `pdh/social/NullSignProvider.kt`.
|
|
9
|
+
*
|
|
10
|
+
* API clients use this as the default — when wiring upgrades to
|
|
11
|
+
* Electron+WebContentsView, swap to `XhsSignBridge` / `ToutiaoSignBridge`
|
|
12
|
+
* etc. **without changing the api-client code**. The signing endpoints
|
|
13
|
+
* gracefully degrade to "best-effort" or "empty result" rather than
|
|
14
|
+
* throwing.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
const { SignProvider } = require("./interface");
|
|
18
|
+
|
|
19
|
+
class NullSignProvider extends SignProvider {
|
|
20
|
+
// Inherits all stubs from base — signUrl returns null,
|
|
21
|
+
// signedHeaders returns {}, shutdown is no-op.
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** Frozen singleton — callers should not create multiple NullSignProviders. */
|
|
25
|
+
const NULL_SIGN_PROVIDER = Object.freeze(new NullSignProvider());
|
|
26
|
+
|
|
27
|
+
module.exports = {
|
|
28
|
+
NullSignProvider,
|
|
29
|
+
NULL_SIGN_PROVIDER,
|
|
30
|
+
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@chainlesschain/personal-data-hub",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.6",
|
|
4
4
|
"description": "Personal Data Hub — UnifiedSchema + validators + KG ingest helpers for the data-back-to-the-individual middleware",
|
|
5
5
|
"type": "commonjs",
|
|
6
6
|
"main": "lib/index.js",
|
|
@@ -27,6 +27,7 @@
|
|
|
27
27
|
"./bridges/cc-llm-adapter": "./lib/bridges/cc-llm-adapter.js",
|
|
28
28
|
"./bridges/cc-kg-sink": "./lib/bridges/cc-kg-sink.js",
|
|
29
29
|
"./bridges/cc-rag-sink": "./lib/bridges/cc-rag-sink.js",
|
|
30
|
+
"./sign-providers": "./lib/sign-providers/index.js",
|
|
30
31
|
"./adapters/email-imap": "./lib/adapters/email-imap/index.js",
|
|
31
32
|
"./adapters/alipay-bill": "./lib/adapters/alipay-bill/index.js",
|
|
32
33
|
"./adapters/system-data": "./lib/adapters/system-data/index.js",
|
|
@@ -53,9 +54,13 @@
|
|
|
53
54
|
"./adapters/shopping-jd": "./lib/adapters/shopping-jd/index.js",
|
|
54
55
|
"./adapters/shopping-meituan": "./lib/adapters/shopping-meituan/index.js",
|
|
55
56
|
"./adapters/social-bilibili": "./lib/adapters/social-bilibili/index.js",
|
|
57
|
+
"./adapters/social-bilibili-adb": "./lib/adapters/social-bilibili-adb/index.js",
|
|
56
58
|
"./adapters/social-weibo": "./lib/adapters/social-weibo/index.js",
|
|
59
|
+
"./adapters/social-weibo-adb": "./lib/adapters/social-weibo-adb/index.js",
|
|
57
60
|
"./adapters/social-douyin": "./lib/adapters/social-douyin/index.js",
|
|
61
|
+
"./adapters/social-douyin-adb": "./lib/adapters/social-douyin-adb/index.js",
|
|
58
62
|
"./adapters/social-xiaohongshu": "./lib/adapters/social-xiaohongshu/index.js",
|
|
63
|
+
"./adapters/social-xiaohongshu-adb": "./lib/adapters/social-xiaohongshu-adb/index.js",
|
|
59
64
|
"./adapters/messaging-qq": "./lib/adapters/messaging-qq/index.js",
|
|
60
65
|
"./adapters/messaging-telegram": "./lib/adapters/messaging-telegram/index.js",
|
|
61
66
|
"./adapters/messaging-whatsapp": "./lib/adapters/messaging-whatsapp/index.js",
|