@chainlesschain/personal-data-hub 0.3.1 → 0.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/__tests__/adapters/email-adapter-snapshot.test.js +237 -0
- package/__tests__/adapters/email-adapter.test.js +1 -1
- package/__tests__/adapters/email-pdf-extractor.test.js +1 -1
- package/__tests__/adapters/email-retry-progress.test.js +1 -1
- package/__tests__/adapters/email-templates.test.js +1 -1
- package/__tests__/adapters/social-bilibili-adb-api-client.test.js +721 -0
- package/__tests__/adapters/social-bilibili-adb-chromium-cookies-reader.test.js +346 -0
- package/__tests__/adapters/social-bilibili-adb-collector.test.js +284 -0
- package/__tests__/adapters/social-bilibili-adb-cookies-extension.test.js +343 -0
- package/__tests__/adapters/social-bilibili-adb-snapshot-builder.test.js +296 -0
- package/__tests__/adapters/social-douyin-adb-collector.test.js +254 -0
- package/__tests__/adapters/social-douyin-adb-im-db-parser.test.js +304 -0
- package/__tests__/adapters/social-douyin-adb-snapshot-builder.test.js +216 -0
- package/__tests__/adapters/social-weibo-adb-api-client.test.js +362 -0
- package/__tests__/adapters/social-weibo-adb-collector.test.js +201 -0
- package/__tests__/adapters/social-weibo-adb-snapshot-builder.test.js +189 -0
- package/__tests__/adapters/social-xiaohongshu-adb-collector.test.js +207 -0
- package/__tests__/adapters/social-xiaohongshu-adb-sign.test.js +130 -0
- package/__tests__/adapters/system-data-android.test.js +32 -1
- package/__tests__/longtail-adapters.test.js +15 -2
- package/__tests__/shopping-adapters.test.js +96 -0
- package/__tests__/sign-providers.test.js +62 -0
- package/__tests__/travel-adapters.test.js +66 -0
- package/__tests__/whatsapp-adapter.test.js +5 -2
- package/lib/adapters/browser-history-chrome/chrome-db-reader.js +11 -1
- package/lib/adapters/email-imap/email-adapter.js +224 -17
- package/lib/adapters/messaging-telegram/index.js +15 -12
- package/lib/adapters/messaging-whatsapp/index.js +15 -12
- package/lib/adapters/shopping-taobao/index.js +161 -21
- package/lib/adapters/social-bilibili-adb/api-client.js +555 -0
- package/lib/adapters/social-bilibili-adb/chromium-cookies-reader.js +296 -0
- package/lib/adapters/social-bilibili-adb/collector.js +190 -0
- package/lib/adapters/social-bilibili-adb/cookies-extension.js +250 -0
- package/lib/adapters/social-bilibili-adb/index.js +51 -0
- package/lib/adapters/social-bilibili-adb/snapshot-builder.js +197 -0
- package/lib/adapters/social-douyin/index.js +4 -0
- package/lib/adapters/social-douyin-adb/collector.js +165 -0
- package/lib/adapters/social-douyin-adb/db-extension.js +281 -0
- package/lib/adapters/social-douyin-adb/im-db-parser.js +287 -0
- package/lib/adapters/social-douyin-adb/index.js +57 -0
- package/lib/adapters/social-douyin-adb/snapshot-builder.js +174 -0
- package/lib/adapters/social-weibo-adb/api-client.js +281 -0
- package/lib/adapters/social-weibo-adb/collector.js +169 -0
- package/lib/adapters/social-weibo-adb/cookies-extension.js +251 -0
- package/lib/adapters/social-weibo-adb/index.js +55 -0
- package/lib/adapters/social-weibo-adb/snapshot-builder.js +145 -0
- package/lib/adapters/social-xiaohongshu-adb/api-client.js +278 -0
- package/lib/adapters/social-xiaohongshu-adb/collector.js +158 -0
- package/lib/adapters/social-xiaohongshu-adb/cookies-extension.js +211 -0
- package/lib/adapters/social-xiaohongshu-adb/index.js +50 -0
- package/lib/adapters/social-xiaohongshu-adb/sign.js +90 -0
- package/lib/adapters/social-xiaohongshu-adb/snapshot-builder.js +126 -0
- package/lib/adapters/system-data-android/adapter.js +77 -3
- package/lib/adapters/travel-amap/index.js +16 -10
- package/lib/adapters/travel-ctrip/index.js +25 -9
- package/lib/adapters/vscode/vscode-reader.js +7 -1
- package/lib/sign-providers/index.js +20 -0
- package/lib/sign-providers/interface.js +82 -0
- package/lib/sign-providers/null-sign-provider.js +30 -0
- package/package.json +6 -1
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Phase 1a (Bilibili C 路径 — 2026-05-25): bilibili.cookies ADB extension
|
|
5
|
+
* factory.
|
|
6
|
+
*
|
|
7
|
+
* Plugs into the `opts.extensions` slot of `createHostAdbBridge` /
|
|
8
|
+
* `createDesktopAdbBridge` (see Phase B0 plugin API). Pipeline:
|
|
9
|
+
*
|
|
10
|
+
* 1. ADB-pull /data/data/tv.danmaku.bili/app_webview/Default/Cookies
|
|
11
|
+
* to a host-side temp file via `su -c base64 ...` streaming. Bilibili
|
|
12
|
+
* release APKs are NOT debuggable, so `run-as` is not an option;
|
|
13
|
+
* base64-over-shell is the cross-vendor-safe path (avoids the
|
|
14
|
+
* MIUI/HyperOS FUSE label remap trap [[android-runas-loopback-selinux-split]]
|
|
15
|
+
* that hits stage-via-sdcard).
|
|
16
|
+
* 2. Parse the chromium-shape sqlite via [chromium-cookies-reader].
|
|
17
|
+
* 3. Filter to BILIBILI_COOKIE_NAMES, assemble a `Cookie:` header.
|
|
18
|
+
* 4. Also derive `uid` from DedeUserID + `displayName` from
|
|
19
|
+
* decode-as-needed (Bilibili stores the UID in DedeUserID as a
|
|
20
|
+
* numeric string — no fetch needed).
|
|
21
|
+
*
|
|
22
|
+
* Returns:
|
|
23
|
+
* {
|
|
24
|
+
* cookie: string, // full Cookie header
|
|
25
|
+
* uid: number, // numeric DedeUserID
|
|
26
|
+
* extractedAt: number, // epoch ms
|
|
27
|
+
* diagnostic: {
|
|
28
|
+
* cookieCount: number, // total cookies in DB for bilibili.com
|
|
29
|
+
* hadEncrypted: boolean, // any encrypted_value rows were skipped
|
|
30
|
+
* }
|
|
31
|
+
* }
|
|
32
|
+
*
|
|
33
|
+
* Failure modes (all throw HostAdbBridgeUnavailableError-class errors so
|
|
34
|
+
* the caller's UI can surface a useful banner):
|
|
35
|
+
* - su not available / device not rooted
|
|
36
|
+
* - Bilibili App not installed (path doesn't exist)
|
|
37
|
+
* - cookies sqlite empty (user never logged into Bilibili App)
|
|
38
|
+
* - any required cookie missing (user logged out, or Keystore-wrapped
|
|
39
|
+
* value our parser can't decrypt yet)
|
|
40
|
+
*/
|
|
41
|
+
|
|
42
|
+
const fs = require("node:fs");
|
|
43
|
+
const path = require("node:path");
|
|
44
|
+
const os = require("node:os");
|
|
45
|
+
const crypto = require("node:crypto");
|
|
46
|
+
|
|
47
|
+
const {
|
|
48
|
+
readChromiumCookies,
|
|
49
|
+
assembleBilibiliCookieHeader,
|
|
50
|
+
} = require("./chromium-cookies-reader");
|
|
51
|
+
|
|
52
|
+
const BILIBILI_COOKIES_REMOTE_PATH =
|
|
53
|
+
"/data/data/tv.danmaku.bili/app_webview/Default/Cookies";
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Pull the Bilibili App's Chromium Cookies sqlite to a host-side temp file
|
|
57
|
+
* via ADB `su -c base64`.
|
|
58
|
+
*
|
|
59
|
+
* Uses base64 streaming rather than stage-via-sdcard because:
|
|
60
|
+
* (a) avoids MIUI/HyperOS SELinux label-remap on /sdcard ([[android-runas-loopback-selinux-split]])
|
|
61
|
+
* (b) avoids leaving a copy in /sdcard if the host-side write fails
|
|
62
|
+
* (c) works identically across vendor ROMs since we never touch the FUSE
|
|
63
|
+
* layer
|
|
64
|
+
*
|
|
65
|
+
* Tradeoff: base64 has 33% size overhead. Bilibili's Cookies file is
|
|
66
|
+
* typically 50-200 KB, so this is negligible (<300 KB over the wire vs
|
|
67
|
+
* raw 200 KB).
|
|
68
|
+
*/
|
|
69
|
+
async function pullCookiesViaSu(adb, serial, opts) {
|
|
70
|
+
const adbOpts = { serial, timeoutMs: opts?.timeoutMs || 60_000 };
|
|
71
|
+
// Probe existence first — gives a cleaner error than a base64-of-missing-file
|
|
72
|
+
// attempt (which would spit "No such file" to stdout).
|
|
73
|
+
const lsOut = await adb(
|
|
74
|
+
[
|
|
75
|
+
"shell",
|
|
76
|
+
"su",
|
|
77
|
+
"-c",
|
|
78
|
+
`ls ${BILIBILI_COOKIES_REMOTE_PATH} 2>/dev/null || echo NOT_FOUND`,
|
|
79
|
+
],
|
|
80
|
+
adbOpts,
|
|
81
|
+
);
|
|
82
|
+
const lsLine = lsOut.replace(/\r+$/gm, "").trim();
|
|
83
|
+
if (lsLine === "NOT_FOUND" || lsLine === "") {
|
|
84
|
+
throw new Error(
|
|
85
|
+
"BILIBILI_NOT_INSTALLED_OR_NEVER_LOGGED_IN: " +
|
|
86
|
+
BILIBILI_COOKIES_REMOTE_PATH +
|
|
87
|
+
" not found. Install Bilibili App + log in once on the phone, then retry.",
|
|
88
|
+
);
|
|
89
|
+
}
|
|
90
|
+
// su availability — `su -c id -u` returns "0" or "uid=0(root)..." on rooted
|
|
91
|
+
// phones; non-zero/non-root → throw a clear error.
|
|
92
|
+
const idOut = await adb(["shell", "su", "-c", "id -u"], adbOpts);
|
|
93
|
+
const idLine = idOut.replace(/\r+$/gm, "").trim();
|
|
94
|
+
if (idLine !== "0" && !idLine.includes("uid=0")) {
|
|
95
|
+
throw new Error(
|
|
96
|
+
"BILIBILI_NO_ROOT: this phone isn't rooted (su returned `" +
|
|
97
|
+
idLine.substring(0, 60) +
|
|
98
|
+
"`). Bilibili release APK isn't debuggable, so root is required to read its Cookies DB.",
|
|
99
|
+
);
|
|
100
|
+
}
|
|
101
|
+
// Base64-stream the file. We pipe through `tr -d '\n'` so the host side
|
|
102
|
+
// sees a single base64 string with no embedded whitespace artifacts (some
|
|
103
|
+
// Android `base64` impls wrap at 76 columns).
|
|
104
|
+
const b64 = await adb(
|
|
105
|
+
[
|
|
106
|
+
"shell",
|
|
107
|
+
"su",
|
|
108
|
+
"-c",
|
|
109
|
+
`base64 ${BILIBILI_COOKIES_REMOTE_PATH} | tr -d '\\n\\r'`,
|
|
110
|
+
],
|
|
111
|
+
{ ...adbOpts, timeoutMs: opts?.timeoutMs || 60_000 },
|
|
112
|
+
);
|
|
113
|
+
const b64Clean = b64.replace(/[\r\n\t ]+/g, "");
|
|
114
|
+
if (b64Clean.length === 0) {
|
|
115
|
+
throw new Error(
|
|
116
|
+
"BILIBILI_COOKIES_EMPTY: base64 stream returned 0 bytes. su exec may have silently failed (MIUI ROM?), retry or check `adb logcat`.",
|
|
117
|
+
);
|
|
118
|
+
}
|
|
119
|
+
let buf;
|
|
120
|
+
try {
|
|
121
|
+
buf = Buffer.from(b64Clean, "base64");
|
|
122
|
+
} catch (e) {
|
|
123
|
+
throw new Error(
|
|
124
|
+
"BILIBILI_BASE64_PARSE: stream from device wasn't valid base64 (" +
|
|
125
|
+
(e.message || String(e)) +
|
|
126
|
+
"). Possible MIUI ROM corrupting stdout — try plug-in via `adb pull` instead.",
|
|
127
|
+
);
|
|
128
|
+
}
|
|
129
|
+
if (buf.length < 1024) {
|
|
130
|
+
// Chromium Cookies sqlite is >=4KB even when empty (page size + magic
|
|
131
|
+
// header). <1KB means truncation.
|
|
132
|
+
throw new Error(
|
|
133
|
+
"BILIBILI_COOKIES_TRUNCATED: decoded file is only " +
|
|
134
|
+
buf.length +
|
|
135
|
+
" bytes — expected ≥4KB sqlite. Possible su silent fail; check `adb logcat`.",
|
|
136
|
+
);
|
|
137
|
+
}
|
|
138
|
+
// Verify sqlite magic header to catch any kind of corruption early.
|
|
139
|
+
// Magic: "SQLite format 3\0" (16 bytes).
|
|
140
|
+
const magic = buf.subarray(0, 16).toString("latin1");
|
|
141
|
+
if (!magic.startsWith("SQLite format 3")) {
|
|
142
|
+
throw new Error(
|
|
143
|
+
"BILIBILI_NOT_SQLITE: decoded file lacks `SQLite format 3` magic header. Got bytes: " +
|
|
144
|
+
buf.subarray(0, 16).toString("hex"),
|
|
145
|
+
);
|
|
146
|
+
}
|
|
147
|
+
// Write to a unique temp path. Use crypto.randomUUID for collision safety
|
|
148
|
+
// when two desktop bridges run in parallel.
|
|
149
|
+
const tmpDir = os.tmpdir();
|
|
150
|
+
const tmpFile = path.join(
|
|
151
|
+
tmpDir,
|
|
152
|
+
`cc-bilibili-cookies-${crypto.randomUUID()}.db`,
|
|
153
|
+
);
|
|
154
|
+
fs.writeFileSync(tmpFile, buf);
|
|
155
|
+
return tmpFile;
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/**
|
|
159
|
+
* Factory: returns an extension handler suitable for the `opts.extensions`
|
|
160
|
+
* map of `createHostAdbBridge` / `createDesktopAdbBridge`. Wiring:
|
|
161
|
+
*
|
|
162
|
+
* const ext = createBilibiliCookiesExtension();
|
|
163
|
+
* const bridge = createHostAdbBridge({
|
|
164
|
+
* extensions: { "bilibili.cookies": ext },
|
|
165
|
+
* });
|
|
166
|
+
* await bridge.invoke("bilibili.cookies"); // → {cookie, uid, ...}
|
|
167
|
+
*
|
|
168
|
+
* The handler is stateless — no closure-captured device serial / cache.
|
|
169
|
+
* Each invocation pulls a fresh Cookies DB (Bilibili cookies rotate
|
|
170
|
+
* ~weekly; caching across a hub-restart would be brittle).
|
|
171
|
+
*
|
|
172
|
+
* @param {object} [factoryOpts]
|
|
173
|
+
* @param {number} [factoryOpts.timeoutMs=60000] per-adb-call timeout
|
|
174
|
+
* @param {(path: string) => void} [factoryOpts.onCleanupFailed]
|
|
175
|
+
* callback for non-fatal temp-file cleanup errors (default = swallow)
|
|
176
|
+
* @returns {(params: object, ctx: object) => Promise<object>}
|
|
177
|
+
*/
|
|
178
|
+
function createBilibiliCookiesExtension(factoryOpts = {}) {
|
|
179
|
+
const timeoutMs = factoryOpts.timeoutMs || 60_000;
|
|
180
|
+
const onCleanupFailed = factoryOpts.onCleanupFailed || (() => {});
|
|
181
|
+
|
|
182
|
+
return async function bilibiliCookiesHandler(_params, ctx) {
|
|
183
|
+
if (!ctx || typeof ctx.adb !== "function" || typeof ctx.pickDevice !== "function") {
|
|
184
|
+
throw new TypeError(
|
|
185
|
+
"bilibili.cookies extension: ctx must provide {adb, pickDevice} (got " +
|
|
186
|
+
typeof ctx +
|
|
187
|
+
")",
|
|
188
|
+
);
|
|
189
|
+
}
|
|
190
|
+
const serial = await ctx.pickDevice();
|
|
191
|
+
let tmpFile = null;
|
|
192
|
+
try {
|
|
193
|
+
tmpFile = await pullCookiesViaSu(ctx.adb, serial, { timeoutMs });
|
|
194
|
+
const cookies = readChromiumCookies(tmpFile, "bilibili.com");
|
|
195
|
+
const cookieCount = cookies.length;
|
|
196
|
+
const hadEncrypted = (cookies._skippedEncryptedCount || 0) > 0;
|
|
197
|
+
const { header, missing } = assembleBilibiliCookieHeader(cookies);
|
|
198
|
+
if (header === null) {
|
|
199
|
+
throw new Error(
|
|
200
|
+
"BILIBILI_COOKIES_INCOMPLETE: missing required cookies " +
|
|
201
|
+
JSON.stringify(missing) +
|
|
202
|
+
". User probably logged out, or Bilibili App version uses Keystore-wrapped values (hadEncrypted=" +
|
|
203
|
+
hadEncrypted +
|
|
204
|
+
"). Tell user to relog on phone.",
|
|
205
|
+
);
|
|
206
|
+
}
|
|
207
|
+
// Derive uid from DedeUserID (we parse it again here because the
|
|
208
|
+
// assembled header has it but the caller may not want to split the
|
|
209
|
+
// header string themselves).
|
|
210
|
+
const dedeRow = cookies.find((c) => c.name === "DedeUserID");
|
|
211
|
+
const uid = dedeRow ? parseInt(dedeRow.value, 10) : null;
|
|
212
|
+
if (!Number.isFinite(uid) || uid <= 0) {
|
|
213
|
+
throw new Error(
|
|
214
|
+
"BILIBILI_INVALID_UID: DedeUserID=" +
|
|
215
|
+
(dedeRow ? dedeRow.value : "<missing>") +
|
|
216
|
+
" is not a positive integer.",
|
|
217
|
+
);
|
|
218
|
+
}
|
|
219
|
+
return {
|
|
220
|
+
cookie: header,
|
|
221
|
+
uid,
|
|
222
|
+
extractedAt: Date.now(),
|
|
223
|
+
diagnostic: {
|
|
224
|
+
cookieCount,
|
|
225
|
+
hadEncrypted,
|
|
226
|
+
},
|
|
227
|
+
};
|
|
228
|
+
} finally {
|
|
229
|
+
// Best-effort cleanup. Leaving a stale temp file isn't a security
|
|
230
|
+
// issue (we wrote it ourselves; nothing else has the path), but it's
|
|
231
|
+
// unhygienic over time.
|
|
232
|
+
if (tmpFile) {
|
|
233
|
+
try {
|
|
234
|
+
fs.unlinkSync(tmpFile);
|
|
235
|
+
} catch (e) {
|
|
236
|
+
onCleanupFailed(tmpFile);
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
};
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
module.exports = {
|
|
244
|
+
createBilibiliCookiesExtension,
|
|
245
|
+
BILIBILI_COOKIES_REMOTE_PATH,
|
|
246
|
+
// Exposed for tests
|
|
247
|
+
_internals: {
|
|
248
|
+
pullCookiesViaSu,
|
|
249
|
+
},
|
|
250
|
+
};
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* social-bilibili-adb — Phase 1 (Bilibili C 路径) entry.
|
|
5
|
+
*
|
|
6
|
+
* Phase 1a (commit `7c12fd253`) — cookies extraction layer
|
|
7
|
+
* Phase 1b (this commit) — Node API client + snapshot builder + collector
|
|
8
|
+
* Phase 1c (next) — wiring injection + UI + real-device E2E
|
|
9
|
+
*
|
|
10
|
+
* Pipeline (see collector.js):
|
|
11
|
+
* bridge.invoke("bilibili.cookies")
|
|
12
|
+
* → BilibiliApiClient (4 endpoints, WBI-signed)
|
|
13
|
+
* → buildSnapshot → writeSnapshotJson
|
|
14
|
+
* → registry.syncAdapter("social-bilibili", { inputPath })
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
const {
|
|
18
|
+
createBilibiliCookiesExtension,
|
|
19
|
+
BILIBILI_COOKIES_REMOTE_PATH,
|
|
20
|
+
} = require("./cookies-extension");
|
|
21
|
+
const {
|
|
22
|
+
readChromiumCookies,
|
|
23
|
+
assembleBilibiliCookieHeader,
|
|
24
|
+
BILIBILI_COOKIE_NAMES,
|
|
25
|
+
} = require("./chromium-cookies-reader");
|
|
26
|
+
const { BilibiliApiClient, extractUid } = require("./api-client");
|
|
27
|
+
const {
|
|
28
|
+
buildSnapshot,
|
|
29
|
+
writeSnapshotJson,
|
|
30
|
+
cleanupSnapshotJson,
|
|
31
|
+
SNAPSHOT_SCHEMA_VERSION,
|
|
32
|
+
} = require("./snapshot-builder");
|
|
33
|
+
const { collect, collectAndSync } = require("./collector");
|
|
34
|
+
|
|
35
|
+
module.exports = {
|
|
36
|
+
// Phase 1a
|
|
37
|
+
createBilibiliCookiesExtension,
|
|
38
|
+
BILIBILI_COOKIES_REMOTE_PATH,
|
|
39
|
+
readChromiumCookies,
|
|
40
|
+
assembleBilibiliCookieHeader,
|
|
41
|
+
BILIBILI_COOKIE_NAMES,
|
|
42
|
+
// Phase 1b
|
|
43
|
+
BilibiliApiClient,
|
|
44
|
+
extractUid,
|
|
45
|
+
buildSnapshot,
|
|
46
|
+
writeSnapshotJson,
|
|
47
|
+
cleanupSnapshotJson,
|
|
48
|
+
SNAPSHOT_SCHEMA_VERSION,
|
|
49
|
+
collect,
|
|
50
|
+
collectAndSync,
|
|
51
|
+
};
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Phase 1b (Bilibili C 路径 — 2026-05-25): convert 4 API response arrays
|
|
5
|
+
* into a snapshot JSON file that the existing `social-bilibili` adapter
|
|
6
|
+
* consumes in snapshot mode.
|
|
7
|
+
*
|
|
8
|
+
* Schema (mirrors `adapter.js`:SNAPSHOT_SCHEMA_VERSION = 1):
|
|
9
|
+
*
|
|
10
|
+
* {
|
|
11
|
+
* "schemaVersion": 1,
|
|
12
|
+
* "snapshottedAt": <epoch-ms>,
|
|
13
|
+
* "account": { "uid": "<numeric uid as string>", "displayName": "" },
|
|
14
|
+
* "events": [
|
|
15
|
+
* { "kind": "history", "id": "BV1...", "capturedAt": <ms>, ...fields },
|
|
16
|
+
* { "kind": "favourite", "id": "fav-BV1...", "capturedAt": <ms>, ...fields },
|
|
17
|
+
* { "kind": "dynamic", "id": "dyn-<rid>", "capturedAt": <ms>, ...fields },
|
|
18
|
+
* { "kind": "follow", "id": "follow-<mid>", "capturedAt": <ms>, ...fields }
|
|
19
|
+
* ]
|
|
20
|
+
* }
|
|
21
|
+
*
|
|
22
|
+
* Field mapping (BilibiliApiClient.js return shapes → event fields):
|
|
23
|
+
* HistoryItem.viewAt → capturedAt
|
|
24
|
+
* FavouriteItem.savedAt → capturedAt
|
|
25
|
+
* DynamicItem.publishedAt → capturedAt
|
|
26
|
+
* FollowItem.followedAt → capturedAt
|
|
27
|
+
*
|
|
28
|
+
* All other fields pass through verbatim (the adapter stores the whole
|
|
29
|
+
* event object as the payload via `{...ev, account}`).
|
|
30
|
+
*
|
|
31
|
+
* Stable `id` derivation matches the Android side
|
|
32
|
+
* (BilibiliLocalCollector.kt does the same prefix-namespacing):
|
|
33
|
+
* history: bvid (fallback "history-<index>")
|
|
34
|
+
* favourite: "fav-" + bvid (fallback "fav-<index>")
|
|
35
|
+
* dynamic: "dyn-" + rid (fallback "dyn-<index>")
|
|
36
|
+
* follow: "follow-" + mid (fallback "follow-<index>")
|
|
37
|
+
*/
|
|
38
|
+
|
|
39
|
+
const fs = require("node:fs");
|
|
40
|
+
const path = require("node:path");
|
|
41
|
+
const os = require("node:os");
|
|
42
|
+
const crypto = require("node:crypto");
|
|
43
|
+
|
|
44
|
+
const SNAPSHOT_SCHEMA_VERSION = 1;
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Build the in-memory snapshot object. Pure function — no disk I/O.
|
|
48
|
+
*
|
|
49
|
+
* @param {{
|
|
50
|
+
* uid: number,
|
|
51
|
+
* displayName?: string,
|
|
52
|
+
* history?: Array,
|
|
53
|
+
* favourites?: Array,
|
|
54
|
+
* dynamics?: Array,
|
|
55
|
+
* follows?: Array,
|
|
56
|
+
* snapshottedAt?: number,
|
|
57
|
+
* }} input
|
|
58
|
+
* @returns {{schemaVersion: number, snapshottedAt: number, account: object, events: Array}}
|
|
59
|
+
*/
|
|
60
|
+
function buildSnapshot(input) {
|
|
61
|
+
if (!input || typeof input !== "object") {
|
|
62
|
+
throw new TypeError("buildSnapshot: input must be an object");
|
|
63
|
+
}
|
|
64
|
+
const uid = input.uid;
|
|
65
|
+
if (!Number.isFinite(uid) || uid <= 0) {
|
|
66
|
+
throw new TypeError(
|
|
67
|
+
"buildSnapshot: input.uid must be a positive integer (was " + uid + ")",
|
|
68
|
+
);
|
|
69
|
+
}
|
|
70
|
+
const snapshottedAt =
|
|
71
|
+
Number.isFinite(input.snapshottedAt) && input.snapshottedAt > 0
|
|
72
|
+
? input.snapshottedAt
|
|
73
|
+
: Date.now();
|
|
74
|
+
const account = {
|
|
75
|
+
uid: String(uid),
|
|
76
|
+
displayName:
|
|
77
|
+
typeof input.displayName === "string" ? input.displayName : "",
|
|
78
|
+
};
|
|
79
|
+
const events = [];
|
|
80
|
+
|
|
81
|
+
// history
|
|
82
|
+
const history = Array.isArray(input.history) ? input.history : [];
|
|
83
|
+
history.forEach((h, idx) => {
|
|
84
|
+
if (!h || typeof h !== "object") return;
|
|
85
|
+
events.push({
|
|
86
|
+
kind: "history",
|
|
87
|
+
id: h.bvid || `history-${idx}`,
|
|
88
|
+
capturedAt: typeof h.viewAt === "number" ? h.viewAt : snapshottedAt,
|
|
89
|
+
title: h.title || null,
|
|
90
|
+
bvid: h.bvid || null,
|
|
91
|
+
avid: typeof h.avid === "number" ? h.avid : null,
|
|
92
|
+
duration: typeof h.duration === "number" ? h.duration : null,
|
|
93
|
+
uploader: h.uploader || null,
|
|
94
|
+
uploaderMid: typeof h.uploaderMid === "number" ? h.uploaderMid : null,
|
|
95
|
+
part: h.part || null,
|
|
96
|
+
});
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
// favourites
|
|
100
|
+
const favs = Array.isArray(input.favourites) ? input.favourites : [];
|
|
101
|
+
favs.forEach((f, idx) => {
|
|
102
|
+
if (!f || typeof f !== "object") return;
|
|
103
|
+
events.push({
|
|
104
|
+
kind: "favourite",
|
|
105
|
+
id: f.bvid ? `fav-${f.bvid}` : `fav-${idx}`,
|
|
106
|
+
capturedAt: typeof f.savedAt === "number" ? f.savedAt : snapshottedAt,
|
|
107
|
+
title: f.title || null,
|
|
108
|
+
bvid: f.bvid || null,
|
|
109
|
+
folderName: f.folderName || null,
|
|
110
|
+
uploader: f.uploader || null,
|
|
111
|
+
});
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
// dynamics
|
|
115
|
+
const dyns = Array.isArray(input.dynamics) ? input.dynamics : [];
|
|
116
|
+
dyns.forEach((d, idx) => {
|
|
117
|
+
if (!d || typeof d !== "object") return;
|
|
118
|
+
events.push({
|
|
119
|
+
kind: "dynamic",
|
|
120
|
+
id: d.rid ? `dyn-${d.rid}` : `dyn-${idx}`,
|
|
121
|
+
capturedAt:
|
|
122
|
+
typeof d.publishedAt === "number" ? d.publishedAt : snapshottedAt,
|
|
123
|
+
summary: d.summary || null,
|
|
124
|
+
dynamicType: d.dynamicType || "unknown",
|
|
125
|
+
authorMid: typeof d.authorMid === "number" ? d.authorMid : null,
|
|
126
|
+
authorName: d.authorName || null,
|
|
127
|
+
});
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
// follows
|
|
131
|
+
const fols = Array.isArray(input.follows) ? input.follows : [];
|
|
132
|
+
fols.forEach((f, idx) => {
|
|
133
|
+
if (!f || typeof f !== "object") return;
|
|
134
|
+
const mid = typeof f.mid === "number" ? f.mid : null;
|
|
135
|
+
events.push({
|
|
136
|
+
kind: "follow",
|
|
137
|
+
id: mid ? `follow-${mid}` : `follow-${idx}`,
|
|
138
|
+
capturedAt:
|
|
139
|
+
typeof f.followedAt === "number" ? f.followedAt : snapshottedAt,
|
|
140
|
+
mid: mid != null ? String(mid) : null,
|
|
141
|
+
uname: f.uname || null,
|
|
142
|
+
face: f.face || null,
|
|
143
|
+
sign: f.sign || null,
|
|
144
|
+
});
|
|
145
|
+
});
|
|
146
|
+
|
|
147
|
+
return {
|
|
148
|
+
schemaVersion: SNAPSHOT_SCHEMA_VERSION,
|
|
149
|
+
snapshottedAt,
|
|
150
|
+
account,
|
|
151
|
+
events,
|
|
152
|
+
};
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* Write a snapshot object to disk as JSON. Default destination is
|
|
157
|
+
* `<os.tmpdir()>/cc-bilibili-snapshot-<uuid>.json`. Returns the absolute
|
|
158
|
+
* path written. Caller is responsible for cleanup (BilibiliAdbCollector
|
|
159
|
+
* does this in a try/finally).
|
|
160
|
+
*
|
|
161
|
+
* @param {object} snapshot output of buildSnapshot
|
|
162
|
+
* @param {{dir?: string, fileName?: string}} [opts]
|
|
163
|
+
* @returns {string} absolute path
|
|
164
|
+
*/
|
|
165
|
+
function writeSnapshotJson(snapshot, opts = {}) {
|
|
166
|
+
const dir = opts.dir || os.tmpdir();
|
|
167
|
+
const fileName =
|
|
168
|
+
opts.fileName || `cc-bilibili-snapshot-${crypto.randomUUID()}.json`;
|
|
169
|
+
if (fileName.includes("/") || fileName.includes("\\")) {
|
|
170
|
+
throw new Error(
|
|
171
|
+
"writeSnapshotJson: opts.fileName must be a basename, not a path",
|
|
172
|
+
);
|
|
173
|
+
}
|
|
174
|
+
const full = path.join(dir, fileName);
|
|
175
|
+
fs.writeFileSync(full, JSON.stringify(snapshot), "utf-8");
|
|
176
|
+
return full;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/**
|
|
180
|
+
* Best-effort delete of a snapshot file. Used in finally blocks; never
|
|
181
|
+
* throws.
|
|
182
|
+
*/
|
|
183
|
+
function cleanupSnapshotJson(filePath) {
|
|
184
|
+
if (!filePath) return;
|
|
185
|
+
try {
|
|
186
|
+
fs.unlinkSync(filePath);
|
|
187
|
+
} catch (_e) {
|
|
188
|
+
// ignore — temp file cleanup is best-effort
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
module.exports = {
|
|
193
|
+
buildSnapshot,
|
|
194
|
+
writeSnapshotJson,
|
|
195
|
+
cleanupSnapshotJson,
|
|
196
|
+
SNAPSHOT_SCHEMA_VERSION,
|
|
197
|
+
};
|
|
@@ -62,6 +62,8 @@ const KIND_HISTORY = "history"; // v0.3 (X-Bogus required)
|
|
|
62
62
|
const KIND_FAVOURITE = "favourite"; // v0.3 (X-Bogus required)
|
|
63
63
|
const KIND_LIKE = "like"; // v0.3 (X-Bogus required)
|
|
64
64
|
const KIND_SEARCH = "search"; // legacy sqlite-mode only
|
|
65
|
+
const KIND_MESSAGE = "message"; // Phase 2a — IM private messages from <uid>_im.db (abrignoni DFIR)
|
|
66
|
+
const KIND_CONTACT = "contact"; // Phase 2a — SIMPLE_USER table contacts/follows from <uid>_im.db
|
|
65
67
|
|
|
66
68
|
// Forward-compat: list every kind v0.3+ may emit so cc adapter accepts
|
|
67
69
|
// snapshots from a newer Android even if this JS hasn't been bumped yet.
|
|
@@ -70,6 +72,8 @@ const VALID_SNAPSHOT_KINDS = Object.freeze([
|
|
|
70
72
|
KIND_HISTORY,
|
|
71
73
|
KIND_FAVOURITE,
|
|
72
74
|
KIND_LIKE,
|
|
75
|
+
KIND_MESSAGE,
|
|
76
|
+
KIND_CONTACT,
|
|
73
77
|
]);
|
|
74
78
|
|
|
75
79
|
function stableOriginalId(kind, id) {
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Phase 2a (Douyin C 路径 — 2026-05-25): end-to-end orchestrator.
|
|
5
|
+
*
|
|
6
|
+
* bridge.invoke("douyin.pull-im-db") ← Phase 2a db-extension
|
|
7
|
+
* │
|
|
8
|
+
* ▼ {tempPath, uid, walPath?, shmPath?, cleanup}
|
|
9
|
+
* parseImDb(tempPath) ← Phase 2a im-db-parser
|
|
10
|
+
* │
|
|
11
|
+
* ▼ {messages, contacts, diagnostic}
|
|
12
|
+
* buildSnapshot + writeSnapshotJson ← Phase 2a snapshot-builder
|
|
13
|
+
* │
|
|
14
|
+
* ▼ staging JSON path
|
|
15
|
+
* registry.syncAdapter("social-douyin", { inputPath }) ← existing
|
|
16
|
+
* snapshot mode
|
|
17
|
+
*
|
|
18
|
+
* Pattern mirrors social-bilibili-adb/collector.js — same try/finally
|
|
19
|
+
* cleanup, same `{ok, report?, reason?, message?}` return shape.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
const { parseImDb } = require("./im-db-parser");
|
|
23
|
+
const {
|
|
24
|
+
buildSnapshot,
|
|
25
|
+
writeSnapshotJson,
|
|
26
|
+
cleanupSnapshotJson,
|
|
27
|
+
} = require("./snapshot-builder");
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Pull IM db → parse → write snapshot. Returns the staging path + counts
|
|
31
|
+
* + diagnostic. Caller decides what to do with the snapshot (typically
|
|
32
|
+
* passes to registry.syncAdapter then cleanup).
|
|
33
|
+
*
|
|
34
|
+
* @param {object} bridge host-adb-bridge instance — must have
|
|
35
|
+
* "douyin.pull-im-db" extension registered
|
|
36
|
+
* @param {{
|
|
37
|
+
* uid?: string, // 19-digit uid to disambiguate multi-account
|
|
38
|
+
* limits?: {messages?: number, contacts?: number},
|
|
39
|
+
* stagingDir?: string,
|
|
40
|
+
* displayName?: string,
|
|
41
|
+
* now?: () => number,
|
|
42
|
+
* }} [opts]
|
|
43
|
+
*/
|
|
44
|
+
async function collect(bridge, opts = {}) {
|
|
45
|
+
if (!bridge || typeof bridge.invoke !== "function") {
|
|
46
|
+
throw new TypeError(
|
|
47
|
+
"DouyinAdbCollector.collect: bridge must expose invoke(method, params)",
|
|
48
|
+
);
|
|
49
|
+
}
|
|
50
|
+
const now = opts.now || Date.now;
|
|
51
|
+
|
|
52
|
+
// 1. Pull the IM db cohort.
|
|
53
|
+
const pullResult = await bridge.invoke("douyin.pull-im-db", {
|
|
54
|
+
uid: opts.uid,
|
|
55
|
+
});
|
|
56
|
+
if (
|
|
57
|
+
!pullResult ||
|
|
58
|
+
typeof pullResult.tempPath !== "string" ||
|
|
59
|
+
typeof pullResult.uid !== "string"
|
|
60
|
+
) {
|
|
61
|
+
throw new Error(
|
|
62
|
+
"DouyinAdbCollector.collect: bridge.invoke('douyin.pull-im-db') returned malformed payload",
|
|
63
|
+
);
|
|
64
|
+
}
|
|
65
|
+
const { tempPath, uid, cleanup: cleanupDbCohort } = pullResult;
|
|
66
|
+
|
|
67
|
+
try {
|
|
68
|
+
// 2. Parse the IM db locally.
|
|
69
|
+
const parsed = parseImDb(tempPath, {
|
|
70
|
+
limitMessages: opts.limits && opts.limits.messages,
|
|
71
|
+
limitContacts: opts.limits && opts.limits.contacts,
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
// 3. Build snapshot + write to staging.
|
|
75
|
+
const snapshot = buildSnapshot({
|
|
76
|
+
uid,
|
|
77
|
+
displayName: opts.displayName,
|
|
78
|
+
messages: parsed.messages,
|
|
79
|
+
contacts: parsed.contacts,
|
|
80
|
+
snapshottedAt: now(),
|
|
81
|
+
});
|
|
82
|
+
const snapshotPath = writeSnapshotJson(snapshot, {
|
|
83
|
+
dir: opts.stagingDir,
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
return {
|
|
87
|
+
snapshotPath,
|
|
88
|
+
uid,
|
|
89
|
+
eventCounts: {
|
|
90
|
+
message: parsed.messages.length,
|
|
91
|
+
contact: parsed.contacts.length,
|
|
92
|
+
total: parsed.messages.length + parsed.contacts.length,
|
|
93
|
+
},
|
|
94
|
+
parserDiagnostic: parsed.diagnostic,
|
|
95
|
+
// Cleanup the pulled db cohort right after parsing — we have the
|
|
96
|
+
// events in memory, no reason to keep the .db lying around.
|
|
97
|
+
_dbCohortCleanup: cleanupDbCohort,
|
|
98
|
+
};
|
|
99
|
+
} catch (err) {
|
|
100
|
+
// On any parse / build / write failure, cleanup the pulled db cohort
|
|
101
|
+
// before re-throwing so we don't leak the temp file.
|
|
102
|
+
if (typeof cleanupDbCohort === "function") {
|
|
103
|
+
try {
|
|
104
|
+
cleanupDbCohort();
|
|
105
|
+
} catch (_e) {
|
|
106
|
+
// best-effort
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
throw err;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* One-shot convenience: collect + syncAdapter("social-douyin") + cleanup
|
|
115
|
+
* everything (both the db cohort AND the snapshot JSON, even if
|
|
116
|
+
* syncAdapter throws).
|
|
117
|
+
*
|
|
118
|
+
* @param {object} bridge host-adb-bridge
|
|
119
|
+
* @param {object} registry AdapterRegistry
|
|
120
|
+
* @param {object} [opts] forwarded to `collect()`
|
|
121
|
+
* @returns {Promise<object>} SyncReport + collector diagnostic
|
|
122
|
+
*/
|
|
123
|
+
async function collectAndSync(bridge, registry, opts = {}) {
|
|
124
|
+
if (!registry || typeof registry.syncAdapter !== "function") {
|
|
125
|
+
throw new TypeError(
|
|
126
|
+
"DouyinAdbCollector.collectAndSync: registry must expose syncAdapter(name, options)",
|
|
127
|
+
);
|
|
128
|
+
}
|
|
129
|
+
const collectResult = await collect(bridge, opts);
|
|
130
|
+
let syncReport = null;
|
|
131
|
+
let cleanupFailed = false;
|
|
132
|
+
try {
|
|
133
|
+
syncReport = await registry.syncAdapter("social-douyin", {
|
|
134
|
+
inputPath: collectResult.snapshotPath,
|
|
135
|
+
});
|
|
136
|
+
} finally {
|
|
137
|
+
try {
|
|
138
|
+
cleanupSnapshotJson(collectResult.snapshotPath);
|
|
139
|
+
} catch (_e) {
|
|
140
|
+
cleanupFailed = true;
|
|
141
|
+
}
|
|
142
|
+
// Always cleanup the pulled db cohort.
|
|
143
|
+
if (typeof collectResult._dbCohortCleanup === "function") {
|
|
144
|
+
try {
|
|
145
|
+
collectResult._dbCohortCleanup();
|
|
146
|
+
} catch (_e) {
|
|
147
|
+
cleanupFailed = true;
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
return {
|
|
152
|
+
...syncReport,
|
|
153
|
+
douyin: {
|
|
154
|
+
uid: collectResult.uid,
|
|
155
|
+
eventCounts: collectResult.eventCounts,
|
|
156
|
+
parserDiagnostic: collectResult.parserDiagnostic,
|
|
157
|
+
cleanupFailed,
|
|
158
|
+
},
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
module.exports = {
|
|
163
|
+
collect,
|
|
164
|
+
collectAndSync,
|
|
165
|
+
};
|