@chainlesschain/personal-data-hub 0.3.1 → 0.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/__tests__/adapters/email-adapter-snapshot.test.js +237 -0
- package/__tests__/adapters/email-adapter.test.js +1 -1
- package/__tests__/adapters/email-pdf-extractor.test.js +1 -1
- package/__tests__/adapters/email-retry-progress.test.js +1 -1
- package/__tests__/adapters/email-templates.test.js +1 -1
- package/__tests__/adapters/social-bilibili-adb-api-client.test.js +721 -0
- package/__tests__/adapters/social-bilibili-adb-chromium-cookies-reader.test.js +346 -0
- package/__tests__/adapters/social-bilibili-adb-collector.test.js +284 -0
- package/__tests__/adapters/social-bilibili-adb-cookies-extension.test.js +343 -0
- package/__tests__/adapters/social-bilibili-adb-snapshot-builder.test.js +296 -0
- package/__tests__/adapters/social-douyin-adb-collector.test.js +254 -0
- package/__tests__/adapters/social-douyin-adb-im-db-parser.test.js +304 -0
- package/__tests__/adapters/social-douyin-adb-snapshot-builder.test.js +216 -0
- package/__tests__/adapters/social-weibo-adb-api-client.test.js +362 -0
- package/__tests__/adapters/social-weibo-adb-collector.test.js +201 -0
- package/__tests__/adapters/social-weibo-adb-snapshot-builder.test.js +189 -0
- package/__tests__/adapters/social-xiaohongshu-adb-collector.test.js +207 -0
- package/__tests__/adapters/social-xiaohongshu-adb-sign.test.js +130 -0
- package/__tests__/adapters/system-data-android.test.js +32 -1
- package/__tests__/longtail-adapters.test.js +15 -2
- package/__tests__/shopping-adapters.test.js +96 -0
- package/__tests__/sign-providers.test.js +62 -0
- package/__tests__/travel-adapters.test.js +66 -0
- package/__tests__/whatsapp-adapter.test.js +5 -2
- package/lib/adapters/browser-history-chrome/chrome-db-reader.js +11 -1
- package/lib/adapters/email-imap/email-adapter.js +224 -17
- package/lib/adapters/messaging-telegram/index.js +15 -12
- package/lib/adapters/messaging-whatsapp/index.js +15 -12
- package/lib/adapters/shopping-taobao/index.js +161 -21
- package/lib/adapters/social-bilibili-adb/api-client.js +555 -0
- package/lib/adapters/social-bilibili-adb/chromium-cookies-reader.js +296 -0
- package/lib/adapters/social-bilibili-adb/collector.js +190 -0
- package/lib/adapters/social-bilibili-adb/cookies-extension.js +250 -0
- package/lib/adapters/social-bilibili-adb/index.js +51 -0
- package/lib/adapters/social-bilibili-adb/snapshot-builder.js +197 -0
- package/lib/adapters/social-douyin/index.js +4 -0
- package/lib/adapters/social-douyin-adb/collector.js +165 -0
- package/lib/adapters/social-douyin-adb/db-extension.js +281 -0
- package/lib/adapters/social-douyin-adb/im-db-parser.js +287 -0
- package/lib/adapters/social-douyin-adb/index.js +57 -0
- package/lib/adapters/social-douyin-adb/snapshot-builder.js +174 -0
- package/lib/adapters/social-weibo-adb/api-client.js +281 -0
- package/lib/adapters/social-weibo-adb/collector.js +169 -0
- package/lib/adapters/social-weibo-adb/cookies-extension.js +251 -0
- package/lib/adapters/social-weibo-adb/index.js +55 -0
- package/lib/adapters/social-weibo-adb/snapshot-builder.js +145 -0
- package/lib/adapters/social-xiaohongshu-adb/api-client.js +278 -0
- package/lib/adapters/social-xiaohongshu-adb/collector.js +158 -0
- package/lib/adapters/social-xiaohongshu-adb/cookies-extension.js +211 -0
- package/lib/adapters/social-xiaohongshu-adb/index.js +50 -0
- package/lib/adapters/social-xiaohongshu-adb/sign.js +90 -0
- package/lib/adapters/social-xiaohongshu-adb/snapshot-builder.js +126 -0
- package/lib/adapters/system-data-android/adapter.js +77 -3
- package/lib/adapters/travel-amap/index.js +16 -10
- package/lib/adapters/travel-ctrip/index.js +25 -9
- package/lib/adapters/vscode/vscode-reader.js +7 -1
- package/lib/sign-providers/index.js +20 -0
- package/lib/sign-providers/interface.js +82 -0
- package/lib/sign-providers/null-sign-provider.js +30 -0
- package/package.json +6 -1
|
@@ -0,0 +1,278 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Phase 3c (Xhs C 路径 — 2026-05-25): Node-side XhsApiClient.
|
|
5
|
+
*
|
|
6
|
+
* Byte-parity port of XhsApiClient.kt 4 endpoints. **Best-effort X-S
|
|
7
|
+
* signing** (~60% GET / <30% POST hit rate) — collector tolerates
|
|
8
|
+
* partial failures.
|
|
9
|
+
*
|
|
10
|
+
* Endpoints:
|
|
11
|
+
* - `/api/sns/web/v1/user/me` — no X-S, cookies-only
|
|
12
|
+
* - `/api/sns/web/v2/user_posted` — needs X-S
|
|
13
|
+
* - `/api/sns/web/v1/note/like/page` — needs X-S
|
|
14
|
+
* - `/api/sns/web/v1/user/follow/list` — needs X-S
|
|
15
|
+
*
|
|
16
|
+
* **Anti-bot signal**: User-Agent must look like desktop Chrome (xhs
|
|
17
|
+
* web is desktop-tuned, NOT mobile like Bilibili/Weibo). Referer +
|
|
18
|
+
* Origin = `https://www.xiaohongshu.com/`.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
const { computeXsXt } = require("./sign");
|
|
22
|
+
|
|
23
|
+
const DEFAULT_BASE_URL = "https://edith.xiaohongshu.com/";
|
|
24
|
+
|
|
25
|
+
const BROWSER_UA =
|
|
26
|
+
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " +
|
|
27
|
+
"(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36";
|
|
28
|
+
|
|
29
|
+
const BROWSER_HEADERS = Object.freeze({
|
|
30
|
+
"User-Agent": BROWSER_UA,
|
|
31
|
+
Referer: "https://www.xiaohongshu.com/",
|
|
32
|
+
Origin: "https://www.xiaohongshu.com",
|
|
33
|
+
Accept: "application/json, text/plain, */*",
|
|
34
|
+
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Parse xhs's interact_info count strings: "1.2万" / "10w+" / "234" / "亿".
|
|
39
|
+
* Mirrors XhsApiClient.kt:parseCount.
|
|
40
|
+
*/
|
|
41
|
+
function parseCount(raw) {
|
|
42
|
+
if (typeof raw !== "string" || raw.length === 0) return 0;
|
|
43
|
+
const trimmed = raw.trim();
|
|
44
|
+
if (trimmed.endsWith("万")) {
|
|
45
|
+
const n = parseFloat(trimmed.slice(0, -1));
|
|
46
|
+
return Number.isFinite(n) ? Math.floor(n * 10000) : 0;
|
|
47
|
+
}
|
|
48
|
+
if (trimmed.endsWith("w+") || trimmed.endsWith("W+")) {
|
|
49
|
+
const n = parseFloat(trimmed.slice(0, -2));
|
|
50
|
+
return Number.isFinite(n) ? Math.floor(n * 10000) : 0;
|
|
51
|
+
}
|
|
52
|
+
if (trimmed.endsWith("w") || trimmed.endsWith("W")) {
|
|
53
|
+
const n = parseFloat(trimmed.slice(0, -1));
|
|
54
|
+
return Number.isFinite(n) ? Math.floor(n * 10000) : 0;
|
|
55
|
+
}
|
|
56
|
+
if (trimmed.endsWith("亿")) {
|
|
57
|
+
const n = parseFloat(trimmed.slice(0, -1));
|
|
58
|
+
return Number.isFinite(n) ? Math.floor(n * 100_000_000) : 0;
|
|
59
|
+
}
|
|
60
|
+
const n = parseInt(trimmed, 10);
|
|
61
|
+
return Number.isFinite(n) ? n : 0;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Normalize a Xhs timestamp to milliseconds (seconds → ms when < 1e12).
|
|
66
|
+
*/
|
|
67
|
+
function normalizeMs(v) {
|
|
68
|
+
if (typeof v !== "number" || !Number.isFinite(v) || v <= 0) return 0;
|
|
69
|
+
return v > 1e12 ? v : v * 1000;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
class XhsApiClient {
|
|
73
|
+
constructor(opts = {}) {
|
|
74
|
+
this.baseUrl = opts.baseUrl || DEFAULT_BASE_URL;
|
|
75
|
+
if (!this.baseUrl.endsWith("/")) this.baseUrl += "/";
|
|
76
|
+
this._fetch = opts.fetch || globalThis.fetch;
|
|
77
|
+
if (typeof this._fetch !== "function") {
|
|
78
|
+
throw new Error(
|
|
79
|
+
"XhsApiClient: fetch not available — pass opts.fetch or run on Node 18+",
|
|
80
|
+
);
|
|
81
|
+
}
|
|
82
|
+
this._now = opts.now || Date.now;
|
|
83
|
+
this.lastErrorCode = 0;
|
|
84
|
+
this.lastErrorMessage = null;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
async _doGetJson(url, cookie, a1, requireSign) {
|
|
88
|
+
const headers = { ...BROWSER_HEADERS, Cookie: cookie };
|
|
89
|
+
if (requireSign && a1) {
|
|
90
|
+
const pathWithQuery = url.pathname + url.search;
|
|
91
|
+
const { xs, xt } = computeXsXt(pathWithQuery, null, a1, {
|
|
92
|
+
now: this._now,
|
|
93
|
+
});
|
|
94
|
+
headers["X-S"] = xs;
|
|
95
|
+
headers["X-T"] = xt;
|
|
96
|
+
}
|
|
97
|
+
try {
|
|
98
|
+
const resp = await this._fetch(url.toString(), {
|
|
99
|
+
method: "GET",
|
|
100
|
+
headers,
|
|
101
|
+
});
|
|
102
|
+
const body = await resp.text();
|
|
103
|
+
if (!resp.ok) {
|
|
104
|
+
this._setLastError(resp.status, `HTTP ${resp.status}`);
|
|
105
|
+
return null;
|
|
106
|
+
}
|
|
107
|
+
const trimmed = body.trimStart();
|
|
108
|
+
if (!trimmed.startsWith("{")) {
|
|
109
|
+
this._setLastError(-4, "non-json (login redirect / anti-bot HTML)");
|
|
110
|
+
return null;
|
|
111
|
+
}
|
|
112
|
+
let obj;
|
|
113
|
+
try {
|
|
114
|
+
obj = JSON.parse(body);
|
|
115
|
+
} catch (e) {
|
|
116
|
+
this._setLastError(-3, "parse: " + (e.message || String(e)));
|
|
117
|
+
return null;
|
|
118
|
+
}
|
|
119
|
+
// xhs returns either {code: N, msg:..., data:...} or {success:bool, code:N, data:...}
|
|
120
|
+
const success = obj.success === undefined ? true : obj.success;
|
|
121
|
+
if (success === false) {
|
|
122
|
+
this._setLastError(-5, "/success=false (no code)");
|
|
123
|
+
return null;
|
|
124
|
+
}
|
|
125
|
+
const code = typeof obj.code === "number" ? obj.code : 0;
|
|
126
|
+
if (code !== 0) {
|
|
127
|
+
this._setLastError(code, (obj.msg || "").toString());
|
|
128
|
+
return null;
|
|
129
|
+
}
|
|
130
|
+
this._clearLastError();
|
|
131
|
+
return obj;
|
|
132
|
+
} catch (e) {
|
|
133
|
+
this._setLastError(-2, "IO: " + (e.message || String(e)));
|
|
134
|
+
return null;
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
_setLastError(code, message) {
|
|
139
|
+
this.lastErrorCode = code;
|
|
140
|
+
this.lastErrorMessage = message;
|
|
141
|
+
}
|
|
142
|
+
_clearLastError() {
|
|
143
|
+
this.lastErrorCode = 0;
|
|
144
|
+
this.lastErrorMessage = null;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* Fetch /api/sns/web/v1/user/me — cookies-only, no X-S required.
|
|
149
|
+
* Returns `{userId, nickname}` or null on failure.
|
|
150
|
+
*/
|
|
151
|
+
async fetchMe(cookie) {
|
|
152
|
+
const url = new URL("api/sns/web/v1/user/me", this.baseUrl);
|
|
153
|
+
const obj = await this._doGetJson(url, cookie, null, false);
|
|
154
|
+
if (!obj) return null;
|
|
155
|
+
const data = obj.data || {};
|
|
156
|
+
const userId = (data.user_id && String(data.user_id)) || null;
|
|
157
|
+
if (!userId) {
|
|
158
|
+
this._setLastError(
|
|
159
|
+
-7,
|
|
160
|
+
"/user/me ok but user_id blank (cookie likely missing web_session)",
|
|
161
|
+
);
|
|
162
|
+
return null;
|
|
163
|
+
}
|
|
164
|
+
return {
|
|
165
|
+
userId,
|
|
166
|
+
nickname: data.nickname || null,
|
|
167
|
+
};
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* Fetch user's posted notes. Requires X-S signing.
|
|
172
|
+
*/
|
|
173
|
+
async fetchNotes(cookie, a1, userId, opts = {}) {
|
|
174
|
+
const limit = Number.isInteger(opts.limit) && opts.limit > 0 ? opts.limit : 30;
|
|
175
|
+
const url = new URL("api/sns/web/v2/user_posted", this.baseUrl);
|
|
176
|
+
url.searchParams.set("user_id", userId);
|
|
177
|
+
url.searchParams.set("num", "30");
|
|
178
|
+
url.searchParams.set("cursor", "");
|
|
179
|
+
url.searchParams.set("image_formats", "jpg,webp,avif");
|
|
180
|
+
const obj = await this._doGetJson(url, cookie, a1, true);
|
|
181
|
+
if (!obj) return [];
|
|
182
|
+
const data = obj.data || {};
|
|
183
|
+
const notes = Array.isArray(data.notes) ? data.notes : [];
|
|
184
|
+
const out = [];
|
|
185
|
+
for (let i = 0; i < Math.min(limit, notes.length); i++) {
|
|
186
|
+
const n = notes[i];
|
|
187
|
+
if (!n) continue;
|
|
188
|
+
const noteId =
|
|
189
|
+
(n.note_id && String(n.note_id)) || (n.id && String(n.id));
|
|
190
|
+
if (!noteId) continue;
|
|
191
|
+
const interact = n.interact_info || {};
|
|
192
|
+
out.push({
|
|
193
|
+
noteId,
|
|
194
|
+
title:
|
|
195
|
+
n.display_title ||
|
|
196
|
+
n.title ||
|
|
197
|
+
"(no title)",
|
|
198
|
+
desc: n.desc || null,
|
|
199
|
+
type: n.type || "normal",
|
|
200
|
+
createdAt: normalizeMs(typeof n.time === "number" ? n.time : 0),
|
|
201
|
+
likedCount: parseCount(interact.liked_count),
|
|
202
|
+
collectedCount: parseCount(interact.collected_count),
|
|
203
|
+
commentCount: parseCount(interact.comment_count),
|
|
204
|
+
});
|
|
205
|
+
}
|
|
206
|
+
return out;
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/**
|
|
210
|
+
* Fetch user's liked notes. Requires X-S.
|
|
211
|
+
*/
|
|
212
|
+
async fetchLiked(cookie, a1, opts = {}) {
|
|
213
|
+
const limit = Number.isInteger(opts.limit) && opts.limit > 0 ? opts.limit : 30;
|
|
214
|
+
const url = new URL("api/sns/web/v1/note/like/page", this.baseUrl);
|
|
215
|
+
url.searchParams.set("num", "20");
|
|
216
|
+
url.searchParams.set("cursor", "");
|
|
217
|
+
const obj = await this._doGetJson(url, cookie, a1, true);
|
|
218
|
+
if (!obj) return [];
|
|
219
|
+
const data = obj.data || {};
|
|
220
|
+
const notes = Array.isArray(data.notes) ? data.notes : [];
|
|
221
|
+
const out = [];
|
|
222
|
+
for (let i = 0; i < Math.min(limit, notes.length); i++) {
|
|
223
|
+
const n = notes[i];
|
|
224
|
+
if (!n) continue;
|
|
225
|
+
const noteId = n.note_id && String(n.note_id);
|
|
226
|
+
if (!noteId) continue;
|
|
227
|
+
const user = n.user || {};
|
|
228
|
+
out.push({
|
|
229
|
+
noteId,
|
|
230
|
+
title: n.display_title || n.title || "(no title)",
|
|
231
|
+
// xhs doesn't return explicit liked_at — collector fills with snapshotted_at
|
|
232
|
+
likedAt: 0,
|
|
233
|
+
authorNickname: user.nickname || null,
|
|
234
|
+
});
|
|
235
|
+
}
|
|
236
|
+
return out;
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
/**
|
|
240
|
+
* Fetch follow list. Requires X-S.
|
|
241
|
+
*/
|
|
242
|
+
async fetchFollows(cookie, a1, userId, opts = {}) {
|
|
243
|
+
const limit = Number.isInteger(opts.limit) && opts.limit > 0 ? opts.limit : 100;
|
|
244
|
+
const url = new URL("api/sns/web/v1/user/follow/list", this.baseUrl);
|
|
245
|
+
url.searchParams.set("user_id", userId);
|
|
246
|
+
url.searchParams.set("num", "20");
|
|
247
|
+
url.searchParams.set("cursor", "");
|
|
248
|
+
const obj = await this._doGetJson(url, cookie, a1, true);
|
|
249
|
+
if (!obj) return [];
|
|
250
|
+
const data = obj.data || {};
|
|
251
|
+
const users = Array.isArray(data.users) ? data.users : [];
|
|
252
|
+
const out = [];
|
|
253
|
+
for (let i = 0; i < Math.min(limit, users.length); i++) {
|
|
254
|
+
const u = users[i];
|
|
255
|
+
if (!u) continue;
|
|
256
|
+
const userIdStr = u.user_id && String(u.user_id);
|
|
257
|
+
if (!userIdStr) continue;
|
|
258
|
+
out.push({
|
|
259
|
+
userId: userIdStr,
|
|
260
|
+
nickname: u.nickname || "(unnamed)",
|
|
261
|
+
image: u.image || null,
|
|
262
|
+
// xhs doesn't return explicit follow time
|
|
263
|
+
followedAt: 0,
|
|
264
|
+
});
|
|
265
|
+
}
|
|
266
|
+
return out;
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
module.exports = {
|
|
271
|
+
XhsApiClient,
|
|
272
|
+
_internals: {
|
|
273
|
+
parseCount,
|
|
274
|
+
normalizeMs,
|
|
275
|
+
BROWSER_UA,
|
|
276
|
+
BROWSER_HEADERS,
|
|
277
|
+
},
|
|
278
|
+
};
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Phase 3c (Xhs C 路径 — 2026-05-25): end-to-end orchestrator.
|
|
5
|
+
*
|
|
6
|
+
* bridge.invoke("xhs.cookies") ← Phase 3c cookies extension
|
|
7
|
+
* │
|
|
8
|
+
* ▼ {cookie, a1, diagnostic}
|
|
9
|
+
* XhsApiClient.fetchMe ← /user/me 拿 user_id (无 X-S)
|
|
10
|
+
* │
|
|
11
|
+
* ▼ {userId, nickname}
|
|
12
|
+
* fetchNotes + fetchLiked + fetchFollows (parallel, X-S 需 a1)
|
|
13
|
+
* │
|
|
14
|
+
* ▼ 3 arrays (partial-failure OK; ~60% GET hit rate)
|
|
15
|
+
* buildSnapshot + writeSnapshotJson ← schemaVersion=1
|
|
16
|
+
* │
|
|
17
|
+
* ▼
|
|
18
|
+
* registry.syncAdapter("social-xiaohongshu", { inputPath })
|
|
19
|
+
*
|
|
20
|
+
* Mirror of social-weibo-adb/collector.js. Key diff: 3 endpoints need
|
|
21
|
+
* X-S signing (best-effort md5 approximation hits ~60% GET, <30% POST;
|
|
22
|
+
* collector tolerates partial failures via lastErrorCode propagation).
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
const { XhsApiClient } = require("./api-client");
|
|
26
|
+
const {
|
|
27
|
+
buildSnapshot,
|
|
28
|
+
writeSnapshotJson,
|
|
29
|
+
cleanupSnapshotJson,
|
|
30
|
+
} = require("./snapshot-builder");
|
|
31
|
+
|
|
32
|
+
async function collect(bridge, opts = {}) {
|
|
33
|
+
if (!bridge || typeof bridge.invoke !== "function") {
|
|
34
|
+
throw new TypeError(
|
|
35
|
+
"XhsAdbCollector.collect: bridge must expose invoke(method, params)",
|
|
36
|
+
);
|
|
37
|
+
}
|
|
38
|
+
const now = opts.now || Date.now;
|
|
39
|
+
const client = opts.apiClient || new XhsApiClient({ now });
|
|
40
|
+
const limits = opts.limits || {};
|
|
41
|
+
|
|
42
|
+
const cookieResult = await bridge.invoke("xhs.cookies");
|
|
43
|
+
if (
|
|
44
|
+
!cookieResult ||
|
|
45
|
+
typeof cookieResult.cookie !== "string" ||
|
|
46
|
+
typeof cookieResult.a1 !== "string"
|
|
47
|
+
) {
|
|
48
|
+
throw new Error(
|
|
49
|
+
"XhsAdbCollector.collect: bridge.invoke('xhs.cookies') returned malformed payload — got cookie=" +
|
|
50
|
+
typeof cookieResult?.cookie +
|
|
51
|
+
" a1=" +
|
|
52
|
+
typeof cookieResult?.a1,
|
|
53
|
+
);
|
|
54
|
+
}
|
|
55
|
+
const { cookie, a1, diagnostic: cookieDiagnostic } = cookieResult;
|
|
56
|
+
|
|
57
|
+
// fetchMe — no X-S required
|
|
58
|
+
const me = await client.fetchMe(cookie);
|
|
59
|
+
if (!me) {
|
|
60
|
+
// Cookie expired or web_session missing — write empty snapshot
|
|
61
|
+
// (build requires userId, use sentinel "0" + emit 0 events).
|
|
62
|
+
const snapshot = buildSnapshot({
|
|
63
|
+
userId: "unknown-user",
|
|
64
|
+
nickname: opts.displayName,
|
|
65
|
+
snapshottedAt: now(),
|
|
66
|
+
});
|
|
67
|
+
const snapshotPath = writeSnapshotJson(snapshot, { dir: opts.stagingDir });
|
|
68
|
+
return {
|
|
69
|
+
snapshotPath,
|
|
70
|
+
userId: null,
|
|
71
|
+
nickname: null,
|
|
72
|
+
eventCounts: { note: 0, liked: 0, follow: 0, total: 0 },
|
|
73
|
+
lastErrorCode: client.lastErrorCode,
|
|
74
|
+
lastErrorMessage: client.lastErrorMessage,
|
|
75
|
+
cookieDiagnostic: cookieDiagnostic || null,
|
|
76
|
+
meFetchFailed: true,
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
// Parallel 3 endpoints — partial failure tolerated (~60% X-S hit rate)
|
|
81
|
+
const [notes, liked, follows] = await Promise.all([
|
|
82
|
+
client.fetchNotes(cookie, a1, me.userId, {
|
|
83
|
+
limit: Number.isInteger(limits.note) ? limits.note : undefined,
|
|
84
|
+
}),
|
|
85
|
+
client.fetchLiked(cookie, a1, {
|
|
86
|
+
limit: Number.isInteger(limits.liked) ? limits.liked : undefined,
|
|
87
|
+
}),
|
|
88
|
+
client.fetchFollows(cookie, a1, me.userId, {
|
|
89
|
+
limit: Number.isInteger(limits.follow) ? limits.follow : undefined,
|
|
90
|
+
}),
|
|
91
|
+
]);
|
|
92
|
+
|
|
93
|
+
const snapshot = buildSnapshot({
|
|
94
|
+
userId: me.userId,
|
|
95
|
+
nickname: opts.displayName || me.nickname,
|
|
96
|
+
notes,
|
|
97
|
+
liked,
|
|
98
|
+
follows,
|
|
99
|
+
snapshottedAt: now(),
|
|
100
|
+
});
|
|
101
|
+
const snapshotPath = writeSnapshotJson(snapshot, { dir: opts.stagingDir });
|
|
102
|
+
|
|
103
|
+
return {
|
|
104
|
+
snapshotPath,
|
|
105
|
+
userId: me.userId,
|
|
106
|
+
nickname: me.nickname,
|
|
107
|
+
eventCounts: {
|
|
108
|
+
note: notes.length,
|
|
109
|
+
liked: liked.length,
|
|
110
|
+
follow: follows.length,
|
|
111
|
+
total: snapshot.events.length,
|
|
112
|
+
},
|
|
113
|
+
lastErrorCode: client.lastErrorCode,
|
|
114
|
+
lastErrorMessage: client.lastErrorMessage,
|
|
115
|
+
cookieDiagnostic: cookieDiagnostic || null,
|
|
116
|
+
meFetchFailed: false,
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
async function collectAndSync(bridge, registry, opts = {}) {
|
|
121
|
+
if (!registry || typeof registry.syncAdapter !== "function") {
|
|
122
|
+
throw new TypeError(
|
|
123
|
+
"XhsAdbCollector.collectAndSync: registry must expose syncAdapter(name, options)",
|
|
124
|
+
);
|
|
125
|
+
}
|
|
126
|
+
const collectResult = await collect(bridge, opts);
|
|
127
|
+
let syncReport = null;
|
|
128
|
+
let cleanupFailed = false;
|
|
129
|
+
try {
|
|
130
|
+
syncReport = await registry.syncAdapter("social-xiaohongshu", {
|
|
131
|
+
inputPath: collectResult.snapshotPath,
|
|
132
|
+
});
|
|
133
|
+
} finally {
|
|
134
|
+
try {
|
|
135
|
+
cleanupSnapshotJson(collectResult.snapshotPath);
|
|
136
|
+
} catch (_e) {
|
|
137
|
+
cleanupFailed = true;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
return {
|
|
141
|
+
...syncReport,
|
|
142
|
+
xhs: {
|
|
143
|
+
userId: collectResult.userId,
|
|
144
|
+
nickname: collectResult.nickname,
|
|
145
|
+
eventCounts: collectResult.eventCounts,
|
|
146
|
+
lastErrorCode: collectResult.lastErrorCode,
|
|
147
|
+
lastErrorMessage: collectResult.lastErrorMessage,
|
|
148
|
+
cookieDiagnostic: collectResult.cookieDiagnostic,
|
|
149
|
+
meFetchFailed: collectResult.meFetchFailed,
|
|
150
|
+
cleanupFailed,
|
|
151
|
+
},
|
|
152
|
+
};
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
module.exports = {
|
|
156
|
+
collect,
|
|
157
|
+
collectAndSync,
|
|
158
|
+
};
|
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Phase 3c (Xhs C 路径 — 2026-05-25): xhs.cookies ADB extension factory.
|
|
5
|
+
*
|
|
6
|
+
* Mirror of `social-weibo-adb/cookies-extension.js` (P3a) but for
|
|
7
|
+
* com.xingin.xhs (Xiaohongshu) — same chromium-cookies-reader reuse,
|
|
8
|
+
* different package + cookie name requirements.
|
|
9
|
+
*
|
|
10
|
+
* Required cookies (without these, X-S signing or auth fails):
|
|
11
|
+
* - `a1` — anti-bot fingerprint, REQUIRED for X-S sig input
|
|
12
|
+
* - `web_session` — login session token
|
|
13
|
+
*
|
|
14
|
+
* Either of the two missing → WEIBO_COOKIES_INCOMPLETE-style error code
|
|
15
|
+
* (XHS_COOKIES_INCOMPLETE) so UI surfaces a "relog on phone" banner.
|
|
16
|
+
*
|
|
17
|
+
* Returns:
|
|
18
|
+
* {
|
|
19
|
+
* cookie: string, // full Cookie header value
|
|
20
|
+
* a1: string, // pre-extracted a1 (saves caller parsing)
|
|
21
|
+
* extractedAt: number,
|
|
22
|
+
* diagnostic: {
|
|
23
|
+
* cookieCount: number,
|
|
24
|
+
* hadEncrypted: boolean,
|
|
25
|
+
* cookieNames: string[],
|
|
26
|
+
* }
|
|
27
|
+
* }
|
|
28
|
+
*/
|
|
29
|
+
|
|
30
|
+
const fs = require("node:fs");
|
|
31
|
+
const path = require("node:path");
|
|
32
|
+
const os = require("node:os");
|
|
33
|
+
const crypto = require("node:crypto");
|
|
34
|
+
|
|
35
|
+
const {
|
|
36
|
+
readChromiumCookies,
|
|
37
|
+
} = require("../social-bilibili-adb/chromium-cookies-reader");
|
|
38
|
+
|
|
39
|
+
const XHS_COOKIES_REMOTE_PATH =
|
|
40
|
+
"/data/data/com.xingin.xhs/app_webview/Default/Cookies";
|
|
41
|
+
|
|
42
|
+
const XHS_COOKIE_HOST_DOMAIN = "xiaohongshu.com";
|
|
43
|
+
|
|
44
|
+
const XHS_REQUIRED_COOKIES = Object.freeze(["a1", "web_session"]);
|
|
45
|
+
|
|
46
|
+
async function pullCookiesViaSu(adb, serial, opts) {
|
|
47
|
+
const adbOpts = { serial, timeoutMs: opts?.timeoutMs || 60_000 };
|
|
48
|
+
const lsOut = await adb(
|
|
49
|
+
[
|
|
50
|
+
"shell",
|
|
51
|
+
"su",
|
|
52
|
+
"-c",
|
|
53
|
+
`ls ${XHS_COOKIES_REMOTE_PATH} 2>/dev/null || echo NOT_FOUND`,
|
|
54
|
+
],
|
|
55
|
+
adbOpts,
|
|
56
|
+
);
|
|
57
|
+
const lsLine = lsOut.replace(/\r+$/gm, "").trim();
|
|
58
|
+
if (lsLine === "NOT_FOUND" || lsLine === "") {
|
|
59
|
+
throw new Error(
|
|
60
|
+
"XHS_NOT_INSTALLED: " +
|
|
61
|
+
XHS_COOKIES_REMOTE_PATH +
|
|
62
|
+
" not found. Install Xiaohongshu App + log in once on the phone, then retry.",
|
|
63
|
+
);
|
|
64
|
+
}
|
|
65
|
+
const idOut = await adb(["shell", "su", "-c", "id -u"], adbOpts);
|
|
66
|
+
const idLine = idOut.replace(/\r+$/gm, "").trim();
|
|
67
|
+
if (idLine !== "0" && !idLine.includes("uid=0")) {
|
|
68
|
+
throw new Error(
|
|
69
|
+
"XHS_NO_ROOT: this phone isn't rooted (su returned `" +
|
|
70
|
+
idLine.substring(0, 60) +
|
|
71
|
+
"`). Xiaohongshu release APK isn't debuggable, so root is required.",
|
|
72
|
+
);
|
|
73
|
+
}
|
|
74
|
+
const b64 = await adb(
|
|
75
|
+
[
|
|
76
|
+
"shell",
|
|
77
|
+
"su",
|
|
78
|
+
"-c",
|
|
79
|
+
`base64 ${XHS_COOKIES_REMOTE_PATH} | tr -d '\\n\\r'`,
|
|
80
|
+
],
|
|
81
|
+
{ ...adbOpts, timeoutMs: opts?.timeoutMs || 60_000 },
|
|
82
|
+
);
|
|
83
|
+
const b64Clean = b64.replace(/[\r\n\t ]+/g, "");
|
|
84
|
+
if (b64Clean.length === 0) {
|
|
85
|
+
throw new Error(
|
|
86
|
+
"XHS_COOKIES_EMPTY: base64 stream returned 0 bytes (su exec may have silently failed on MIUI / OEM ROM)",
|
|
87
|
+
);
|
|
88
|
+
}
|
|
89
|
+
let buf;
|
|
90
|
+
try {
|
|
91
|
+
buf = Buffer.from(b64Clean, "base64");
|
|
92
|
+
} catch (e) {
|
|
93
|
+
throw new Error(
|
|
94
|
+
"XHS_BASE64_PARSE: stream wasn't valid base64 (" +
|
|
95
|
+
(e.message || String(e)) +
|
|
96
|
+
")",
|
|
97
|
+
);
|
|
98
|
+
}
|
|
99
|
+
if (buf.length < 1024) {
|
|
100
|
+
throw new Error(
|
|
101
|
+
"XHS_COOKIES_TRUNCATED: decoded file is only " +
|
|
102
|
+
buf.length +
|
|
103
|
+
" bytes — expected ≥4KB sqlite",
|
|
104
|
+
);
|
|
105
|
+
}
|
|
106
|
+
const magic = buf.subarray(0, 16).toString("latin1");
|
|
107
|
+
if (!magic.startsWith("SQLite format 3")) {
|
|
108
|
+
throw new Error(
|
|
109
|
+
"XHS_NOT_SQLITE: decoded file lacks `SQLite format 3` magic header",
|
|
110
|
+
);
|
|
111
|
+
}
|
|
112
|
+
const tmpDir = os.tmpdir();
|
|
113
|
+
const tmpFile = path.join(tmpDir, `cc-xhs-cookies-${crypto.randomUUID()}.db`);
|
|
114
|
+
fs.writeFileSync(tmpFile, buf);
|
|
115
|
+
return tmpFile;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* Build a Cookie header + extract a1 from a chromium-cookies array.
|
|
120
|
+
*
|
|
121
|
+
* Xhs requires BOTH a1 and web_session — without either, X-S signing
|
|
122
|
+
* fails (a1) or auth fails (web_session).
|
|
123
|
+
*/
|
|
124
|
+
function assembleXhsCookieHeader(cookies) {
|
|
125
|
+
if (!Array.isArray(cookies)) {
|
|
126
|
+
throw new TypeError("assembleXhsCookieHeader: cookies must be an array");
|
|
127
|
+
}
|
|
128
|
+
const byName = new Map();
|
|
129
|
+
for (const c of cookies) {
|
|
130
|
+
if (
|
|
131
|
+
!byName.has(c.name) ||
|
|
132
|
+
c.hostKey.length > (byName.get(c.name).hostKey || "").length
|
|
133
|
+
) {
|
|
134
|
+
byName.set(c.name, c);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
const missing = XHS_REQUIRED_COOKIES.filter((n) => !byName.has(n));
|
|
138
|
+
const present = new Set(byName.keys());
|
|
139
|
+
if (missing.length > 0) {
|
|
140
|
+
return { header: null, a1: null, present, missing };
|
|
141
|
+
}
|
|
142
|
+
const header = Array.from(byName.values())
|
|
143
|
+
.map((c) => `${c.name}=${c.value}`)
|
|
144
|
+
.join("; ");
|
|
145
|
+
const a1 = byName.get("a1")?.value || null;
|
|
146
|
+
return { header, a1, present, missing: [] };
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
function createXhsCookiesExtension(factoryOpts = {}) {
|
|
150
|
+
const timeoutMs = factoryOpts.timeoutMs || 60_000;
|
|
151
|
+
const onCleanupFailed = factoryOpts.onCleanupFailed || (() => {});
|
|
152
|
+
|
|
153
|
+
return async function xhsCookiesHandler(_params, ctx) {
|
|
154
|
+
if (
|
|
155
|
+
!ctx ||
|
|
156
|
+
typeof ctx.adb !== "function" ||
|
|
157
|
+
typeof ctx.pickDevice !== "function"
|
|
158
|
+
) {
|
|
159
|
+
throw new TypeError(
|
|
160
|
+
"xhs.cookies extension: ctx must provide {adb, pickDevice}",
|
|
161
|
+
);
|
|
162
|
+
}
|
|
163
|
+
const serial = await ctx.pickDevice();
|
|
164
|
+
let tmpFile = null;
|
|
165
|
+
try {
|
|
166
|
+
tmpFile = await pullCookiesViaSu(ctx.adb, serial, { timeoutMs });
|
|
167
|
+
const cookies = readChromiumCookies(tmpFile, XHS_COOKIE_HOST_DOMAIN);
|
|
168
|
+
const cookieCount = cookies.length;
|
|
169
|
+
const hadEncrypted = (cookies._skippedEncryptedCount || 0) > 0;
|
|
170
|
+
const { header, a1, missing, present } = assembleXhsCookieHeader(cookies);
|
|
171
|
+
if (header === null) {
|
|
172
|
+
throw new Error(
|
|
173
|
+
"XHS_COOKIES_INCOMPLETE: missing required cookies " +
|
|
174
|
+
JSON.stringify(missing) +
|
|
175
|
+
". User probably logged out (relog on phone) or Xhs version uses non-default WebView storage path (hadEncrypted=" +
|
|
176
|
+
hadEncrypted +
|
|
177
|
+
").",
|
|
178
|
+
);
|
|
179
|
+
}
|
|
180
|
+
return {
|
|
181
|
+
cookie: header,
|
|
182
|
+
a1,
|
|
183
|
+
extractedAt: Date.now(),
|
|
184
|
+
diagnostic: {
|
|
185
|
+
cookieCount,
|
|
186
|
+
hadEncrypted,
|
|
187
|
+
cookieNames: Array.from(present),
|
|
188
|
+
},
|
|
189
|
+
};
|
|
190
|
+
} finally {
|
|
191
|
+
if (tmpFile) {
|
|
192
|
+
try {
|
|
193
|
+
fs.unlinkSync(tmpFile);
|
|
194
|
+
} catch (_e) {
|
|
195
|
+
onCleanupFailed(tmpFile);
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
};
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
module.exports = {
|
|
203
|
+
createXhsCookiesExtension,
|
|
204
|
+
XHS_COOKIES_REMOTE_PATH,
|
|
205
|
+
XHS_COOKIE_HOST_DOMAIN,
|
|
206
|
+
XHS_REQUIRED_COOKIES,
|
|
207
|
+
assembleXhsCookieHeader,
|
|
208
|
+
_internals: {
|
|
209
|
+
pullCookiesViaSu,
|
|
210
|
+
},
|
|
211
|
+
};
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* social-xiaohongshu-adb — Phase 3c (Xhs C 路径) entry.
|
|
5
|
+
*
|
|
6
|
+
* Phase 3c — desktop ADB Chromium cookies + edith.xiaohongshu.com HTTP
|
|
7
|
+
* with best-effort X-S signing (md5 approximation, ~60% GET hit rate).
|
|
8
|
+
*
|
|
9
|
+
* Pipeline:
|
|
10
|
+
* bridge.invoke("xhs.cookies") → {cookie, a1}
|
|
11
|
+
* → XhsApiClient.fetchMe (no X-S, cookies-only)
|
|
12
|
+
* → fetchNotes + fetchLiked + fetchFollows (X-S signed, parallel)
|
|
13
|
+
* → buildSnapshot + writeSnapshotJson
|
|
14
|
+
* → registry.syncAdapter("social-xiaohongshu", { inputPath })
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
const {
|
|
18
|
+
createXhsCookiesExtension,
|
|
19
|
+
XHS_COOKIES_REMOTE_PATH,
|
|
20
|
+
XHS_COOKIE_HOST_DOMAIN,
|
|
21
|
+
XHS_REQUIRED_COOKIES,
|
|
22
|
+
assembleXhsCookieHeader,
|
|
23
|
+
} = require("./cookies-extension");
|
|
24
|
+
const { computeXsXt, extractA1, XS_PREFIX } = require("./sign");
|
|
25
|
+
const { XhsApiClient } = require("./api-client");
|
|
26
|
+
const {
|
|
27
|
+
buildSnapshot,
|
|
28
|
+
writeSnapshotJson,
|
|
29
|
+
cleanupSnapshotJson,
|
|
30
|
+
SNAPSHOT_SCHEMA_VERSION,
|
|
31
|
+
} = require("./snapshot-builder");
|
|
32
|
+
const { collect, collectAndSync } = require("./collector");
|
|
33
|
+
|
|
34
|
+
module.exports = {
|
|
35
|
+
createXhsCookiesExtension,
|
|
36
|
+
XHS_COOKIES_REMOTE_PATH,
|
|
37
|
+
XHS_COOKIE_HOST_DOMAIN,
|
|
38
|
+
XHS_REQUIRED_COOKIES,
|
|
39
|
+
assembleXhsCookieHeader,
|
|
40
|
+
computeXsXt,
|
|
41
|
+
extractA1,
|
|
42
|
+
XS_PREFIX,
|
|
43
|
+
XhsApiClient,
|
|
44
|
+
buildSnapshot,
|
|
45
|
+
writeSnapshotJson,
|
|
46
|
+
cleanupSnapshotJson,
|
|
47
|
+
SNAPSHOT_SCHEMA_VERSION,
|
|
48
|
+
collect,
|
|
49
|
+
collectAndSync,
|
|
50
|
+
};
|