@chainlesschain/personal-data-hub 0.2.2 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/__tests__/adapters/social-toutiao-kuaishou-scaffold.test.js +58 -16
- package/__tests__/longtail-adapters.test.js +60 -14
- package/__tests__/messaging-qq-snapshot.test.js +294 -0
- package/__tests__/shopping-pinduoduo-snapshot.test.js +302 -0
- package/__tests__/shopping-snapshot.test.js +438 -0
- package/__tests__/social-adapters.test.js +28 -3
- package/__tests__/social-douyin-snapshot.test.js +253 -0
- package/__tests__/social-kuaishou-snapshot.test.js +309 -0
- package/__tests__/social-toutiao-snapshot.test.js +314 -0
- package/__tests__/social-weibo-snapshot.test.js +234 -0
- package/__tests__/social-xiaohongshu-snapshot.test.js +232 -0
- package/__tests__/travel-maps-snapshot.test.js +426 -0
- package/__tests__/vault-driver-error.test.js +74 -0
- package/lib/adapters/messaging-qq/index.js +498 -92
- package/lib/adapters/shopping-jd/index.js +228 -25
- package/lib/adapters/shopping-meituan/index.js +222 -26
- package/lib/adapters/shopping-pinduoduo/index.js +275 -0
- package/lib/adapters/social-douyin/index.js +454 -63
- package/lib/adapters/social-kuaishou/index.js +379 -127
- package/lib/adapters/social-toutiao/index.js +400 -130
- package/lib/adapters/social-weibo/index.js +393 -95
- package/lib/adapters/social-xiaohongshu/index.js +389 -49
- package/lib/adapters/travel-baidu-map/index.js +286 -26
- package/lib/adapters/travel-tencent-map/index.js +414 -0
- package/lib/index.js +5 -1
- package/lib/vault.js +60 -8
- package/package.json +2 -1
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* §2.4c 购物三联 v0.2 — Pinduoduo (拼多多) adapter, snapshot-only.
|
|
3
|
+
*
|
|
4
|
+
* Mirror of shopping-jd / shopping-meituan snapshot-mode pattern, **but
|
|
5
|
+
* without a cookie-mode fallback** because:
|
|
6
|
+
*
|
|
7
|
+
* 1. mobile.yangkeduo.com web endpoint `/proxy/api/galerie/transaction/
|
|
8
|
+
* transaction_list` requires `anti_token` signing computed by client-side
|
|
9
|
+
* JS (similar to 抖音 X-Bogus). No pure-Node implementation survives
|
|
10
|
+
* pinduoduo's monthly anti_token rotation.
|
|
11
|
+
* 2. Pinduoduo Android app has no built-in "export orders" feature, so
|
|
12
|
+
* there's no SAF source-format to parse directly either.
|
|
13
|
+
*
|
|
14
|
+
* v0.2 deliverable = **scaffold + snapshot-mode JSON ingest**. User-facing
|
|
15
|
+
* paths for producing the snapshot JSON:
|
|
16
|
+
*
|
|
17
|
+
* a) Browser extension (planned v0.3) that scrapes yangkeduo.com order
|
|
18
|
+
* pages while logged in and exports JSON matching this schema.
|
|
19
|
+
* b) Manual hand-roll (rare; for testing).
|
|
20
|
+
*
|
|
21
|
+
* UI surface: pinduoduo card appears alongside alipay/taobao/jd/meituan in
|
|
22
|
+
* 推文 §"支付与购物" 大类, with an explicit "v0.2 待用户导出 — 需 web
|
|
23
|
+
* extension 或手抄" banner so user knows the limitation.
|
|
24
|
+
*
|
|
25
|
+
* Snapshot schema (mirrors PinduoduoLocalCollector.SNAPSHOT_SCHEMA_VERSION
|
|
26
|
+
* once the Kotlin collector lands in v0.3+):
|
|
27
|
+
*
|
|
28
|
+
* {
|
|
29
|
+
* "schemaVersion": 1,
|
|
30
|
+
* "snapshottedAt": <epoch-ms>,
|
|
31
|
+
* "vendor": "pinduoduo",
|
|
32
|
+
* "account": { "uid": "...", "displayName": "..." },
|
|
33
|
+
* "events": [
|
|
34
|
+
* { "kind": "order", "id": "order-<orderSn>", "capturedAt": <ms>,
|
|
35
|
+
* "orderId": "...", // pinduoduo's order_sn
|
|
36
|
+
* "merchantName": "...", // mall_name
|
|
37
|
+
* "items": [{ "name": ..., "quantity": ..., "unitPrice": ..., "sku": ... }],
|
|
38
|
+
* "placedAt": ..., // create_at_text or order_create_at
|
|
39
|
+
* "paidAt": ...,
|
|
40
|
+
* "status": "placed|shipped|delivered|cancelled|refunded",
|
|
41
|
+
* "totalAmount": { "value": ..., "currency": "CNY" },
|
|
42
|
+
* "recipient": "...",
|
|
43
|
+
* "shippingAddress": "...",
|
|
44
|
+
* "trackingNumber": "..." }
|
|
45
|
+
* ]
|
|
46
|
+
* }
|
|
47
|
+
*
|
|
48
|
+
* Future v0.3: HTML parsing (`Save As Webpage` from `mobile.yangkeduo.com/
|
|
49
|
+
* users/orders.html` — pinduoduo's order list endpoint).
|
|
50
|
+
*/
|
|
51
|
+
|
|
52
|
+
"use strict";
|
|
53
|
+
|
|
54
|
+
const fs = require("node:fs");
|
|
55
|
+
const { normalizeOrderRecord } = require("../shopping-base");
|
|
56
|
+
|
|
57
|
+
const NAME = "shopping-pinduoduo";
|
|
58
|
+
const VERSION = "0.1.0";
|
|
59
|
+
const SNAPSHOT_SCHEMA_VERSION = 1;
|
|
60
|
+
|
|
61
|
+
const KIND_ORDER = "order";
|
|
62
|
+
const VALID_SNAPSHOT_KINDS = Object.freeze([KIND_ORDER]);
|
|
63
|
+
|
|
64
|
+
class PinduoduoAdapter {
|
|
65
|
+
constructor(opts = {}) {
|
|
66
|
+
// §2.4c v0.2: account is OPTIONAL — snapshot mode is stateless. There's
|
|
67
|
+
// no cookie mode at all (anti_token signing path deferred to v0.3+).
|
|
68
|
+
this.account = opts.account || null;
|
|
69
|
+
|
|
70
|
+
this.name = NAME;
|
|
71
|
+
this.version = VERSION;
|
|
72
|
+
this.capabilities = ["sync:snapshot", "parse:pinduoduo-orders"];
|
|
73
|
+
this.extractMode = "user-export";
|
|
74
|
+
this.rateLimits = {};
|
|
75
|
+
this.dataDisclosure = {
|
|
76
|
+
fields: [
|
|
77
|
+
"pinduoduo:order_sn / mall_name / goods_list / order_amount / address",
|
|
78
|
+
],
|
|
79
|
+
sensitivity: "high",
|
|
80
|
+
legalGate: false,
|
|
81
|
+
defaultInclude: { order: true },
|
|
82
|
+
};
|
|
83
|
+
|
|
84
|
+
// _deps injection seam — vi.mock fs doesn't intercept inlined CJS require
|
|
85
|
+
// (see .claude/rules/testing.md).
|
|
86
|
+
this._deps = { fs };
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
async authenticate(ctx = {}) {
|
|
90
|
+
if (ctx && typeof ctx.inputPath === "string" && ctx.inputPath.length > 0) {
|
|
91
|
+
try {
|
|
92
|
+
this._deps.fs.accessSync(ctx.inputPath, this._deps.fs.constants.R_OK);
|
|
93
|
+
} catch (err) {
|
|
94
|
+
return {
|
|
95
|
+
ok: false,
|
|
96
|
+
reason: "INPUT_PATH_UNREADABLE",
|
|
97
|
+
message: `snapshot not readable at ${ctx.inputPath}: ${err.message}`,
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
return { ok: true, mode: "snapshot-file" };
|
|
101
|
+
}
|
|
102
|
+
return {
|
|
103
|
+
ok: false,
|
|
104
|
+
reason: "NO_INPUT",
|
|
105
|
+
message:
|
|
106
|
+
"PinduoduoAdapter.authenticate: needs opts.inputPath (snapshot mode — no cookie mode in v0.2)",
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
async healthCheck() {
|
|
111
|
+
return { ok: true, lastChecked: Date.now() };
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
async *sync(opts = {}) {
|
|
115
|
+
if (typeof opts.inputPath === "string" && opts.inputPath.length > 0) {
|
|
116
|
+
yield* this._syncViaSnapshot(opts);
|
|
117
|
+
return;
|
|
118
|
+
}
|
|
119
|
+
throw new Error(
|
|
120
|
+
"PinduoduoAdapter.sync: needs opts.inputPath (snapshot mode; no cookie/api mode in v0.2 because pinduoduo's web API requires anti_token JS-VM signing)",
|
|
121
|
+
);
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
async *_syncViaSnapshot(opts) {
|
|
125
|
+
const raw = this._deps.fs.readFileSync(opts.inputPath, "utf-8");
|
|
126
|
+
// v0.2 explicit JSON-only. HTML parsing (SAF-exported webpage from
|
|
127
|
+
// yangkeduo.com order list) is future v0.3 work.
|
|
128
|
+
let snapshot;
|
|
129
|
+
try {
|
|
130
|
+
snapshot = JSON.parse(raw);
|
|
131
|
+
} catch (err) {
|
|
132
|
+
throw new Error(
|
|
133
|
+
`shopping-pinduoduo.sync: snapshot must be JSON (v0.3 will add HTML parsing). Got parse error: ${err.message}`,
|
|
134
|
+
);
|
|
135
|
+
}
|
|
136
|
+
if (
|
|
137
|
+
!snapshot ||
|
|
138
|
+
typeof snapshot !== "object" ||
|
|
139
|
+
snapshot.schemaVersion !== SNAPSHOT_SCHEMA_VERSION
|
|
140
|
+
) {
|
|
141
|
+
throw new Error(
|
|
142
|
+
`shopping-pinduoduo.sync: snapshot schemaVersion mismatch (got ${snapshot && snapshot.schemaVersion}, expected ${SNAPSHOT_SCHEMA_VERSION})`,
|
|
143
|
+
);
|
|
144
|
+
}
|
|
145
|
+
const fallbackCapturedAt =
|
|
146
|
+
Number.isFinite(snapshot.snapshottedAt) && snapshot.snapshottedAt > 0
|
|
147
|
+
? Math.floor(snapshot.snapshottedAt)
|
|
148
|
+
: Date.now();
|
|
149
|
+
const account =
|
|
150
|
+
snapshot.account && typeof snapshot.account === "object"
|
|
151
|
+
? snapshot.account
|
|
152
|
+
: null;
|
|
153
|
+
const include = opts.include || {};
|
|
154
|
+
const limit =
|
|
155
|
+
Number.isInteger(opts.limit) && opts.limit > 0 ? opts.limit : Infinity;
|
|
156
|
+
|
|
157
|
+
const events = Array.isArray(snapshot.events) ? snapshot.events : [];
|
|
158
|
+
let emitted = 0;
|
|
159
|
+
for (const ev of events) {
|
|
160
|
+
if (emitted >= limit) return;
|
|
161
|
+
if (!ev || typeof ev !== "object") continue;
|
|
162
|
+
const kind = ev.kind;
|
|
163
|
+
if (!VALID_SNAPSHOT_KINDS.includes(kind)) continue;
|
|
164
|
+
if (include[kind] === false) continue;
|
|
165
|
+
|
|
166
|
+
const capturedAt =
|
|
167
|
+
parseTime(ev.capturedAt) ||
|
|
168
|
+
parseTime(ev.placedAt) ||
|
|
169
|
+
parseTime(ev.paidAt) ||
|
|
170
|
+
fallbackCapturedAt;
|
|
171
|
+
const id =
|
|
172
|
+
(typeof ev.id === "string" && ev.id.length > 0 && ev.id) ||
|
|
173
|
+
ev.orderId ||
|
|
174
|
+
null;
|
|
175
|
+
|
|
176
|
+
yield {
|
|
177
|
+
adapter: NAME,
|
|
178
|
+
kind,
|
|
179
|
+
originalId: stableOriginalId(kind, id),
|
|
180
|
+
capturedAt,
|
|
181
|
+
payload: { ...ev, account },
|
|
182
|
+
};
|
|
183
|
+
emitted += 1;
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
normalize(raw) {
|
|
188
|
+
if (!raw || !raw.payload) {
|
|
189
|
+
throw new Error("PinduoduoAdapter.normalize: payload missing");
|
|
190
|
+
}
|
|
191
|
+
// Snapshot-mode only — payload carries fields directly on the event.
|
|
192
|
+
const rec = snapshotEventToRecord(raw.payload);
|
|
193
|
+
return normalizeOrderRecord(rec, {
|
|
194
|
+
adapterName: NAME,
|
|
195
|
+
adapterVersion: VERSION,
|
|
196
|
+
});
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
function stableOriginalId(kind, id) {
|
|
201
|
+
const stringified =
|
|
202
|
+
(typeof id === "string" && id.length > 0 && id) ||
|
|
203
|
+
(typeof id === "number" && Number.isFinite(id) && String(id)) ||
|
|
204
|
+
null;
|
|
205
|
+
const safe =
|
|
206
|
+
stringified ||
|
|
207
|
+
`unknown-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
|
208
|
+
return `pinduoduo:${kind}:${safe}`;
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
function snapshotEventToRecord(ev) {
|
|
212
|
+
const items = [];
|
|
213
|
+
const rawItems = Array.isArray(ev.items) ? ev.items : [];
|
|
214
|
+
for (const it of rawItems) {
|
|
215
|
+
if (!it) continue;
|
|
216
|
+
items.push({
|
|
217
|
+
name: it.name || it.goods_name || it.skuName,
|
|
218
|
+
quantity: parseInt(it.quantity || it.goods_count || 1, 10),
|
|
219
|
+
unitPrice: parseFloat(it.unitPrice || it.goods_price || 0),
|
|
220
|
+
sku: it.sku || it.sku_id || null,
|
|
221
|
+
});
|
|
222
|
+
}
|
|
223
|
+
return {
|
|
224
|
+
vendorId: "pinduoduo",
|
|
225
|
+
orderId: String(ev.orderId || ev.id || "unknown"),
|
|
226
|
+
placedAt: parseTime(ev.placedAt),
|
|
227
|
+
paidAt: parseTime(ev.paidAt),
|
|
228
|
+
status: mapStatus(ev.status),
|
|
229
|
+
merchantName: ev.merchantName || ev.mall_name || "拼多多",
|
|
230
|
+
totalAmount:
|
|
231
|
+
ev.totalAmount && typeof ev.totalAmount === "object"
|
|
232
|
+
? {
|
|
233
|
+
value: parseFloat(ev.totalAmount.value || 0),
|
|
234
|
+
currency: ev.totalAmount.currency || "CNY",
|
|
235
|
+
}
|
|
236
|
+
: { value: 0, currency: "CNY" },
|
|
237
|
+
items,
|
|
238
|
+
recipient: ev.recipient || null,
|
|
239
|
+
shippingAddress: ev.shippingAddress || null,
|
|
240
|
+
trackingNumber: ev.trackingNumber || null,
|
|
241
|
+
extras: { capturedBy: "snapshot", platform: "pinduoduo" },
|
|
242
|
+
};
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
function parseTime(v) {
|
|
246
|
+
if (Number.isFinite(v)) return v < 1e12 ? v * 1000 : v;
|
|
247
|
+
if (typeof v === "string") {
|
|
248
|
+
if (/^\d+$/.test(v)) {
|
|
249
|
+
const n = parseInt(v, 10);
|
|
250
|
+
return n < 1e12 ? n * 1000 : n;
|
|
251
|
+
}
|
|
252
|
+
const t = Date.parse(v);
|
|
253
|
+
if (Number.isFinite(t)) return t;
|
|
254
|
+
}
|
|
255
|
+
return null;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
function mapStatus(s) {
|
|
259
|
+
const t = String(s || "").toLowerCase();
|
|
260
|
+
if (t.includes("退款") || t.includes("refund")) return "refunded";
|
|
261
|
+
if (t.includes("取消") || t.includes("cancel") || t.includes("已关闭")) return "cancelled";
|
|
262
|
+
if (t.includes("已发货") || t.includes("配送") || t.includes("shipped")) return "shipped";
|
|
263
|
+
if (t.includes("已完成") || t.includes("已收货") || t.includes("delivered")) return "delivered";
|
|
264
|
+
// Pinduoduo-specific statuses
|
|
265
|
+
if (t === "placed" || t.includes("待付款") || t.includes("待支付")) return "placed";
|
|
266
|
+
return "placed";
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
module.exports = {
|
|
270
|
+
PinduoduoAdapter,
|
|
271
|
+
NAME,
|
|
272
|
+
VERSION,
|
|
273
|
+
SNAPSHOT_SCHEMA_VERSION,
|
|
274
|
+
VALID_SNAPSHOT_KINDS,
|
|
275
|
+
};
|