@chainlesschain/personal-data-hub 0.4.7 → 0.4.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/__tests__/adapters/doc-baidu-netdisk.test.js +102 -0
- package/__tests__/adapters/doc-platforms.test.js +177 -0
- package/__tests__/adapters/music-kugou.test.js +187 -0
- package/__tests__/adapters/recruit-boss.test.js +180 -0
- package/__tests__/adapters/shopping-dianping.test.js +239 -0
- package/__tests__/adapters/social-csdn.test.js +175 -0
- package/__tests__/adapters/social-zhihu.test.js +246 -0
- package/__tests__/adapters/travel-ctrip.test.js +175 -1
- package/__tests__/adapters/travel-didi.test.js +204 -0
- package/__tests__/adapters/travel-tongcheng.test.js +289 -0
- package/__tests__/adapters/video-platforms.test.js +152 -0
- package/lib/adapter-guide.js +13 -1
- package/lib/adapters/_document-base.js +370 -0
- package/lib/adapters/_video-base.js +331 -0
- package/lib/adapters/doc-baidu-netdisk/index.js +91 -0
- package/lib/adapters/doc-tencent-docs/index.js +94 -0
- package/lib/adapters/doc-wps/index.js +77 -0
- package/lib/adapters/music-kugou/index.js +418 -0
- package/lib/adapters/recruit-boss/index.js +442 -0
- package/lib/adapters/shopping-dianping/index.js +473 -0
- package/lib/adapters/social-csdn/index.js +444 -0
- package/lib/adapters/social-zhihu/index.js +488 -0
- package/lib/adapters/travel-ctrip/index.js +255 -40
- package/lib/adapters/travel-didi/index.js +327 -0
- package/lib/adapters/travel-tongcheng/index.js +393 -0
- package/lib/adapters/video-iqiyi/index.js +75 -0
- package/lib/adapters/video-tencent/index.js +78 -0
- package/lib/index.js +24 -0
- package/package.json +1 -1
|
@@ -0,0 +1,327 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* §9.3e — Didi 企业版 (滴滴企业版 / 滴滴出差, com.didi.es.psngr) ride adapter,
|
|
3
|
+
* dual-mode (snapshot + cookie-api). Phase 9 travel ⭐⭐ "出差打车" — the last
|
|
4
|
+
* Phase-9 roadmap entry (§12.1), completing 完成阶段 travel coverage.
|
|
5
|
+
*
|
|
6
|
+
* A ride-hailing trip maps cleanly onto the vendor-neutral TravelRecord: each
|
|
7
|
+
* ride is a `car` trip with start/end address, board/alight time, and fare. So
|
|
8
|
+
* this adapter mirrors travel-ctrip / travel-tongcheng's two-mode shape.
|
|
9
|
+
*
|
|
10
|
+
* 1. snapshot / file-import mode (opts.inputPath | opts.dataPath): JSON/JSONL
|
|
11
|
+
* dump from an Android in-APK collector / curated file. account OPTIONAL.
|
|
12
|
+
*
|
|
13
|
+
* 2. cookie-api mode (opts.account.cookies): fetch the user's ride history
|
|
14
|
+
* from the 滴滴企业版 order centre (es.xiaojukeji.com) via the injected
|
|
15
|
+
* `fetchFn` (Android in-APK cc → OkHttp; desktop hub → Electron WebView net
|
|
16
|
+
* request), paginate, map each ride → a car TravelRecord. A sign seam
|
|
17
|
+
* (opts.signProvider) covers Didi's anti-bot signature; best-effort unsigned
|
|
18
|
+
* when absent. Endpoint overridable via opts.ordersUrl (best-effort, not
|
|
19
|
+
* field-verified — FAMILY-23 playbook).
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
"use strict";
|
|
23
|
+
|
|
24
|
+
const fs = require("node:fs");
|
|
25
|
+
const { normalizeTravelRecord, parseChineseDateTime } = require("../travel-base");
|
|
26
|
+
const { CookieAuth } = require("../shopping-base");
|
|
27
|
+
|
|
28
|
+
const NAME = "travel-didi";
|
|
29
|
+
const VERSION = "0.1.0";
|
|
30
|
+
|
|
31
|
+
// Best-effort 滴滴企业版 ride-order list endpoint. Overridable via opts.ordersUrl.
|
|
32
|
+
const DIDI_ORDERS_URL = "https://es.xiaojukeji.com/river/Order/list";
|
|
33
|
+
const DEFAULT_PAGE_SIZE = 20;
|
|
34
|
+
const DEFAULT_MAX_PAGES = 10;
|
|
35
|
+
|
|
36
|
+
// Didi car-product codes/names → keep all as "car" (vehicleType), but record the
|
|
37
|
+
// finer product label in extras.productType.
|
|
38
|
+
function rideProductLabel(o) {
|
|
39
|
+
return (
|
|
40
|
+
o.productName ||
|
|
41
|
+
o.product_name ||
|
|
42
|
+
o.carLevel ||
|
|
43
|
+
o.requireLevelName ||
|
|
44
|
+
o.product ||
|
|
45
|
+
null
|
|
46
|
+
);
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
class DidiAdapter {
|
|
50
|
+
constructor(opts = {}) {
|
|
51
|
+
this.account = opts.account || null;
|
|
52
|
+
this._dataPath = opts.dataPath || null;
|
|
53
|
+
|
|
54
|
+
this._cookieAuth =
|
|
55
|
+
opts.account && opts.account.cookies
|
|
56
|
+
? new CookieAuth({ platform: "didi", cookies: opts.account.cookies })
|
|
57
|
+
: null;
|
|
58
|
+
this._fetchFn = typeof opts.fetchFn === "function" ? opts.fetchFn : defaultFetch;
|
|
59
|
+
this._signProvider =
|
|
60
|
+
typeof opts.signProvider === "function" ? opts.signProvider : null;
|
|
61
|
+
this._ordersUrl =
|
|
62
|
+
typeof opts.ordersUrl === "string" && opts.ordersUrl.length > 0
|
|
63
|
+
? opts.ordersUrl
|
|
64
|
+
: DIDI_ORDERS_URL;
|
|
65
|
+
|
|
66
|
+
this.name = NAME;
|
|
67
|
+
this.version = VERSION;
|
|
68
|
+
this.capabilities = [
|
|
69
|
+
"import:json",
|
|
70
|
+
"sync:snapshot",
|
|
71
|
+
"sync:cookie-api",
|
|
72
|
+
"parse:didi-rides",
|
|
73
|
+
];
|
|
74
|
+
this.extractMode = "file-import";
|
|
75
|
+
this.rateLimits = {};
|
|
76
|
+
this.dataDisclosure = {
|
|
77
|
+
fields: [
|
|
78
|
+
"didi:orderId / fromAddress / toAddress / departTime / arriveTime / fare / carType",
|
|
79
|
+
],
|
|
80
|
+
sensitivity: "medium",
|
|
81
|
+
legalGate: false,
|
|
82
|
+
};
|
|
83
|
+
|
|
84
|
+
this._deps = { fs };
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
async authenticate(ctx = {}) {
|
|
88
|
+
const filePath = (ctx && ctx.inputPath) || ctx.dataPath || this._dataPath;
|
|
89
|
+
if (filePath) {
|
|
90
|
+
try {
|
|
91
|
+
this._deps.fs.accessSync(filePath, this._deps.fs.constants.R_OK);
|
|
92
|
+
} catch (err) {
|
|
93
|
+
return {
|
|
94
|
+
ok: false,
|
|
95
|
+
reason: "INPUT_PATH_UNREADABLE",
|
|
96
|
+
message: `not readable at ${filePath}: ${err.message}`,
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
return { ok: true, mode: "snapshot-file" };
|
|
100
|
+
}
|
|
101
|
+
if (this._cookieAuth) {
|
|
102
|
+
const ok = await this._cookieAuth.validate();
|
|
103
|
+
if (!ok) return { ok: false, reason: "INVALID_COOKIE", error: "cookies missing" };
|
|
104
|
+
return {
|
|
105
|
+
ok: true,
|
|
106
|
+
account: (this.account && this.account.email) || null,
|
|
107
|
+
mode: "cookie",
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
return { ok: true, account: this.account ? this.account.email : null, mode: "ready" };
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
async healthCheck() {
|
|
114
|
+
if (this._cookieAuth) {
|
|
115
|
+
const r = await this.authenticate();
|
|
116
|
+
return r.ok
|
|
117
|
+
? { ok: true, lastChecked: Date.now() }
|
|
118
|
+
: { ok: false, reason: r.reason, error: r.error };
|
|
119
|
+
}
|
|
120
|
+
return { ok: true, lastChecked: Date.now() };
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
async *sync(opts = {}) {
|
|
124
|
+
const dataPath = opts.inputPath || opts.dataPath || this._dataPath;
|
|
125
|
+
if (dataPath) {
|
|
126
|
+
if (!this._deps.fs.existsSync(dataPath)) return;
|
|
127
|
+
const text = this._deps.fs.readFileSync(dataPath, "utf-8");
|
|
128
|
+
let records;
|
|
129
|
+
try {
|
|
130
|
+
records = parseRecords(text);
|
|
131
|
+
} catch (err) {
|
|
132
|
+
throw new Error(`DidiAdapter: parse failed: ${err.message}`);
|
|
133
|
+
}
|
|
134
|
+
for (const r of records) {
|
|
135
|
+
yield {
|
|
136
|
+
adapter: NAME,
|
|
137
|
+
originalId: r.recordId,
|
|
138
|
+
capturedAt: r.bookedAt || r.departureMs || Date.now(),
|
|
139
|
+
payload: { record: r },
|
|
140
|
+
};
|
|
141
|
+
}
|
|
142
|
+
return;
|
|
143
|
+
}
|
|
144
|
+
if (this._cookieAuth) {
|
|
145
|
+
yield* this._syncViaCookie(opts);
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
async *_syncViaCookie(opts = {}) {
|
|
150
|
+
if (!(await this._cookieAuth.validate())) return;
|
|
151
|
+
const cookies = this._cookieAuth.toHeader();
|
|
152
|
+
const sinceMs =
|
|
153
|
+
opts.sinceWatermark != null
|
|
154
|
+
? parseInt(String(opts.sinceWatermark), 10) || 0
|
|
155
|
+
: Date.now() - 365 * 24 * 3600_000;
|
|
156
|
+
const pageSize = Number.isFinite(opts.pageSize) ? opts.pageSize : DEFAULT_PAGE_SIZE;
|
|
157
|
+
const maxPages =
|
|
158
|
+
Number.isInteger(opts.maxPages) && opts.maxPages > 0 ? opts.maxPages : DEFAULT_MAX_PAGES;
|
|
159
|
+
const limit = Number.isInteger(opts.limit) && opts.limit > 0 ? opts.limit : Infinity;
|
|
160
|
+
|
|
161
|
+
let emitted = 0;
|
|
162
|
+
let pageIndex = 1;
|
|
163
|
+
while (pageIndex <= maxPages) {
|
|
164
|
+
const query = { pageIndex, pageSize, ts: Date.now() };
|
|
165
|
+
let sign = null;
|
|
166
|
+
if (this._signProvider) {
|
|
167
|
+
sign = await this._signProvider({ url: this._ordersUrl, query, cookies });
|
|
168
|
+
}
|
|
169
|
+
const resp = await this._fetchFn({ url: this._ordersUrl, cookies, query, sign });
|
|
170
|
+
const rides = extractOrders(resp);
|
|
171
|
+
if (!rides.length) break;
|
|
172
|
+
|
|
173
|
+
let pageHasNew = false;
|
|
174
|
+
let reachedWatermark = false;
|
|
175
|
+
for (const raw of rides) {
|
|
176
|
+
const rec = orderToRecord(raw, { capturedVia: "cookie-api" });
|
|
177
|
+
if (!rec) continue;
|
|
178
|
+
const ts = rec.departureMs || rec.bookedAt || null;
|
|
179
|
+
if (ts && ts < sinceMs) {
|
|
180
|
+
reachedWatermark = true;
|
|
181
|
+
break;
|
|
182
|
+
}
|
|
183
|
+
pageHasNew = true;
|
|
184
|
+
if (emitted >= limit) return;
|
|
185
|
+
yield {
|
|
186
|
+
adapter: NAME,
|
|
187
|
+
originalId: rec.recordId,
|
|
188
|
+
capturedAt: ts || Date.now(),
|
|
189
|
+
payload: { record: rec },
|
|
190
|
+
};
|
|
191
|
+
emitted += 1;
|
|
192
|
+
}
|
|
193
|
+
if (reachedWatermark || !pageHasNew || rides.length < pageSize) break;
|
|
194
|
+
pageIndex += 1;
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
normalize(raw) {
|
|
199
|
+
if (!raw || !raw.payload || !raw.payload.record) {
|
|
200
|
+
throw new Error("DidiAdapter.normalize: raw.payload.record missing");
|
|
201
|
+
}
|
|
202
|
+
return normalizeTravelRecord(raw.payload.record, {
|
|
203
|
+
adapterName: NAME,
|
|
204
|
+
adapterVersion: VERSION,
|
|
205
|
+
});
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
function parseRecords(text) {
|
|
210
|
+
let raw;
|
|
211
|
+
try {
|
|
212
|
+
raw = JSON.parse(text);
|
|
213
|
+
} catch (_e) {
|
|
214
|
+
raw = text
|
|
215
|
+
.split(/\r?\n/)
|
|
216
|
+
.filter((l) => l.trim().startsWith("{"))
|
|
217
|
+
.map((l) => JSON.parse(l));
|
|
218
|
+
}
|
|
219
|
+
const rides = Array.isArray(raw) ? raw : raw.orders || raw.rides || [];
|
|
220
|
+
return rides.map((o) => orderToRecord(o)).filter(Boolean);
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
/**
|
|
224
|
+
* Map one Didi ride object → vendor-neutral car TravelRecord. Field names are
|
|
225
|
+
* best-effort across endpoint versions (camelCase + snake_case + Chinese).
|
|
226
|
+
*/
|
|
227
|
+
function orderToRecord(o, opts = {}) {
|
|
228
|
+
if (!o || typeof o !== "object") return null;
|
|
229
|
+
const recordId = o.orderId || o.oid || o.id || o.order_id || o.travelId;
|
|
230
|
+
if (!recordId) return null;
|
|
231
|
+
const product = rideProductLabel(o);
|
|
232
|
+
|
|
233
|
+
const fareRaw = firstNonNull([
|
|
234
|
+
o.fare,
|
|
235
|
+
o.totalFee,
|
|
236
|
+
o.total_fee,
|
|
237
|
+
o.payAmount,
|
|
238
|
+
o.pay_amount,
|
|
239
|
+
o.amount,
|
|
240
|
+
o.price,
|
|
241
|
+
o.totalPrice,
|
|
242
|
+
o.total_price,
|
|
243
|
+
]);
|
|
244
|
+
|
|
245
|
+
const fromAddr = o.fromAddress || o.from_address || o.startName || o.startAddress || o.fromName;
|
|
246
|
+
const toAddr = o.toAddress || o.to_address || o.endName || o.endAddress || o.toName;
|
|
247
|
+
|
|
248
|
+
return {
|
|
249
|
+
vendorId: "didi",
|
|
250
|
+
recordId: String(recordId),
|
|
251
|
+
vehicleType: "car",
|
|
252
|
+
from: fromAddr ? { name: fromAddr } : null,
|
|
253
|
+
to: toAddr ? { name: toAddr } : null,
|
|
254
|
+
departureMs: numberOrParse(
|
|
255
|
+
o.departTime || o.depart_time || o.boardTime || o.startTime || o.beginChargeTime || o.setupTime,
|
|
256
|
+
),
|
|
257
|
+
arrivalMs: numberOrParse(o.arriveTime || o.arrive_time || o.endTime || o.finishTime),
|
|
258
|
+
carrier: "滴滴",
|
|
259
|
+
vehicleNumber: o.carNo || o.plateNo || o.car_plate || null,
|
|
260
|
+
totalCost: fareRaw != null ? { value: parseFareYuan(fareRaw), currency: "CNY" } : null,
|
|
261
|
+
traveler: o.passengerName || o.passenger || o.riderName || o.userName,
|
|
262
|
+
confirmationCode: null,
|
|
263
|
+
bookedAt: numberOrParse(o.createTime || o.create_time || o.orderTime || o.bookedAt),
|
|
264
|
+
extras: {
|
|
265
|
+
type: "car",
|
|
266
|
+
...(product ? { productType: product } : {}),
|
|
267
|
+
...(o.driverName ? { driver: o.driverName } : {}),
|
|
268
|
+
...(opts.capturedVia ? { capturedVia: opts.capturedVia } : {}),
|
|
269
|
+
},
|
|
270
|
+
};
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
/** Didi fares are sometimes 分 (integer cents), sometimes 元 (decimal). */
|
|
274
|
+
function parseFareYuan(v) {
|
|
275
|
+
const n = typeof v === "number" ? v : parseFloat(String(v));
|
|
276
|
+
if (!Number.isFinite(n)) return 0;
|
|
277
|
+
// Heuristic: large integers (>= 1000 with no decimal) are very likely 分.
|
|
278
|
+
if (Number.isInteger(n) && n >= 1000) return Math.round(n) / 100;
|
|
279
|
+
return n;
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
function extractOrders(resp) {
|
|
283
|
+
if (!resp || typeof resp !== "object") return [];
|
|
284
|
+
if (Array.isArray(resp.orders)) return resp.orders;
|
|
285
|
+
if (Array.isArray(resp.rides)) return resp.rides;
|
|
286
|
+
if (Array.isArray(resp.list)) return resp.list;
|
|
287
|
+
const data = resp.data && typeof resp.data === "object" ? resp.data : null;
|
|
288
|
+
if (data) {
|
|
289
|
+
if (Array.isArray(data.orders)) return data.orders;
|
|
290
|
+
if (Array.isArray(data.list)) return data.list;
|
|
291
|
+
if (Array.isArray(data.records)) return data.records;
|
|
292
|
+
}
|
|
293
|
+
return [];
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
function firstNonNull(arr) {
|
|
297
|
+
for (const v of arr) if (v != null) return v;
|
|
298
|
+
return null;
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
// 13-digit epoch (>= 1e12) is already ms; 10-digit (1e9..<1e12) is seconds → ms.
|
|
302
|
+
function toMs(n) {
|
|
303
|
+
return n >= 1e12 ? n : n >= 1e9 ? n * 1000 : n;
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
function numberOrParse(v) {
|
|
307
|
+
if (Number.isFinite(v)) return toMs(v);
|
|
308
|
+
if (typeof v === "string") {
|
|
309
|
+
if (/^\d+$/.test(v)) return toMs(parseInt(v, 10));
|
|
310
|
+
return parseChineseDateTime(v);
|
|
311
|
+
}
|
|
312
|
+
return null;
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
async function defaultFetch(_opts) {
|
|
316
|
+
throw new Error("travel-didi: no fetchFn configured for cookie-api mode");
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
module.exports = {
|
|
320
|
+
DidiAdapter,
|
|
321
|
+
parseRecords,
|
|
322
|
+
orderToRecord,
|
|
323
|
+
extractOrders,
|
|
324
|
+
parseFareYuan,
|
|
325
|
+
NAME,
|
|
326
|
+
VERSION,
|
|
327
|
+
};
|