@swimmingliu/autovpn 1.8.0 → 2.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/artifacts/list.js +48 -6
- package/dist/cli/accounts-commands.js +304 -0
- package/dist/cli/commands/index.js +6 -0
- package/dist/cli/native-commands.js +10 -0
- package/dist/cli/output.js +1 -0
- package/dist/config/profile.js +119 -5
- package/dist/events/schema.js +2 -0
- package/dist/pipeline/apk-discovery.js +504 -0
- package/dist/pipeline/brand-cache.js +126 -0
- package/dist/pipeline/brand-protocol.js +210 -0
- package/dist/pipeline/credential-source.js +1406 -0
- package/dist/pipeline/extract.js +55 -2
- package/dist/pipeline/orchestrator.js +88 -9
- package/dist/pool/account-store.js +484 -0
- package/dist/pool/brands.js +110 -0
- package/dist/pool/csv.js +388 -0
- package/dist/server/http.js +25 -0
- package/dist/server/runtime.js +76 -2
- package/dist/server/web-adapter.js +18 -2
- package/dist/web/renderer/THIRD_PARTY_NOTICES.txt +345 -0
- package/dist/web/renderer/analytics.bundle.css +2 -0
- package/dist/web/renderer/analytics.bundle.js +66 -0
- package/dist/web/renderer/app.js +385 -80
- package/dist/web/renderer/appearance.js +76 -0
- package/dist/web/renderer/assets/vpn-auto-logo-v3.png +0 -0
- package/dist/web/renderer/i18n.js +1 -1
- package/dist/web/renderer/icons.js +38 -0
- package/dist/web/renderer/index.html +24 -2
- package/dist/web/renderer/state.js +0 -1
- package/dist/web/renderer/styles.css +584 -48
- package/dist/web/renderer/tables.js +208 -0
- package/dist/web/renderer/views.js +483 -255
- package/dist/web/renderer/workspace.css +381 -0
- package/package.json +4 -3
|
@@ -0,0 +1,1406 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import { fetch as upstreamFetch, ProxyAgent } from 'undici';
|
|
3
|
+
import { androidChunkUrls, apkDiscoveryFromBytes, inlineDownloadSpec, candidateApkUrls, cdnConfigUrls, parseCdnConfig, apkSlot } from './apk-discovery.js';
|
|
4
|
+
import { buildUrl, commonParams, flattenRouteGroups, isPlaceholderNode, LOGIN_SALT, pathAndQuery, selectApiDomain, selectForeignRoutes } from './brand-protocol.js';
|
|
5
|
+
import { decryptPayload, extractLinksFromPlaintext, linkDigest, linkFingerprint } from './extract.js';
|
|
6
|
+
import { AccountStore } from '../pool/account-store.js';
|
|
7
|
+
import { brandFromSite, normalizeBrand } from '../pool/brands.js';
|
|
8
|
+
/**
|
|
9
|
+
* Each brand concatenates `getCommParams()` in its own order with its own
|
|
10
|
+
* literal block. The layout can be overridden from the profile, and is picked
|
|
11
|
+
* from the brand name otherwise.
|
|
12
|
+
*/
|
|
13
|
+
export function resolveBrandLayout(source) {
|
|
14
|
+
const explicit = String(source.brand_flags ?? '').trim();
|
|
15
|
+
if (explicit) {
|
|
16
|
+
try {
|
|
17
|
+
const parsed = JSON.parse(explicit);
|
|
18
|
+
if (parsed && typeof parsed === 'object') {
|
|
19
|
+
return parsed;
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
catch {
|
|
23
|
+
// A bare literal block is treated as an after-package flag.
|
|
24
|
+
return { afterPackage: explicit.startsWith('&') ? explicit : `&${explicit}` };
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
const brand = String(source.brand ?? '').trim();
|
|
28
|
+
if (brand.includes('旋风') || /xuanfeng/i.test(brand)) {
|
|
29
|
+
return { vipFirst: true, viewport: true };
|
|
30
|
+
}
|
|
31
|
+
if (brand.includes('雷霆') || /leiting/i.test(brand)) {
|
|
32
|
+
return { viewport: true, afterPackage: '&newlt=1', beforeApps: '&update_action=web' };
|
|
33
|
+
}
|
|
34
|
+
if (brand.includes('黑洞') || /heidong|blackhole/i.test(brand)) {
|
|
35
|
+
return { afterPackage: '&newhd=1' };
|
|
36
|
+
}
|
|
37
|
+
return {};
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Archive filename prefixes, keyed by every name a brand is addressed by.
|
|
41
|
+
*
|
|
42
|
+
* A profile normally names its sources in English (`leiting`), while the
|
|
43
|
+
* downloader uses a Chinese product name, so both are mapped. The official
|
|
44
|
+
* host is the last resort when a profile uses a custom source name.
|
|
45
|
+
*/
|
|
46
|
+
const BRAND_APK_PREFIXES = {
|
|
47
|
+
雷霆: 'lt',
|
|
48
|
+
旋风: 'xf',
|
|
49
|
+
黑洞: 'hd',
|
|
50
|
+
蜜蜂: 'mf',
|
|
51
|
+
leiting: 'lt',
|
|
52
|
+
xuanfeng: 'xf',
|
|
53
|
+
heidong: 'hd',
|
|
54
|
+
mifeng: 'mf',
|
|
55
|
+
'www.68123836.com': 'lt',
|
|
56
|
+
'www.16863988.com': 'xf',
|
|
57
|
+
'www.67986457.com': 'hd',
|
|
58
|
+
'www.86933221.com': 'mf'
|
|
59
|
+
};
|
|
60
|
+
/** Host of an official site, without scheme or path. */
|
|
61
|
+
function siteHost(site) {
|
|
62
|
+
try {
|
|
63
|
+
return new URL(site).host.toLowerCase();
|
|
64
|
+
}
|
|
65
|
+
catch {
|
|
66
|
+
return '';
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Stable key for a brand's fingerprint.
|
|
71
|
+
*
|
|
72
|
+
* A profile may name two sources of the same brand (`leiting`, `leiting_ios`),
|
|
73
|
+
* and both resolve to the same APK: the package, certificate and seeds do not
|
|
74
|
+
* depend on the profile entry. Keying the cache and the in-run memo by that
|
|
75
|
+
* identity means one download and one inspection per brand, and one cache file
|
|
76
|
+
* that both sources share. Falls back to the source name when the brand can't
|
|
77
|
+
* be established, which keeps custom entries working.
|
|
78
|
+
*/
|
|
79
|
+
export function credentialBrandKey(sourceName, source) {
|
|
80
|
+
const site = String(source.official_site ?? '').trim();
|
|
81
|
+
const host = siteHost(site);
|
|
82
|
+
if (host) {
|
|
83
|
+
return BRAND_APK_PREFIXES[host] ?? host;
|
|
84
|
+
}
|
|
85
|
+
const name = String(sourceName ?? '').trim().toLowerCase();
|
|
86
|
+
const prefix = BRAND_APK_PREFIXES[sourceName] ?? BRAND_APK_PREFIXES[name];
|
|
87
|
+
return prefix ?? (name || 'unknown');
|
|
88
|
+
}
|
|
89
|
+
export function createFingerprintMemo(store, brand) {
|
|
90
|
+
let value = store?.read(brand) ?? null;
|
|
91
|
+
let pending = null;
|
|
92
|
+
return {
|
|
93
|
+
get: () => value,
|
|
94
|
+
set: (next) => {
|
|
95
|
+
value = next;
|
|
96
|
+
},
|
|
97
|
+
resolve: (loader) => {
|
|
98
|
+
// Android and iOS start together. Without this both would see an empty
|
|
99
|
+
// memo and download the same 20-30 MB APK before either could set it.
|
|
100
|
+
pending ??= loader().finally(() => {
|
|
101
|
+
pending = null;
|
|
102
|
+
});
|
|
103
|
+
return pending;
|
|
104
|
+
}
|
|
105
|
+
};
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* AES key candidates for the control plane.
|
|
109
|
+
*
|
|
110
|
+
* The key is not a literal in the dex, so it is offered as the configured value
|
|
111
|
+
* plus the two constants the current builds share; `/api/eversion` is what
|
|
112
|
+
* decides which one is right.
|
|
113
|
+
*/
|
|
114
|
+
export function credentialKeyCandidates(source, discovery) {
|
|
115
|
+
const out = [];
|
|
116
|
+
const add = (value) => {
|
|
117
|
+
const trimmed = String(value ?? '').trim();
|
|
118
|
+
if (trimmed && !out.includes(trimmed)) {
|
|
119
|
+
out.push(trimmed);
|
|
120
|
+
}
|
|
121
|
+
};
|
|
122
|
+
// A configured key wins; the dex pool is what makes rotation survivable,
|
|
123
|
+
// and the two shared constants cover a cache written before the pool existed.
|
|
124
|
+
add(source.key);
|
|
125
|
+
for (const candidate of discovery?.key_candidates ?? []) {
|
|
126
|
+
add(candidate);
|
|
127
|
+
}
|
|
128
|
+
add('ks9KUrbWJj46AftX');
|
|
129
|
+
add('awdtif20190619ti');
|
|
130
|
+
return out;
|
|
131
|
+
}
|
|
132
|
+
const DEFAULT_DEVICE_ID = '6C3B3033-4FAD-42FA-B8AD-DF3EDBCFA872';
|
|
133
|
+
const DEFAULT_TIMEOUT_MS = 20_000;
|
|
134
|
+
function numberOrDefault(value, fallback) {
|
|
135
|
+
const parsed = Number(value);
|
|
136
|
+
return Number.isFinite(parsed) ? parsed : fallback;
|
|
137
|
+
}
|
|
138
|
+
function today(now) {
|
|
139
|
+
const pad = (value) => String(value).padStart(2, '0');
|
|
140
|
+
return `${now.getFullYear()}-${pad(now.getMonth() + 1)}-${pad(now.getDate())}`;
|
|
141
|
+
}
|
|
142
|
+
export function resolveRoundPolicy(source) {
|
|
143
|
+
const maxIterations = Math.max(1, Math.trunc(numberOrDefault(source.max_iterations, 1)));
|
|
144
|
+
const configuredMin = Math.max(0, Math.trunc(numberOrDefault(source.min_iterations, 0)));
|
|
145
|
+
return {
|
|
146
|
+
max_iterations: maxIterations,
|
|
147
|
+
// A minimum above the maximum would otherwise make the cap unreachable.
|
|
148
|
+
min_iterations: Math.min(configuredMin, maxIterations),
|
|
149
|
+
plateau_limit: Math.max(1, Math.trunc(numberOrDefault(source.plateau_limit, 2))),
|
|
150
|
+
max_runtime_seconds: Math.max(0, numberOrDefault(source.max_runtime_seconds, 0))
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
/** The floor always runs; past it, the runtime budget can stop the loop. */
|
|
154
|
+
export function shouldStartRound(round, policy, elapsedSeconds) {
|
|
155
|
+
if (round > policy.max_iterations) {
|
|
156
|
+
return false;
|
|
157
|
+
}
|
|
158
|
+
if (policy.max_runtime_seconds <= 0 || round <= policy.min_iterations) {
|
|
159
|
+
return true;
|
|
160
|
+
}
|
|
161
|
+
return elapsedSeconds < policy.max_runtime_seconds;
|
|
162
|
+
}
|
|
163
|
+
/** Stops only once the floor has run and the pool has plateaued. */
|
|
164
|
+
export function shouldStopOnPlateau(round, plateau, policy) {
|
|
165
|
+
return plateau >= policy.plateau_limit && round >= policy.min_iterations;
|
|
166
|
+
}
|
|
167
|
+
/** `doLogin()` signs `account + password + salt`. */
|
|
168
|
+
export function loginSignature(account, password) {
|
|
169
|
+
return createHash('md5').update(`${account}${password}${LOGIN_SALT}`, 'utf8').digest('hex');
|
|
170
|
+
}
|
|
171
|
+
function isEnabled(value) {
|
|
172
|
+
return ['1', 'true', 'yes', 'on'].includes(String(value ?? '').trim().toLowerCase());
|
|
173
|
+
}
|
|
174
|
+
function upstreamProxyUrl(env) {
|
|
175
|
+
if (!env || !isEnabled(env.VPN_AUTOMATION_USE_UPSTREAM_PROXY)) {
|
|
176
|
+
return '';
|
|
177
|
+
}
|
|
178
|
+
const value = String(env.VPN_AUTOMATION_UPSTREAM_PROXY ?? 'http://127.0.0.1:7897').trim();
|
|
179
|
+
return ['', 'off', 'none', 'false', '0'].includes(value.toLowerCase()) ? '' : value;
|
|
180
|
+
}
|
|
181
|
+
async function fetchText(fetchImpl, url, timeoutMs, egress) {
|
|
182
|
+
const controller = new AbortController();
|
|
183
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
184
|
+
try {
|
|
185
|
+
const response = await fetchImpl(url, { signal: controller.signal, redirect: 'follow' });
|
|
186
|
+
if (!response.ok) {
|
|
187
|
+
return { text: '', url, ok: false };
|
|
188
|
+
}
|
|
189
|
+
return { text: await response.text(), url, ok: true };
|
|
190
|
+
}
|
|
191
|
+
catch {
|
|
192
|
+
// Same fallback the url mode applies: curl handles a proxy or a
|
|
193
|
+
// self-signed/intercepted TLS chain that Node's fetch rejects.
|
|
194
|
+
const proxyUrl = upstreamProxyUrl(egress?.env);
|
|
195
|
+
if (!egress?.curlFetch) {
|
|
196
|
+
return { text: '', url, ok: false };
|
|
197
|
+
}
|
|
198
|
+
try {
|
|
199
|
+
const text = await egress.curlFetch(url, proxyUrl);
|
|
200
|
+
return text.trim() ? { text, url, ok: true } : { text: '', url, ok: false };
|
|
201
|
+
}
|
|
202
|
+
catch {
|
|
203
|
+
return { text: '', url, ok: false };
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
finally {
|
|
207
|
+
clearTimeout(timer);
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
/** Control-plane responses are AES-CBC with key == IV. */
|
|
211
|
+
export function decodeControlPlane(cipherText, key) {
|
|
212
|
+
if (!cipherText.trim() || !key) {
|
|
213
|
+
return '';
|
|
214
|
+
}
|
|
215
|
+
try {
|
|
216
|
+
return decryptPayload(cipherText.trim(), key);
|
|
217
|
+
}
|
|
218
|
+
catch {
|
|
219
|
+
return '';
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
function buildComm(ctx, timestampMs) {
|
|
223
|
+
return commonParams({
|
|
224
|
+
identity: ctx.identity,
|
|
225
|
+
device: ctx.device,
|
|
226
|
+
platform: ctx.platform,
|
|
227
|
+
timestampMs,
|
|
228
|
+
token: ctx.token,
|
|
229
|
+
code: ctx.code,
|
|
230
|
+
isVip: ctx.isVip,
|
|
231
|
+
unlimited: ctx.isUnlimited
|
|
232
|
+
}, ctx.layout);
|
|
233
|
+
}
|
|
234
|
+
/**
|
|
235
|
+
* `getCommStr()` domain priority: last active, then the api_links array, then
|
|
236
|
+
* `getApiDomain()`, then the remaining seeds. The first non-empty response wins.
|
|
237
|
+
*/
|
|
238
|
+
async function getCommStr(ctx, apiLinks, apiDomain, uri, params, seedDomains) {
|
|
239
|
+
const lastActive = ctx.lastActiveDomain;
|
|
240
|
+
const candidates = [];
|
|
241
|
+
// `last_active_domain` is first in the app: the host that last answered is
|
|
242
|
+
// kept, so every later call stays on the control plane that served eversion.
|
|
243
|
+
for (const value of [lastActive, ...apiLinks, apiDomain, ...seedDomains]) {
|
|
244
|
+
if (value && !candidates.includes(value)) {
|
|
245
|
+
candidates.push(value);
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
for (const base of candidates) {
|
|
249
|
+
// `build_url()` is domain + uri + comm params + the call's own params.
|
|
250
|
+
const url = buildUrl(`${base.replace(/\/+$/, '')}${uri}`, buildComm(ctx, Date.now()) + params);
|
|
251
|
+
const response = await fetchText(ctx.fetchImpl, url, ctx.timeoutMs, ctx.egress);
|
|
252
|
+
const decrypted = decodeControlPlane(response.text, ctx.key ?? '');
|
|
253
|
+
if (decrypted) {
|
|
254
|
+
ctx.lastActiveDomain = base;
|
|
255
|
+
return decrypted;
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
return '';
|
|
259
|
+
}
|
|
260
|
+
/**
|
|
261
|
+
* Same domain walk as `getCommStr`, but keeps the response as sent.
|
|
262
|
+
*
|
|
263
|
+
* `/myapi/apinodelist` is the one control-plane call that answers in plain
|
|
264
|
+
* JSON; everything else is AES-wrapped, and only this endpoint needs the raw
|
|
265
|
+
* body for the `res` tree the app renders.
|
|
266
|
+
*/
|
|
267
|
+
async function getRawCommStr(ctx, apiLinks, apiDomain, uri, params, seedDomains) {
|
|
268
|
+
const candidates = [];
|
|
269
|
+
for (const value of [ctx.lastActiveDomain, ...apiLinks, apiDomain, ...seedDomains]) {
|
|
270
|
+
if (value && !candidates.includes(value)) {
|
|
271
|
+
candidates.push(value);
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
for (const base of candidates) {
|
|
275
|
+
const url = buildUrl(`${base.replace(/\/+$/, '')}${uri}`, buildComm(ctx, Date.now()) + params);
|
|
276
|
+
const response = await fetchText(ctx.fetchImpl, url, ctx.timeoutMs, ctx.egress);
|
|
277
|
+
if (response.text.trim()) {
|
|
278
|
+
ctx.lastActiveDomain = base;
|
|
279
|
+
return response.text;
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
return '';
|
|
283
|
+
}
|
|
284
|
+
/**
|
|
285
|
+
* `/api/eversion` is the one endpoint that must succeed without a session, and
|
|
286
|
+
* its response reveals the AES key: the key is the one that turns the body into
|
|
287
|
+
* JSON. Candidates come from the configured value and the two known constants.
|
|
288
|
+
*/
|
|
289
|
+
export async function resolveEversion(fetchImpl, seedDomains, comm, keyCandidates, timeoutMs, egress) {
|
|
290
|
+
for (const base of seedDomains) {
|
|
291
|
+
const url = buildUrl(`${base.replace(/\/+$/, '')}/api/eversion`, comm(Date.now()));
|
|
292
|
+
const response = await fetchText(fetchImpl, url, timeoutMs, egress);
|
|
293
|
+
if (!response.text) {
|
|
294
|
+
continue;
|
|
295
|
+
}
|
|
296
|
+
for (const key of keyCandidates) {
|
|
297
|
+
const plain = decodeControlPlane(response.text, key);
|
|
298
|
+
if (!plain) {
|
|
299
|
+
continue;
|
|
300
|
+
}
|
|
301
|
+
try {
|
|
302
|
+
const payload = JSON.parse(plain);
|
|
303
|
+
return {
|
|
304
|
+
domain: base,
|
|
305
|
+
key,
|
|
306
|
+
api_links: apiLinkValues(payload.api_links),
|
|
307
|
+
api_lines: apiLineValues(payload.api_lines),
|
|
308
|
+
payload
|
|
309
|
+
};
|
|
310
|
+
}
|
|
311
|
+
catch {
|
|
312
|
+
// Wrong key produces mojibake, not JSON; try the next candidate.
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
return null;
|
|
317
|
+
}
|
|
318
|
+
export function parseRouteGroups(payload) {
|
|
319
|
+
try {
|
|
320
|
+
const parsed = JSON.parse(payload);
|
|
321
|
+
const groups = Array.isArray(parsed.res) ? parsed.res : [];
|
|
322
|
+
return groups.map((group) => {
|
|
323
|
+
const record = group;
|
|
324
|
+
const nodes = Array.isArray(record.data) ? record.data : [];
|
|
325
|
+
return {
|
|
326
|
+
cid: Number(record.cid ?? -1),
|
|
327
|
+
cname: String(record.cname ?? ''),
|
|
328
|
+
open: Number(record.open ?? 0),
|
|
329
|
+
nodes: nodes.map((node) => {
|
|
330
|
+
const item = node;
|
|
331
|
+
return {
|
|
332
|
+
id: Number(item.id ?? -1),
|
|
333
|
+
name: String(item.name ?? ''),
|
|
334
|
+
enable: Number(item.enable ?? 0),
|
|
335
|
+
enable_desc: String(item.enable_desc ?? '')
|
|
336
|
+
};
|
|
337
|
+
})
|
|
338
|
+
};
|
|
339
|
+
});
|
|
340
|
+
}
|
|
341
|
+
catch {
|
|
342
|
+
return [];
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
/**
|
|
346
|
+
* `api_links` is a plain string array on most brands and an `{key, value}`
|
|
347
|
+
* array on 旋风, whose `value` field is the endpoint.
|
|
348
|
+
*/
|
|
349
|
+
export function apiLinkValues(raw) {
|
|
350
|
+
if (!Array.isArray(raw)) {
|
|
351
|
+
return [];
|
|
352
|
+
}
|
|
353
|
+
const out = [];
|
|
354
|
+
for (const item of raw) {
|
|
355
|
+
if (typeof item === 'string' && item.trim()) {
|
|
356
|
+
out.push(item.trim());
|
|
357
|
+
}
|
|
358
|
+
else if (item && typeof item === 'object') {
|
|
359
|
+
const value = item.value;
|
|
360
|
+
if (typeof value === 'string' && value.trim()) {
|
|
361
|
+
out.push(value.trim());
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
return out;
|
|
366
|
+
}
|
|
367
|
+
/**
|
|
368
|
+
* `api_lines` is a single URL on most brands and a `{key, value}` list on 旋风,
|
|
369
|
+
* where each entry is a separately labelled route table.
|
|
370
|
+
*
|
|
371
|
+
* Every entry is returned. 旋风 ships two tables (黄金/白金) whose area ids
|
|
372
|
+
* overlap but do not coincide, so taking only the first silently dropped half
|
|
373
|
+
* its routes; the caller merges them and dedupes by area.
|
|
374
|
+
*/
|
|
375
|
+
export function apiLineValues(raw) {
|
|
376
|
+
if (typeof raw === 'string') {
|
|
377
|
+
return raw.trim() ? [raw.trim()] : [];
|
|
378
|
+
}
|
|
379
|
+
if (!Array.isArray(raw)) {
|
|
380
|
+
return [];
|
|
381
|
+
}
|
|
382
|
+
const out = [];
|
|
383
|
+
for (const item of raw) {
|
|
384
|
+
if (typeof item === 'string' && item.trim()) {
|
|
385
|
+
if (!out.includes(item.trim()))
|
|
386
|
+
out.push(item.trim());
|
|
387
|
+
continue;
|
|
388
|
+
}
|
|
389
|
+
if (item && typeof item === 'object') {
|
|
390
|
+
const value = item.value;
|
|
391
|
+
if (typeof value === 'string' && value.trim() && !out.includes(value.trim())) {
|
|
392
|
+
out.push(value.trim());
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
return out;
|
|
397
|
+
}
|
|
398
|
+
function eversionStrings(payload) {
|
|
399
|
+
return {
|
|
400
|
+
apiLinks: apiLinkValues(payload.api_links),
|
|
401
|
+
apiLines: apiLineValues(payload.api_lines)
|
|
402
|
+
};
|
|
403
|
+
}
|
|
404
|
+
function event(options, type, payload) {
|
|
405
|
+
options.eventCallback?.(type, payload);
|
|
406
|
+
}
|
|
407
|
+
/**
|
|
408
|
+
* A real APK is 20-30 MB. The download host is scraped from vendor JS rather
|
|
409
|
+
* than pinned, so an oversized body (or a redirect to one) must be refused
|
|
410
|
+
* instead of buffered into memory.
|
|
411
|
+
*/
|
|
412
|
+
const MAX_APK_BYTES = 96 * 1024 * 1024;
|
|
413
|
+
/** Refuses a response whose declared size already exceeds the cap. */
|
|
414
|
+
function declaredSizeTooLarge(response) {
|
|
415
|
+
const raw = response?.headers?.get?.('content-length');
|
|
416
|
+
if (!raw) {
|
|
417
|
+
return false;
|
|
418
|
+
}
|
|
419
|
+
const length = Number(raw);
|
|
420
|
+
return Number.isFinite(length) && length > MAX_APK_BYTES;
|
|
421
|
+
}
|
|
422
|
+
async function defaultLoadApk(source, fetchImpl, timeoutMs, now, log, brandName = '', egress) {
|
|
423
|
+
if (source.apk_path) {
|
|
424
|
+
try {
|
|
425
|
+
const { readFile } = await import('node:fs/promises');
|
|
426
|
+
return await readFile(source.apk_path);
|
|
427
|
+
}
|
|
428
|
+
catch {
|
|
429
|
+
return null;
|
|
430
|
+
}
|
|
431
|
+
}
|
|
432
|
+
if (source.apk_url) {
|
|
433
|
+
const controller = new AbortController();
|
|
434
|
+
const timer = setTimeout(() => controller.abort(), Math.max(timeoutMs, 120_000));
|
|
435
|
+
try {
|
|
436
|
+
const response = await fetchImpl(source.apk_url, { signal: controller.signal });
|
|
437
|
+
if (!response.ok || typeof response.arrayBuffer !== 'function') {
|
|
438
|
+
return null;
|
|
439
|
+
}
|
|
440
|
+
if (declaredSizeTooLarge(response)) {
|
|
441
|
+
log(`[extract] apk at ${source.apk_url} exceeds ${MAX_APK_BYTES} bytes; refusing`);
|
|
442
|
+
return null;
|
|
443
|
+
}
|
|
444
|
+
const bytes = Buffer.from(await response.arrayBuffer());
|
|
445
|
+
return bytes.length > MAX_APK_BYTES ? null : bytes;
|
|
446
|
+
}
|
|
447
|
+
catch {
|
|
448
|
+
return null;
|
|
449
|
+
}
|
|
450
|
+
finally {
|
|
451
|
+
clearTimeout(timer);
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
const site = String(source.official_site ?? '').trim().replace(/\/+$/, '');
|
|
455
|
+
if (!site) {
|
|
456
|
+
return null;
|
|
457
|
+
}
|
|
458
|
+
// The site names its download chunk; that chunk points at a cdn.js which
|
|
459
|
+
// declares the download hosts and the archive cadence.
|
|
460
|
+
const home = await fetchText(fetchImpl, `${site}/`, timeoutMs, egress);
|
|
461
|
+
if (!home.text) {
|
|
462
|
+
log(`[extract] official site unreachable: ${site}`);
|
|
463
|
+
return null;
|
|
464
|
+
}
|
|
465
|
+
const hosts = [];
|
|
466
|
+
let interval = 0;
|
|
467
|
+
let prefix = '';
|
|
468
|
+
const consumeChunk = async (chunkText, chunkUrl) => {
|
|
469
|
+
// Some brands hardcode the download origin and slot formula in the chunk.
|
|
470
|
+
// The helper keeps the published scheme and port, so nothing is rewritten.
|
|
471
|
+
const inline = inlineDownloadSpec(chunkText);
|
|
472
|
+
if (inline) {
|
|
473
|
+
hosts.push(inline.host);
|
|
474
|
+
prefix = prefix || inline.prefix;
|
|
475
|
+
if (inline.interval_minutes > 0) {
|
|
476
|
+
interval = interval || inline.interval_minutes;
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
// Others load a `cdn.js` that declares the hosts and the archive cadence.
|
|
480
|
+
const cdnRefs = [
|
|
481
|
+
...chunkText.matchAll(/["'`]([^"'`]*cdn[^"'`]*\.js[^"'`]*)["'`]/gi)
|
|
482
|
+
].map((match) => match[1]);
|
|
483
|
+
for (const reference of cdnRefs) {
|
|
484
|
+
let cdnUrl;
|
|
485
|
+
try {
|
|
486
|
+
cdnUrl = new URL(reference, chunkUrl).toString();
|
|
487
|
+
}
|
|
488
|
+
catch {
|
|
489
|
+
continue;
|
|
490
|
+
}
|
|
491
|
+
const cdn = await fetchText(fetchImpl, cdnUrl, timeoutMs, egress);
|
|
492
|
+
if (!cdn.text) {
|
|
493
|
+
continue;
|
|
494
|
+
}
|
|
495
|
+
const config = parseCdnConfig(cdn.text);
|
|
496
|
+
hosts.push(...config.download_hosts);
|
|
497
|
+
if (config.apk_interval_minutes) {
|
|
498
|
+
interval = interval || config.apk_interval_minutes;
|
|
499
|
+
}
|
|
500
|
+
}
|
|
501
|
+
};
|
|
502
|
+
// The page's own entry bundle references its lazy route chunks; the Android
|
|
503
|
+
// downloader lives in one of those, so the dep map has to be followed.
|
|
504
|
+
const seenChunks = new Set();
|
|
505
|
+
const pending = androidChunkUrls(home.text, site);
|
|
506
|
+
while (pending.length > 0 && hosts.length === 0) {
|
|
507
|
+
const chunkUrl = pending.shift();
|
|
508
|
+
if (seenChunks.has(chunkUrl)) {
|
|
509
|
+
continue;
|
|
510
|
+
}
|
|
511
|
+
seenChunks.add(chunkUrl);
|
|
512
|
+
const chunk = await fetchText(fetchImpl, chunkUrl, timeoutMs, egress);
|
|
513
|
+
if (!chunk.text) {
|
|
514
|
+
continue;
|
|
515
|
+
}
|
|
516
|
+
await consumeChunk(chunk.text, chunkUrl);
|
|
517
|
+
if (hosts.length === 0) {
|
|
518
|
+
// Follow one level of chunk references to reach the actual downloader.
|
|
519
|
+
for (const nested of chunk.text.matchAll(/["'`](\.\/?[\w.-]+\.js|\.\.\/[\w.-]+\.js)["'`]/g)) {
|
|
520
|
+
try {
|
|
521
|
+
const next = new URL(nested[1], chunkUrl).toString();
|
|
522
|
+
if (!seenChunks.has(next)) {
|
|
523
|
+
pending.push(next);
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
catch {
|
|
527
|
+
// Ignore unresolvable references.
|
|
528
|
+
}
|
|
529
|
+
}
|
|
530
|
+
}
|
|
531
|
+
}
|
|
532
|
+
// Fall back to the root `cdn.js`, which two of the brands serve directly.
|
|
533
|
+
if (hosts.length === 0) {
|
|
534
|
+
for (const cdnUrl of cdnConfigUrls(site, now.getTime())) {
|
|
535
|
+
const cdn = await fetchText(fetchImpl, cdnUrl, timeoutMs, egress);
|
|
536
|
+
if (!cdn.text) {
|
|
537
|
+
continue;
|
|
538
|
+
}
|
|
539
|
+
const config = parseCdnConfig(cdn.text);
|
|
540
|
+
hosts.push(...config.download_hosts);
|
|
541
|
+
if (config.apk_interval_minutes) {
|
|
542
|
+
interval = interval || config.apk_interval_minutes;
|
|
543
|
+
}
|
|
544
|
+
if (hosts.length) {
|
|
545
|
+
break;
|
|
546
|
+
}
|
|
547
|
+
}
|
|
548
|
+
}
|
|
549
|
+
if (!hosts.length) {
|
|
550
|
+
log(`[extract] no download host found on ${site}`);
|
|
551
|
+
return null;
|
|
552
|
+
}
|
|
553
|
+
// The archive filename prefix (`20261108_lt463.apk`) is not published by every
|
|
554
|
+
// site. It is derived from the site host when it is not in the chunk, and the
|
|
555
|
+
// per-brand table covers the case where neither is available.
|
|
556
|
+
if (!prefix) {
|
|
557
|
+
prefix = BRAND_APK_PREFIXES[brandName] ?? BRAND_APK_PREFIXES[siteHost(site)] ?? 'apk';
|
|
558
|
+
}
|
|
559
|
+
if (!interval) {
|
|
560
|
+
interval = 1;
|
|
561
|
+
}
|
|
562
|
+
for (const candidate of candidateApkUrls([...new Set(hosts)], prefix, now, interval)) {
|
|
563
|
+
const controller = new AbortController();
|
|
564
|
+
const timer = setTimeout(() => controller.abort(), Math.max(timeoutMs, 120_000));
|
|
565
|
+
try {
|
|
566
|
+
const response = await fetchImpl(candidate, { signal: controller.signal });
|
|
567
|
+
if (!response.ok || typeof response.arrayBuffer !== 'function') {
|
|
568
|
+
continue;
|
|
569
|
+
}
|
|
570
|
+
if (declaredSizeTooLarge(response)) {
|
|
571
|
+
continue;
|
|
572
|
+
}
|
|
573
|
+
const bytes = Buffer.from(await response.arrayBuffer());
|
|
574
|
+
if (bytes.length > MAX_APK_BYTES) {
|
|
575
|
+
continue;
|
|
576
|
+
}
|
|
577
|
+
if (bytes.length > 1_000_000 && bytes[0] === 0x50 && bytes[1] === 0x4b) {
|
|
578
|
+
log(`[extract] apk ${candidate} (${(bytes.length / 1_048_576).toFixed(1)} MB)`);
|
|
579
|
+
return bytes;
|
|
580
|
+
}
|
|
581
|
+
}
|
|
582
|
+
catch {
|
|
583
|
+
// Try the next slot.
|
|
584
|
+
}
|
|
585
|
+
finally {
|
|
586
|
+
clearTimeout(timer);
|
|
587
|
+
}
|
|
588
|
+
}
|
|
589
|
+
log(`[extract] no APK published for ${prefix} near slot ${apkSlot(now, interval)}`);
|
|
590
|
+
return null;
|
|
591
|
+
}
|
|
592
|
+
export function resolveCredentialPlatforms(source) {
|
|
593
|
+
const requested = Array.isArray(source.platforms) ? source.platforms : [];
|
|
594
|
+
const unique = [...new Set(requested.filter((value) => value === 'android' || value === 'ios'))];
|
|
595
|
+
// The product contract is both client channels, concurrently. A legacy
|
|
596
|
+
// `platform = "ios"` row is not a choice any more; only the internal
|
|
597
|
+
// `platforms` override can narrow a run for tests or protocol debugging.
|
|
598
|
+
return unique.length > 0 ? unique : ['android', 'ios'];
|
|
599
|
+
}
|
|
600
|
+
/** A transport failure or stale signing cert must not retire a good account. */
|
|
601
|
+
/**
|
|
602
|
+
* Throttling and session wording, which mentions the account without meaning
|
|
603
|
+
* the password is wrong.
|
|
604
|
+
*
|
|
605
|
+
* A vendor rate limit reads `账号登录过于频繁`; retiring the account for that
|
|
606
|
+
* burns a healthy credential and moves to the next one, which is then also
|
|
607
|
+
* throttled, until the whole brand's pool is marked invalid.
|
|
608
|
+
*/
|
|
609
|
+
const TRANSIENT_LOGIN_PATTERN = /频繁|頻繁|过于|過於|稍后|稍後|稍候|重试|重試|too\s*many|rate\s*limit|frequency|try\s*(again|later)|timeout|超时|超時|繁忙|busy|服务器|服務器|server\s*error|network|网络|網絡/i;
|
|
610
|
+
/**
|
|
611
|
+
* True only when the vendor said the credential itself is wrong.
|
|
612
|
+
*
|
|
613
|
+
* Conservative on purpose: a wrong classification retires a working account,
|
|
614
|
+
* while a missed one merely fails the run, which the retry path already covers.
|
|
615
|
+
*/
|
|
616
|
+
export function isCredentialLoginRejection(message) {
|
|
617
|
+
const text = String(message ?? '');
|
|
618
|
+
if (!text.trim())
|
|
619
|
+
return false;
|
|
620
|
+
if (TRANSIENT_LOGIN_PATTERN.test(text))
|
|
621
|
+
return false;
|
|
622
|
+
return /密码错误|密碼錯誤|密码不正确|密碼不正確|账号不存在|帳號不存在|账户不存在|账户或密码|账号或密码|账号已被|帳號已被|密码已|被封禁|已被禁用|登录失败|登錄失敗|does\s*not\s*exist|not\s*(found|exist)|invalid\s*(password|account|credential|user)|incorrect\s*password|wrong\s*password|unauthorized/i.test(text);
|
|
623
|
+
}
|
|
624
|
+
export function membershipDate(value) {
|
|
625
|
+
if (typeof value === 'number' && Number.isFinite(value) && value > 0) {
|
|
626
|
+
const ms = value > 10_000_000_000 ? value : value * 1000;
|
|
627
|
+
const date = new Date(ms);
|
|
628
|
+
return Number.isNaN(date.getTime()) ? '' : date.toISOString().slice(0, 10);
|
|
629
|
+
}
|
|
630
|
+
const text = String(value ?? '').trim();
|
|
631
|
+
if (!text)
|
|
632
|
+
return '';
|
|
633
|
+
if (/^\d+$/.test(text)) {
|
|
634
|
+
// A bare number too short to be an epoch is a placeholder, not a date:
|
|
635
|
+
// `new Date('0')` would otherwise read as the year 2000.
|
|
636
|
+
return /^\d{10,13}$/.test(text) ? membershipDate(Number(text)) : '';
|
|
637
|
+
}
|
|
638
|
+
const parsed = new Date(text);
|
|
639
|
+
return Number.isNaN(parsed.getTime()) ? '' : parsed.toISOString().slice(0, 10);
|
|
640
|
+
}
|
|
641
|
+
function credentialRejectionMessage(payload) {
|
|
642
|
+
return String(payload.msg ?? payload.message ?? '');
|
|
643
|
+
}
|
|
644
|
+
/**
|
|
645
|
+
* Pool-backed brand extraction. Android and iOS run concurrently against the
|
|
646
|
+
* same leased account; their links merge into the one source stream the rest of
|
|
647
|
+
* the pipeline already consumes.
|
|
648
|
+
*/
|
|
649
|
+
export async function fetchCredentialSourceLinks(input, options = {}) {
|
|
650
|
+
const proxyUrl = upstreamProxyUrl(options.env);
|
|
651
|
+
if (!proxyUrl) {
|
|
652
|
+
return fetchCredentialSourceLinksWithTransport(input, options);
|
|
653
|
+
}
|
|
654
|
+
let proxy;
|
|
655
|
+
try {
|
|
656
|
+
proxy = new URL(proxyUrl);
|
|
657
|
+
}
|
|
658
|
+
catch {
|
|
659
|
+
throw new Error('The configured upstream proxy URL is invalid');
|
|
660
|
+
}
|
|
661
|
+
if (proxy.protocol !== 'http:' && proxy.protocol !== 'https:') {
|
|
662
|
+
throw new Error('Credential sources require an HTTP or HTTPS upstream proxy');
|
|
663
|
+
}
|
|
664
|
+
// The proxy previously reached only a text fallback: native site and APK
|
|
665
|
+
// requests still went direct. Keep one source-scoped dispatcher for both
|
|
666
|
+
// response types, without changing the per-node measurement transports.
|
|
667
|
+
const dispatcher = new ProxyAgent({ uri: proxy.href, proxyTunnel: false });
|
|
668
|
+
const fetchImpl = (url, init) => options.fetchImpl
|
|
669
|
+
? options.fetchImpl(url, { ...init, dispatcher })
|
|
670
|
+
: upstreamFetch(url, { ...init, dispatcher });
|
|
671
|
+
try {
|
|
672
|
+
return await fetchCredentialSourceLinksWithTransport(input, { ...options, fetchImpl });
|
|
673
|
+
}
|
|
674
|
+
finally {
|
|
675
|
+
await dispatcher.destroy();
|
|
676
|
+
}
|
|
677
|
+
}
|
|
678
|
+
async function fetchCredentialSourceLinksWithTransport(input, options) {
|
|
679
|
+
const source = input.source;
|
|
680
|
+
const platforms = resolveCredentialPlatforms(source);
|
|
681
|
+
const policy = resolveRoundPolicy(source);
|
|
682
|
+
const errors = [];
|
|
683
|
+
const diagnostics = {
|
|
684
|
+
brand: input.source_name,
|
|
685
|
+
platform: platforms.length === 1 ? platforms[0] : 'android+ios',
|
|
686
|
+
platforms,
|
|
687
|
+
apk_url: '',
|
|
688
|
+
package_name: '',
|
|
689
|
+
version_name: '',
|
|
690
|
+
seed_domains: [],
|
|
691
|
+
api_domain: '',
|
|
692
|
+
route_count: 0,
|
|
693
|
+
node_count: 0,
|
|
694
|
+
placeholder_count: 0,
|
|
695
|
+
rounds: 0,
|
|
696
|
+
requests: 0,
|
|
697
|
+
errors
|
|
698
|
+
};
|
|
699
|
+
const empty = {
|
|
700
|
+
source_name: input.source_name,
|
|
701
|
+
requested_iterations: policy.max_iterations,
|
|
702
|
+
successful_iterations: 0,
|
|
703
|
+
failed_iterations: 0,
|
|
704
|
+
links: [],
|
|
705
|
+
diagnostics
|
|
706
|
+
};
|
|
707
|
+
event(options, 'extract_source_started', {
|
|
708
|
+
source_name: input.source_name,
|
|
709
|
+
requested_iterations: policy.max_iterations,
|
|
710
|
+
min_iterations: policy.min_iterations,
|
|
711
|
+
resume_from_iteration: 1,
|
|
712
|
+
platforms
|
|
713
|
+
});
|
|
714
|
+
const explicitAccount = String(source.account ?? '').trim();
|
|
715
|
+
const explicitPassword = String(source.password ?? '').trim();
|
|
716
|
+
const brand = normalizeBrand(input.source_name)
|
|
717
|
+
?? brandFromSite(source.official_site)
|
|
718
|
+
?? normalizeBrand(credentialBrandKey(input.source_name, source));
|
|
719
|
+
let store = options.accountStore ?? null;
|
|
720
|
+
let ownsStore = false;
|
|
721
|
+
let leased = null;
|
|
722
|
+
let account = explicitAccount;
|
|
723
|
+
let password = explicitPassword;
|
|
724
|
+
if (!account || !password) {
|
|
725
|
+
if (!brand) {
|
|
726
|
+
errors.push('account pool brand could not be resolved');
|
|
727
|
+
}
|
|
728
|
+
else {
|
|
729
|
+
store = options.accountStore ?? new AccountStore({ env: options.env, now: options.now });
|
|
730
|
+
ownsStore = !options.accountStore;
|
|
731
|
+
leased = store.lease(brand);
|
|
732
|
+
if (leased) {
|
|
733
|
+
account = leased.account;
|
|
734
|
+
password = leased.password;
|
|
735
|
+
}
|
|
736
|
+
else {
|
|
737
|
+
errors.push(`account pool has no active ${brand} account`);
|
|
738
|
+
}
|
|
739
|
+
}
|
|
740
|
+
}
|
|
741
|
+
if (!account || !password) {
|
|
742
|
+
if (ownsStore)
|
|
743
|
+
store?.close();
|
|
744
|
+
event(options, 'extract_source_completed', {
|
|
745
|
+
source_name: input.source_name,
|
|
746
|
+
requested_iterations: policy.max_iterations,
|
|
747
|
+
successful_iterations: 0,
|
|
748
|
+
failed_iterations: 1,
|
|
749
|
+
raw_links: 0
|
|
750
|
+
});
|
|
751
|
+
return { ...empty, failed_iterations: 1 };
|
|
752
|
+
}
|
|
753
|
+
const memo = options.memo ?? createFingerprintMemo(options.fingerprintStore ?? null, credentialBrandKey(input.source_name, source));
|
|
754
|
+
let linkQueue = Promise.resolve();
|
|
755
|
+
// A non-numeric retry limit (a hand-edited `account_retry_limit = "three"`)
|
|
756
|
+
// must not turn the attempt loop into zero passes, which reported the source
|
|
757
|
+
// as successful with no links and no error.
|
|
758
|
+
const retryLimit = Math.trunc(Number(source.account_retry_limit ?? 3));
|
|
759
|
+
const attempts = explicitAccount && explicitPassword ? 1 : (Number.isFinite(retryLimit) && retryLimit >= 1 ? retryLimit : 3);
|
|
760
|
+
let results = [];
|
|
761
|
+
let callbackFailure = null;
|
|
762
|
+
// The dashboard reads per-source totals from one `extract_iteration` stream.
|
|
763
|
+
// Both platforms emit their own, so they are merged here. Raw links use the
|
|
764
|
+
// exact-string identity each platform already dedupes with: summing the
|
|
765
|
+
// per-platform counts would report the same link twice and leave the card
|
|
766
|
+
// above the final union for the rest of the run. Dedupe keeps the canonical
|
|
767
|
+
// fingerprint union, the same identity the global dedupe card counts.
|
|
768
|
+
const liveLinkIds = new Set();
|
|
769
|
+
const liveFingerprints = new Set();
|
|
770
|
+
try {
|
|
771
|
+
for (let attempt = 1; attempt <= attempts; attempt += 1) {
|
|
772
|
+
liveLinkIds.clear();
|
|
773
|
+
liveFingerprints.clear();
|
|
774
|
+
results = await Promise.all(platforms.map((platform) => {
|
|
775
|
+
const scoped = {
|
|
776
|
+
...options,
|
|
777
|
+
memo,
|
|
778
|
+
eventCallback: (type, payload) => {
|
|
779
|
+
if (type === 'extract_source_started') {
|
|
780
|
+
event(options, 'extract_platform_started', { ...payload, platform });
|
|
781
|
+
return;
|
|
782
|
+
}
|
|
783
|
+
if (type === 'extract_source_completed') {
|
|
784
|
+
event(options, 'extract_platform_completed', { ...payload, platform });
|
|
785
|
+
return;
|
|
786
|
+
}
|
|
787
|
+
if (type === 'extract_iteration') {
|
|
788
|
+
const fingerprints = Array.isArray(payload.new_item_fingerprints)
|
|
789
|
+
? payload.new_item_fingerprints.map((value) => String(value)).filter(Boolean)
|
|
790
|
+
: [];
|
|
791
|
+
for (const fingerprint of fingerprints)
|
|
792
|
+
liveFingerprints.add(fingerprint);
|
|
793
|
+
const linkIds = Array.isArray(payload.new_item_link_ids)
|
|
794
|
+
? payload.new_item_link_ids.map((value) => String(value)).filter(Boolean)
|
|
795
|
+
: [];
|
|
796
|
+
for (const id of linkIds)
|
|
797
|
+
liveLinkIds.add(id);
|
|
798
|
+
event(options, 'extract_iteration', {
|
|
799
|
+
source_name: input.source_name,
|
|
800
|
+
iteration: Number(payload.iteration ?? 0),
|
|
801
|
+
requested_iterations: policy.max_iterations,
|
|
802
|
+
new_items: Number(payload.new_items ?? 0),
|
|
803
|
+
extracted_links: Number(payload.extracted_links ?? 0),
|
|
804
|
+
total_links: liveLinkIds.size,
|
|
805
|
+
deduped_links: liveFingerprints.size,
|
|
806
|
+
new_item_fingerprints: fingerprints
|
|
807
|
+
});
|
|
808
|
+
return;
|
|
809
|
+
}
|
|
810
|
+
if (type === 'log') {
|
|
811
|
+
// Four brands × two channels run at once; a bare `[extract]`
|
|
812
|
+
// line is unattributable in the log center.
|
|
813
|
+
event(options, 'log', {
|
|
814
|
+
message: String(payload.message ?? '').replace(/^\[extract\]/, `[extract][${input.source_name}][${platform}]`)
|
|
815
|
+
});
|
|
816
|
+
return;
|
|
817
|
+
}
|
|
818
|
+
event(options, type, { ...payload, platform });
|
|
819
|
+
},
|
|
820
|
+
linksCallback: (links) => {
|
|
821
|
+
// SQLite writes are synchronous, but the downstream speed-test
|
|
822
|
+
// submission is async. Serialize callbacks so two platforms cannot
|
|
823
|
+
// interleave a partially processed batch. A downstream failure is
|
|
824
|
+
// recorded and re-raised at the end instead of swallowed: the run
|
|
825
|
+
// must not report a batch as delivered when the speed pool rejected
|
|
826
|
+
// it, and the remaining rounds must stop feeding a dead chain.
|
|
827
|
+
linkQueue = linkQueue.then(async () => {
|
|
828
|
+
if (callbackFailure)
|
|
829
|
+
return;
|
|
830
|
+
try {
|
|
831
|
+
await options.linksCallback?.(links);
|
|
832
|
+
}
|
|
833
|
+
catch (error) {
|
|
834
|
+
callbackFailure = error instanceof Error ? error : new Error(String(error));
|
|
835
|
+
}
|
|
836
|
+
});
|
|
837
|
+
return linkQueue;
|
|
838
|
+
}
|
|
839
|
+
};
|
|
840
|
+
return fetchCredentialPlatformLinks({
|
|
841
|
+
source_name: input.source_name,
|
|
842
|
+
source: { ...source, account, password, platform, platforms: [platform] }
|
|
843
|
+
}, scoped);
|
|
844
|
+
}));
|
|
845
|
+
const loginOk = results.some((result) => result.diagnostics.login_ok);
|
|
846
|
+
if (loginOk) {
|
|
847
|
+
if (store && leased) {
|
|
848
|
+
store.markSuccess(leased.id);
|
|
849
|
+
const expires = results.map((result) => result.diagnostics.membership_expires_at ?? '').find(Boolean);
|
|
850
|
+
if (expires)
|
|
851
|
+
store.setExpiry(leased.brand, leased.account, expires);
|
|
852
|
+
const level = results.map((result) => result.diagnostics.membership_level).find((value) => value !== undefined);
|
|
853
|
+
const linked = results.some((result) => result.links.length > 0);
|
|
854
|
+
// A free-tier account (level 0/1) only ever produces placeholders.
|
|
855
|
+
// Retire it, but only when it actually produced nothing: a paid
|
|
856
|
+
// account misreported as level 1 must not lose its row.
|
|
857
|
+
if (level !== undefined && level < 2 && !linked)
|
|
858
|
+
store.markInvalid(leased.id, 'no paid membership');
|
|
859
|
+
}
|
|
860
|
+
break;
|
|
861
|
+
}
|
|
862
|
+
const credentialRejected = results.length > 0 && results.every((result) => result.diagnostics.credential_rejected);
|
|
863
|
+
if (store && leased && credentialRejected) {
|
|
864
|
+
store.markInvalid(leased.id, 'login rejected');
|
|
865
|
+
if (attempt < attempts && brand) {
|
|
866
|
+
leased = store.lease(brand);
|
|
867
|
+
if (leased) {
|
|
868
|
+
account = leased.account;
|
|
869
|
+
password = leased.password;
|
|
870
|
+
continue;
|
|
871
|
+
}
|
|
872
|
+
}
|
|
873
|
+
}
|
|
874
|
+
break;
|
|
875
|
+
}
|
|
876
|
+
}
|
|
877
|
+
finally {
|
|
878
|
+
if (ownsStore)
|
|
879
|
+
store?.close();
|
|
880
|
+
}
|
|
881
|
+
await linkQueue;
|
|
882
|
+
const links = [];
|
|
883
|
+
const seen = new Set();
|
|
884
|
+
for (const result of results) {
|
|
885
|
+
for (const link of result.links) {
|
|
886
|
+
if (!seen.has(link)) {
|
|
887
|
+
seen.add(link);
|
|
888
|
+
links.push(link);
|
|
889
|
+
}
|
|
890
|
+
}
|
|
891
|
+
for (const error of result.diagnostics.errors) {
|
|
892
|
+
errors.push(`[${result.diagnostics.platform}] ${error}`);
|
|
893
|
+
}
|
|
894
|
+
}
|
|
895
|
+
const first = results[0]?.diagnostics;
|
|
896
|
+
if (first) {
|
|
897
|
+
diagnostics.apk_url = first.apk_url;
|
|
898
|
+
diagnostics.package_name = first.package_name;
|
|
899
|
+
diagnostics.version_name = first.version_name;
|
|
900
|
+
diagnostics.seed_domains = first.seed_domains;
|
|
901
|
+
diagnostics.api_domain = first.api_domain;
|
|
902
|
+
}
|
|
903
|
+
diagnostics.route_count = Math.max(0, ...results.map((result) => result.diagnostics.route_count));
|
|
904
|
+
diagnostics.route_tables = Math.max(0, ...results.map((result) => result.diagnostics.route_tables ?? 0));
|
|
905
|
+
diagnostics.node_count = links.length;
|
|
906
|
+
diagnostics.placeholder_count = results.reduce((total, result) => total + result.diagnostics.placeholder_count, 0);
|
|
907
|
+
diagnostics.rounds = Math.max(0, ...results.map((result) => result.diagnostics.rounds));
|
|
908
|
+
diagnostics.requests = results.reduce((total, result) => total + result.diagnostics.requests, 0);
|
|
909
|
+
diagnostics.platform_results = results.map((result) => ({
|
|
910
|
+
platform: result.diagnostics.platform,
|
|
911
|
+
links: result.links.length,
|
|
912
|
+
rounds: result.diagnostics.rounds,
|
|
913
|
+
requests: result.diagnostics.requests,
|
|
914
|
+
errors: result.diagnostics.errors
|
|
915
|
+
}));
|
|
916
|
+
const successful = Math.max(0, ...results.map((result) => result.successful_iterations));
|
|
917
|
+
const failed = results.length === 0 ? 1 : Math.max(0, ...results.map((result) => result.failed_iterations));
|
|
918
|
+
if (callbackFailure) {
|
|
919
|
+
// Surfaced rather than logged: the orchestrator treats a thrown extract as
|
|
920
|
+
// a failed source, which is the truth — its downstream already died.
|
|
921
|
+
throw callbackFailure;
|
|
922
|
+
}
|
|
923
|
+
event(options, 'extract_source_completed', {
|
|
924
|
+
source_name: input.source_name,
|
|
925
|
+
requested_iterations: policy.max_iterations,
|
|
926
|
+
successful_iterations: successful,
|
|
927
|
+
failed_iterations: failed,
|
|
928
|
+
raw_links: links.length,
|
|
929
|
+
platforms
|
|
930
|
+
});
|
|
931
|
+
return {
|
|
932
|
+
source_name: input.source_name,
|
|
933
|
+
requested_iterations: policy.max_iterations,
|
|
934
|
+
successful_iterations: successful,
|
|
935
|
+
failed_iterations: failed,
|
|
936
|
+
links,
|
|
937
|
+
diagnostics
|
|
938
|
+
};
|
|
939
|
+
}
|
|
940
|
+
/** One client channel. The public wrapper runs both and merges the results. */
|
|
941
|
+
export async function fetchCredentialPlatformLinks(input, options = {}) {
|
|
942
|
+
const source = input.source;
|
|
943
|
+
const fetchImpl = options.fetchImpl ?? globalThis.fetch?.bind(globalThis);
|
|
944
|
+
const now = (options.now ?? (() => new Date()))();
|
|
945
|
+
const timeoutMs = Number(source.timeout_ms ?? DEFAULT_TIMEOUT_MS);
|
|
946
|
+
const platform = source.platform === 'ios' ? 'ios' : 'android';
|
|
947
|
+
const device = {
|
|
948
|
+
device_id: source.device_id ?? DEFAULT_DEVICE_ID,
|
|
949
|
+
install_date: source.install_date ?? today(now)
|
|
950
|
+
};
|
|
951
|
+
const errors = [];
|
|
952
|
+
const log = (message) => event(options, 'log', { message });
|
|
953
|
+
const diagnostics = {
|
|
954
|
+
brand: input.source_name,
|
|
955
|
+
platform,
|
|
956
|
+
apk_url: '',
|
|
957
|
+
package_name: '',
|
|
958
|
+
version_name: '',
|
|
959
|
+
seed_domains: [],
|
|
960
|
+
api_domain: '',
|
|
961
|
+
route_count: 0,
|
|
962
|
+
node_count: 0,
|
|
963
|
+
placeholder_count: 0,
|
|
964
|
+
rounds: 0,
|
|
965
|
+
requests: 0,
|
|
966
|
+
errors
|
|
967
|
+
};
|
|
968
|
+
const empty = {
|
|
969
|
+
source_name: input.source_name,
|
|
970
|
+
requested_iterations: 0,
|
|
971
|
+
successful_iterations: 0,
|
|
972
|
+
failed_iterations: 0,
|
|
973
|
+
links: [],
|
|
974
|
+
diagnostics
|
|
975
|
+
};
|
|
976
|
+
if (!fetchImpl) {
|
|
977
|
+
errors.push('fetch is unavailable');
|
|
978
|
+
return empty;
|
|
979
|
+
}
|
|
980
|
+
const account = String(source.account ?? '').trim();
|
|
981
|
+
const password = String(source.password ?? '').trim();
|
|
982
|
+
if (!account || !password) {
|
|
983
|
+
errors.push('account and password are required');
|
|
984
|
+
return empty;
|
|
985
|
+
}
|
|
986
|
+
event(options, 'extract_source_started', {
|
|
987
|
+
source_name: input.source_name,
|
|
988
|
+
requested_iterations: 0,
|
|
989
|
+
min_iterations: 0,
|
|
990
|
+
resume_from_iteration: 1
|
|
991
|
+
});
|
|
992
|
+
// 1) Brand fingerprint: package, version, certificate and seed domains.
|
|
993
|
+
//
|
|
994
|
+
// These only change when the vendor publishes a new build, so a cached entry
|
|
995
|
+
// is used first and the APK is inspected only when the cached seeds all stop
|
|
996
|
+
// answering. That keeps a normal run to a handful of HTTP calls.
|
|
997
|
+
const brandKey = credentialBrandKey(input.source_name, source);
|
|
998
|
+
const store = options.fingerprintStore ?? null;
|
|
999
|
+
const memo = options.memo ?? createFingerprintMemo(store, brandKey);
|
|
1000
|
+
// A fingerprint past its trust window is still used when its seeds answer,
|
|
1001
|
+
// but it is re-verified against the APK rather than trusted indefinitely: a
|
|
1002
|
+
// vendor can rotate the signing certificate while the old seeds still serve
|
|
1003
|
+
// `/api/eversion`, and then every login fails on a stale certificate.
|
|
1004
|
+
const fresh = store?.readFresh(brandKey) ?? null;
|
|
1005
|
+
const storedEntry = store?.read(brandKey) ?? null;
|
|
1006
|
+
const staleButUsable = Boolean(storedEntry) && !fresh;
|
|
1007
|
+
const cached = staleButUsable ? storedEntry : memo.get();
|
|
1008
|
+
const egress = { env: options.env, curlFetch: options.curlFetch };
|
|
1009
|
+
// Report real extraction work instead of making the UI infer APK/domain
|
|
1010
|
+
// activity from text logs. Cache reuse remains distinct from executed work.
|
|
1011
|
+
const substage = (name, status) => event(options, 'extract_substage', { source_name: input.source_name, platform, substage: name, status });
|
|
1012
|
+
const inspectApk = async () => {
|
|
1013
|
+
substage('apk_download', 'running');
|
|
1014
|
+
let apkBytes = null;
|
|
1015
|
+
try {
|
|
1016
|
+
apkBytes = options.loadApk
|
|
1017
|
+
? await options.loadApk(source)
|
|
1018
|
+
: await defaultLoadApk(source, fetchImpl, timeoutMs, now, log, input.source_name, egress);
|
|
1019
|
+
}
|
|
1020
|
+
catch (error) {
|
|
1021
|
+
errors.push(`apk: ${error instanceof Error ? error.message : String(error)}`);
|
|
1022
|
+
}
|
|
1023
|
+
if (!apkBytes) {
|
|
1024
|
+
substage('apk_download', 'failed');
|
|
1025
|
+
errors.push('APK could not be located');
|
|
1026
|
+
return null;
|
|
1027
|
+
}
|
|
1028
|
+
substage('apk_download', 'success');
|
|
1029
|
+
substage('apk_analysis', 'running');
|
|
1030
|
+
let discovered;
|
|
1031
|
+
try {
|
|
1032
|
+
discovered = apkDiscoveryFromBytes(apkBytes, source.apk_url ?? '');
|
|
1033
|
+
}
|
|
1034
|
+
catch (error) {
|
|
1035
|
+
substage('apk_analysis', 'failed');
|
|
1036
|
+
throw error;
|
|
1037
|
+
}
|
|
1038
|
+
substage('apk_analysis', discovered.seed_domains.length > 0 ? 'success' : 'failed');
|
|
1039
|
+
log(`[extract] ${discovered.package_name} ${discovered.version_name} seeds=${discovered.seed_domains.length}`);
|
|
1040
|
+
return discovered.seed_domains.length > 0 ? discovered : null;
|
|
1041
|
+
};
|
|
1042
|
+
const inspectSharedApk = () => (memo.resolve ? memo.resolve(inspectApk) : inspectApk());
|
|
1043
|
+
// `apk_url` / `apk_path` are an explicit override: when either is set the
|
|
1044
|
+
// caller is asserting a specific build, so a cached fingerprint must not
|
|
1045
|
+
// shadow it. Otherwise the cache would make the override a no-op.
|
|
1046
|
+
const hasApkOverride = Boolean(String(source.apk_url ?? '').trim() || String(source.apk_path ?? '').trim());
|
|
1047
|
+
let discovery = hasApkOverride ? null : cached;
|
|
1048
|
+
let keyCandidates = credentialKeyCandidates(source, discovery ?? undefined);
|
|
1049
|
+
let eversion = null;
|
|
1050
|
+
let cachedInUse = false;
|
|
1051
|
+
if (discovery) {
|
|
1052
|
+
substage('apk_download', 'cached');
|
|
1053
|
+
substage('apk_analysis', 'cached');
|
|
1054
|
+
log(`[extract] cached fingerprint ${discovery.package_name} ${discovery.version_name}`);
|
|
1055
|
+
eversion = await tryEversion(discovery, keyCandidates);
|
|
1056
|
+
if (eversion) {
|
|
1057
|
+
// An entry past its trust window is re-verified against the APK even
|
|
1058
|
+
// though its seeds answered: the vendor can rotate the signing
|
|
1059
|
+
// certificate while the old seeds keep serving `/api/eversion`, and a
|
|
1060
|
+
// stale certificate fails every login with no other signal.
|
|
1061
|
+
if (staleButUsable) {
|
|
1062
|
+
log('[extract] cached fingerprint is past its trust window; re-inspecting the APK');
|
|
1063
|
+
const refreshed = await inspectSharedApk();
|
|
1064
|
+
if (refreshed) {
|
|
1065
|
+
const refreshedEversion = await tryEversion(refreshed, credentialKeyCandidates(source, refreshed));
|
|
1066
|
+
if (refreshedEversion) {
|
|
1067
|
+
discovery = refreshed;
|
|
1068
|
+
eversion = refreshedEversion;
|
|
1069
|
+
keyCandidates = credentialKeyCandidates(source, refreshed);
|
|
1070
|
+
memo.set(refreshed);
|
|
1071
|
+
store?.write(brandKey, refreshed);
|
|
1072
|
+
}
|
|
1073
|
+
}
|
|
1074
|
+
if (discovery === cached) {
|
|
1075
|
+
store?.touch(brandKey);
|
|
1076
|
+
}
|
|
1077
|
+
cachedInUse = true;
|
|
1078
|
+
}
|
|
1079
|
+
else {
|
|
1080
|
+
store?.touch(brandKey);
|
|
1081
|
+
cachedInUse = true;
|
|
1082
|
+
}
|
|
1083
|
+
}
|
|
1084
|
+
else {
|
|
1085
|
+
log('[extract] cached seeds stopped answering; re-inspecting the APK');
|
|
1086
|
+
discovery = null;
|
|
1087
|
+
}
|
|
1088
|
+
}
|
|
1089
|
+
if (!discovery) {
|
|
1090
|
+
discovery = await inspectSharedApk();
|
|
1091
|
+
if (!discovery) {
|
|
1092
|
+
errors.push('APK could not be located');
|
|
1093
|
+
return { ...empty, failed_iterations: 1 };
|
|
1094
|
+
}
|
|
1095
|
+
keyCandidates = credentialKeyCandidates(source, discovery);
|
|
1096
|
+
eversion = await tryEversion(discovery, keyCandidates);
|
|
1097
|
+
if (!eversion) {
|
|
1098
|
+
errors.push('eversion failed on every seed');
|
|
1099
|
+
return { ...empty, failed_iterations: 1 };
|
|
1100
|
+
}
|
|
1101
|
+
memo.set(discovery);
|
|
1102
|
+
store?.write(brandKey, discovery);
|
|
1103
|
+
}
|
|
1104
|
+
diagnostics.apk_url = discovery.apk_url;
|
|
1105
|
+
diagnostics.package_name = discovery.package_name;
|
|
1106
|
+
diagnostics.version_name = discovery.version_name;
|
|
1107
|
+
diagnostics.seed_domains = discovery.seed_domains;
|
|
1108
|
+
if (!eversion) {
|
|
1109
|
+
errors.push('eversion failed on every seed');
|
|
1110
|
+
return { ...empty, failed_iterations: 1 };
|
|
1111
|
+
}
|
|
1112
|
+
const identity = {
|
|
1113
|
+
seed_domains: discovery.seed_domains,
|
|
1114
|
+
package_name: discovery.package_name,
|
|
1115
|
+
version_name: discovery.version_name,
|
|
1116
|
+
cert_sha1: discovery.cert_sha1,
|
|
1117
|
+
protocol: discovery.protocol,
|
|
1118
|
+
core_version: discovery.core_version,
|
|
1119
|
+
lite_version: discovery.lite_version
|
|
1120
|
+
};
|
|
1121
|
+
/** Runs eversion for one candidate fingerprint. */
|
|
1122
|
+
async function tryEversion(candidate, keys) {
|
|
1123
|
+
substage('domain_discovery', 'running');
|
|
1124
|
+
const resolved = await resolveEversion(fetchImpl, candidate.seed_domains, (ts) => commonParams({
|
|
1125
|
+
identity: {
|
|
1126
|
+
seed_domains: candidate.seed_domains,
|
|
1127
|
+
package_name: candidate.package_name,
|
|
1128
|
+
version_name: candidate.version_name,
|
|
1129
|
+
cert_sha1: candidate.cert_sha1,
|
|
1130
|
+
protocol: candidate.protocol,
|
|
1131
|
+
core_version: candidate.core_version,
|
|
1132
|
+
lite_version: candidate.lite_version
|
|
1133
|
+
},
|
|
1134
|
+
device,
|
|
1135
|
+
platform,
|
|
1136
|
+
timestampMs: ts,
|
|
1137
|
+
code: ''
|
|
1138
|
+
}, resolveBrandLayout({ ...source, brand: input.source_name })), keys, timeoutMs);
|
|
1139
|
+
substage('domain_discovery', resolved ? 'success' : 'failed');
|
|
1140
|
+
return resolved;
|
|
1141
|
+
}
|
|
1142
|
+
// Event lines are not passed through the text redactor, so nothing derived
|
|
1143
|
+
// from the key may reach them — not even a prefix.
|
|
1144
|
+
log(`[extract] eversion ok key=<redacted> api_links=${eversion.api_links.length}`);
|
|
1145
|
+
const ctx = {
|
|
1146
|
+
identity,
|
|
1147
|
+
device,
|
|
1148
|
+
platform,
|
|
1149
|
+
layout: resolveBrandLayout({ ...source, brand: input.source_name }),
|
|
1150
|
+
token: '',
|
|
1151
|
+
// Before login the app's local KV has no `code`, so the request carries an
|
|
1152
|
+
// empty one; `&code=` only holds a value once the account has one.
|
|
1153
|
+
code: '',
|
|
1154
|
+
lastActiveDomain: eversion.domain,
|
|
1155
|
+
key: eversion.key,
|
|
1156
|
+
fetchImpl,
|
|
1157
|
+
timeoutMs
|
|
1158
|
+
};
|
|
1159
|
+
const apiDomain = selectApiDomain(discovery.seed_domains, device.device_id);
|
|
1160
|
+
diagnostics.api_domain = apiDomain;
|
|
1161
|
+
const eversionFields = eversionStrings(eversion.payload);
|
|
1162
|
+
// 3) `/myapi/eapilogin`: obtain the session token.
|
|
1163
|
+
const loginParams = `&phone=${encodeURIComponent(account)}&passwd=${encodeURIComponent(password)}&ver_code=&sign=${loginSignature(account, password)}`;
|
|
1164
|
+
const loginPlain = await getCommStr(ctx, eversionFields.apiLinks, apiDomain, '/myapi/eapilogin?', loginParams, discovery.seed_domains);
|
|
1165
|
+
if (!loginPlain) {
|
|
1166
|
+
errors.push('login returned nothing');
|
|
1167
|
+
return { ...empty, failed_iterations: 1 };
|
|
1168
|
+
}
|
|
1169
|
+
let token = '';
|
|
1170
|
+
try {
|
|
1171
|
+
const payload = JSON.parse(loginPlain);
|
|
1172
|
+
if (Number(payload.code) !== 0) {
|
|
1173
|
+
const message = credentialRejectionMessage(payload);
|
|
1174
|
+
diagnostics.login_rejected = true;
|
|
1175
|
+
diagnostics.credential_rejected = isCredentialLoginRejection(message);
|
|
1176
|
+
errors.push(`login rejected: code=${String(payload.code)}`);
|
|
1177
|
+
// A cached fingerprint whose seeds answer eversion but whose signing
|
|
1178
|
+
// cert the vendor rotated keeps failing here on every run. Drop the
|
|
1179
|
+
// cache so the next run re-inspects the APK instead of repeating this.
|
|
1180
|
+
if (cachedInUse) {
|
|
1181
|
+
store?.clear(brandKey);
|
|
1182
|
+
}
|
|
1183
|
+
return { ...empty, failed_iterations: 1 };
|
|
1184
|
+
}
|
|
1185
|
+
token = String(payload.token ?? '');
|
|
1186
|
+
diagnostics.membership_expires_at = membershipDate(payload.endtime ?? payload.expire_time);
|
|
1187
|
+
diagnostics.membership_level = Number.isFinite(Number(payload.level)) ? Number(payload.level) : undefined;
|
|
1188
|
+
}
|
|
1189
|
+
catch {
|
|
1190
|
+
errors.push('login response was not JSON');
|
|
1191
|
+
return { ...empty, failed_iterations: 1 };
|
|
1192
|
+
}
|
|
1193
|
+
if (!token) {
|
|
1194
|
+
errors.push('login returned no token');
|
|
1195
|
+
return { ...empty, failed_iterations: 1 };
|
|
1196
|
+
}
|
|
1197
|
+
diagnostics.login_ok = true;
|
|
1198
|
+
ctx.token = token;
|
|
1199
|
+
// `&code=` must be non-empty from here on: the server answers an empty body
|
|
1200
|
+
// when it is blank, so the value from the profile (or the app's default) is
|
|
1201
|
+
// used once a session exists.
|
|
1202
|
+
ctx.code = String(source.code ?? '').trim() || 'Z2E4QG';
|
|
1203
|
+
log('[extract] login ok');
|
|
1204
|
+
// `/myapi/eapiuserinfo` carries the real level and unlimit flags.
|
|
1205
|
+
try {
|
|
1206
|
+
const infoPlain = await getCommStr(ctx, eversionFields.apiLinks, apiDomain, '/myapi/eapiuserinfo', '', discovery.seed_domains);
|
|
1207
|
+
if (infoPlain) {
|
|
1208
|
+
const info = JSON.parse(infoPlain);
|
|
1209
|
+
const detail = (info.info ?? {});
|
|
1210
|
+
const codeInfo = (info.code_info ?? {});
|
|
1211
|
+
// An omitted `level` means "not reported", not "free tier": reading it as
|
|
1212
|
+
// 0 sent isvip=0 and then retired a paid account as having no membership.
|
|
1213
|
+
// Only a field the vendor actually sent counts as evidence, so the paid
|
|
1214
|
+
// defaults survive an older build or a proxy that strips `info`.
|
|
1215
|
+
const levelRaw = detail.level ?? info.level;
|
|
1216
|
+
const level = levelRaw === undefined || levelRaw === null || String(levelRaw).trim() === ''
|
|
1217
|
+
? undefined
|
|
1218
|
+
: Number(levelRaw);
|
|
1219
|
+
const unlimit = String(codeInfo.unlimit ?? '').trim();
|
|
1220
|
+
if (level !== undefined && Number.isFinite(level)) {
|
|
1221
|
+
ctx.isVip = level >= 2;
|
|
1222
|
+
diagnostics.membership_level = level;
|
|
1223
|
+
}
|
|
1224
|
+
if (unlimit)
|
|
1225
|
+
ctx.isUnlimited = unlimit === '1';
|
|
1226
|
+
const expires = membershipDate(detail.endtime ?? detail.expire_time ?? info.endtime ?? info.expire_time);
|
|
1227
|
+
if (expires)
|
|
1228
|
+
diagnostics.membership_expires_at = expires;
|
|
1229
|
+
}
|
|
1230
|
+
}
|
|
1231
|
+
catch {
|
|
1232
|
+
// Defaults keep isvip/unlimit enabled, matching a paid account.
|
|
1233
|
+
}
|
|
1234
|
+
// 4) `/myapi/apinodelist`: the clickable route ids.
|
|
1235
|
+
//
|
|
1236
|
+
// Every advertised table is fetched and merged. 旋风 publishes two (黄金 and
|
|
1237
|
+
// 白金); the others publish one. Area ids repeat across tables, so the merge
|
|
1238
|
+
// dedupes by id — the same route asked twice in a round is a wasted request,
|
|
1239
|
+
// not a second node.
|
|
1240
|
+
const collected = [];
|
|
1241
|
+
for (const table of eversionFields.apiLines) {
|
|
1242
|
+
const groups = parseRouteGroups(await getRawCommStr(ctx, eversionFields.apiLinks, apiDomain, pathAndQuery(table), '', discovery.seed_domains));
|
|
1243
|
+
collected.push(...flattenRouteGroups(groups));
|
|
1244
|
+
}
|
|
1245
|
+
// Free-tier rows issue placeholders and mainland rows are the wrong
|
|
1246
|
+
// direction, so neither is worth a request per round.
|
|
1247
|
+
const selection = selectForeignRoutes(collected, {
|
|
1248
|
+
includeFree: source.include_free_routes === true,
|
|
1249
|
+
includeChina: source.include_china_routes === true
|
|
1250
|
+
});
|
|
1251
|
+
let routes = selection.routes;
|
|
1252
|
+
if (source.max_areas && source.max_areas > 0) {
|
|
1253
|
+
routes = routes.slice(0, source.max_areas);
|
|
1254
|
+
}
|
|
1255
|
+
diagnostics.route_count = routes.length;
|
|
1256
|
+
diagnostics.route_tables = eversionFields.apiLines.length;
|
|
1257
|
+
diagnostics.routes_dropped = {
|
|
1258
|
+
free: selection.dropped_free,
|
|
1259
|
+
china: selection.dropped_china,
|
|
1260
|
+
disabled: selection.dropped_disabled,
|
|
1261
|
+
duplicate: selection.dropped_duplicate
|
|
1262
|
+
};
|
|
1263
|
+
if (!routes.length) {
|
|
1264
|
+
errors.push(collected.length ? 'node list had no foreign paid routes' : 'node list was empty');
|
|
1265
|
+
return { ...empty, failed_iterations: 1 };
|
|
1266
|
+
}
|
|
1267
|
+
log(`[extract] routes=${routes.length} tables=${eversionFields.apiLines.length}` +
|
|
1268
|
+
` dropped(free=${selection.dropped_free} china=${selection.dropped_china}` +
|
|
1269
|
+
` disabled=${selection.dropped_disabled} dup=${selection.dropped_duplicate})`);
|
|
1270
|
+
// 5) `/api/evmess?area=ID`, repeated.
|
|
1271
|
+
//
|
|
1272
|
+
// The server picks a landing host per request, so asking the same `area` again
|
|
1273
|
+
// usually returns a different machine: the route id chooses the region, not
|
|
1274
|
+
// the host. A single pass therefore sees only one host per route, and the
|
|
1275
|
+
// rounds below are what turn a handful of routes into a full node pool.
|
|
1276
|
+
//
|
|
1277
|
+
// Round control mirrors the URL mode's iteration semantics:
|
|
1278
|
+
// max_iterations upper bound on rounds (default 1)
|
|
1279
|
+
// min_iterations rounds that always run before early stopping is allowed
|
|
1280
|
+
// plateau_limit consecutive rounds with no new link that stop the run
|
|
1281
|
+
const links = [];
|
|
1282
|
+
const seen = new Set();
|
|
1283
|
+
let placeholders = 0;
|
|
1284
|
+
let requests = 0;
|
|
1285
|
+
const activeRoutes = routes.filter((route) => route.enabled);
|
|
1286
|
+
// The dashboard's dedupe card counts distinct fingerprints, not raw links: a
|
|
1287
|
+
// link repeated across two platforms has to collapse there or the number
|
|
1288
|
+
// only ever grows.
|
|
1289
|
+
const fingerprints = new Set();
|
|
1290
|
+
// Raw totals use the exact-string identity the pipeline dedupes with, so the
|
|
1291
|
+
// live number and `extract_source_completed.raw_links` agree.
|
|
1292
|
+
const linkIds = new Set();
|
|
1293
|
+
const policy = resolveRoundPolicy(source);
|
|
1294
|
+
const maxIterations = policy.max_iterations;
|
|
1295
|
+
const minIterations = policy.min_iterations;
|
|
1296
|
+
const plateauLimit = policy.plateau_limit;
|
|
1297
|
+
const startedAt = Date.now();
|
|
1298
|
+
let plateau = 0;
|
|
1299
|
+
let rounds = 0;
|
|
1300
|
+
let successfulRounds = 0;
|
|
1301
|
+
let failedRounds = 0;
|
|
1302
|
+
for (let round = 1; round <= maxIterations; round += 1) {
|
|
1303
|
+
if (!shouldStartRound(round, policy, (Date.now() - startedAt) / 1000)) {
|
|
1304
|
+
break;
|
|
1305
|
+
}
|
|
1306
|
+
let newInRound = 0;
|
|
1307
|
+
let answeredInRound = 0;
|
|
1308
|
+
const newFingerprints = [];
|
|
1309
|
+
const newLinkIds = [];
|
|
1310
|
+
for (const route of activeRoutes) {
|
|
1311
|
+
const plain = await getCommStr(ctx, eversionFields.apiLinks, apiDomain, '/api/evmess', `&area=${route.area}`, discovery.seed_domains);
|
|
1312
|
+
requests += 1;
|
|
1313
|
+
if (!plain) {
|
|
1314
|
+
continue;
|
|
1315
|
+
}
|
|
1316
|
+
answeredInRound += 1;
|
|
1317
|
+
// A malformed body must cost this route, not the run: the URL mode wraps
|
|
1318
|
+
// the identical call per iteration for the same reason.
|
|
1319
|
+
let decoded = [];
|
|
1320
|
+
try {
|
|
1321
|
+
decoded = extractLinksFromPlaintext(`${route.region}-${route.name}`, plain);
|
|
1322
|
+
}
|
|
1323
|
+
catch (error) {
|
|
1324
|
+
errors.push(`route ${route.area}: ${error instanceof Error ? error.message : String(error)}`);
|
|
1325
|
+
continue;
|
|
1326
|
+
}
|
|
1327
|
+
for (const link of decoded) {
|
|
1328
|
+
if (isPlaceholderLink(link)) {
|
|
1329
|
+
placeholders += 1;
|
|
1330
|
+
continue;
|
|
1331
|
+
}
|
|
1332
|
+
if (!seen.has(link)) {
|
|
1333
|
+
seen.add(link);
|
|
1334
|
+
links.push(link);
|
|
1335
|
+
newInRound += 1;
|
|
1336
|
+
const fingerprint = linkFingerprint(link);
|
|
1337
|
+
fingerprints.add(fingerprint);
|
|
1338
|
+
newFingerprints.push(fingerprint);
|
|
1339
|
+
const digest = linkDigest(link);
|
|
1340
|
+
linkIds.add(digest);
|
|
1341
|
+
newLinkIds.push(digest);
|
|
1342
|
+
}
|
|
1343
|
+
}
|
|
1344
|
+
}
|
|
1345
|
+
rounds = round;
|
|
1346
|
+
if (answeredInRound > 0) {
|
|
1347
|
+
successfulRounds += 1;
|
|
1348
|
+
}
|
|
1349
|
+
else {
|
|
1350
|
+
failedRounds += 1;
|
|
1351
|
+
}
|
|
1352
|
+
if (newInRound > 0) {
|
|
1353
|
+
await options.linksCallback?.(links.slice(-newInRound));
|
|
1354
|
+
}
|
|
1355
|
+
// The dashboard's raw/dedupe cards read these numbers while the run is in
|
|
1356
|
+
// flight. Without a per-round event a credential source stays at 0 until
|
|
1357
|
+
// it finishes, which reads as a stalled run: the URL mode already emits
|
|
1358
|
+
// `extract_iteration` for exactly this reason.
|
|
1359
|
+
event(options, 'extract_iteration', {
|
|
1360
|
+
source_name: input.source_name,
|
|
1361
|
+
iteration: round,
|
|
1362
|
+
requested_iterations: maxIterations,
|
|
1363
|
+
new_items: newInRound,
|
|
1364
|
+
extracted_links: newInRound,
|
|
1365
|
+
total_links: links.length,
|
|
1366
|
+
deduped_links: fingerprints.size,
|
|
1367
|
+
new_item_fingerprints: newFingerprints,
|
|
1368
|
+
new_item_link_ids: newLinkIds
|
|
1369
|
+
});
|
|
1370
|
+
log(`[extract] round ${round}/${maxIterations} +${newInRound} new (total ${links.length})`);
|
|
1371
|
+
plateau = newInRound === 0 ? plateau + 1 : 0;
|
|
1372
|
+
if (shouldStopOnPlateau(round, plateau, policy)) {
|
|
1373
|
+
break;
|
|
1374
|
+
}
|
|
1375
|
+
}
|
|
1376
|
+
diagnostics.placeholder_count = placeholders;
|
|
1377
|
+
diagnostics.node_count = links.length;
|
|
1378
|
+
diagnostics.rounds = rounds;
|
|
1379
|
+
diagnostics.requests = requests;
|
|
1380
|
+
event(options, 'extract_source_completed', {
|
|
1381
|
+
source_name: input.source_name,
|
|
1382
|
+
requested_iterations: maxIterations,
|
|
1383
|
+
successful_iterations: successfulRounds,
|
|
1384
|
+
failed_iterations: failedRounds,
|
|
1385
|
+
raw_links: links.length
|
|
1386
|
+
});
|
|
1387
|
+
return {
|
|
1388
|
+
source_name: input.source_name,
|
|
1389
|
+
requested_iterations: maxIterations,
|
|
1390
|
+
successful_iterations: successfulRounds,
|
|
1391
|
+
failed_iterations: failedRounds,
|
|
1392
|
+
links,
|
|
1393
|
+
diagnostics
|
|
1394
|
+
};
|
|
1395
|
+
}
|
|
1396
|
+
function isPlaceholderLink(link) {
|
|
1397
|
+
try {
|
|
1398
|
+
const encoded = link.replace(/^vmess:\/\//, '');
|
|
1399
|
+
const padded = encoded + '='.repeat((4 - (encoded.length % 4)) % 4);
|
|
1400
|
+
const payload = JSON.parse(Buffer.from(padded, 'base64url').toString('utf8'));
|
|
1401
|
+
return isPlaceholderNode(String(payload.add ?? ''));
|
|
1402
|
+
}
|
|
1403
|
+
catch {
|
|
1404
|
+
return false;
|
|
1405
|
+
}
|
|
1406
|
+
}
|