@swimmingliu/autovpn 1.8.0 → 2.0.2

package/dist/pipeline/extract.js

@@ -42,7 +42,19 @@ function generateVmessLink(payload) {
     const encoded = Buffer.from(json, 'utf8').toString('base64').replaceAll('+', '-').replaceAll('/', '_');
     return `vmess://${encoded}`;
 }
-function linkFingerprint(link) {
+/**
+ * Exact-string identity for a link.
+ *
+ * `linkFingerprint` collapses links that describe the same node, which is what
+ * the dedupe card counts. Raw link totals have to use the same `seen` identity
+ * the pipeline itself uses, or two platforms that label one node differently
+ * would make the live raw count disagree with the final result. A digest keeps
+ * the link out of the event stream.
+ */
+export function linkDigest(link) {
+    return crypto.createHash('sha256').update(String(link)).digest('hex');
+}
+export function linkFingerprint(link) {
     try {
         const encoded = String(link).replace(/^vmess:\/\//, '');
         const padded = encoded + '='.repeat((4 - (encoded.length % 4)) % 4);
@@ -350,5 +362,46 @@ async function fetchSourceLinksInNode(input, options) {
     };
 }
 export async function fetchSourceLinksWithBackend(input, options = {}) {
-    return options.fetchSourceLinks ? options.fetchSourceLinks(input) : fetchSourceLinksInNode(input, options);
+    if (options.fetchSourceLinks) {
+        return options.fetchSourceLinks(input);
+    }
+    if (isCredentialSource(input.source)) {
+        const { fetchCredentialSourceLinks } = await import('./credential-source.js');
+        const result = await fetchCredentialSourceLinks(input, {
+            eventCallback: options.eventCallback,
+            linksCallback: options.linksCallback,
+            fingerprintStore: options.fingerprintStore,
+            memo: options.fingerprintMemo,
+            // The credential flow talks to the same hosts as the url flow, so it
+            // needs the same egress handling: an injected fetch for tests, the
+            // configured upstream proxy, and the curl TLS fallback.
+            fetchImpl: options.fetch,
+            env: options.env,
+            curlFetch: options.curlFetch
+        });
+        return {
+            source_name: result.source_name,
+            requested_iterations: result.requested_iterations,
+            successful_iterations: result.successful_iterations,
+            failed_iterations: result.failed_iterations,
+            links: result.links,
+            errors: result.diagnostics.errors
+        };
+    }
+    return fetchSourceLinksInNode(input, options);
+}
+/**
+ * A credential source is identified by its mode or by carrying an account
+ * instead of a request URL; this keeps older profiles working unchanged.
+ */
+export function isCredentialSource(source) {
+    if (source.mode === 'credential') {
+        return true;
+    }
+    if (source.mode === 'url') {
+        return false;
+    }
+    const hasAccount = Boolean(String(source.account ?? '').trim());
+    const hasUrl = Boolean(String(source.url ?? '').trim());
+    return hasAccount && !hasUrl;
 }

package/dist/pipeline/orchestrator.js

@@ -2,8 +2,9 @@ import { copyFile, mkdir, readFile, writeFile } from 'node:fs/promises';
 import fs from 'node:fs';
 import path from 'node:path';
 import { parse } from '@iarna/toml';
+import { normalizeOfficialSources } from '../config/profile.js';
 import { mergeProjectEnv } from '../runtime/env.js';
-import { resolveArtifactsRoot, resolveProfilePath } from '../runtime/paths.js';
+import { resolveArtifactsRoot, resolveProfilePath, resolveUserRuntimeRoot } from '../runtime/paths.js';
 import { redactText } from '../runtime/redaction.js';
 import { resolveWorkerTemplatePath } from '../runtime/templates.js';
 import { fetchSourceLinksWithBackend } from './extract.js';
@@ -18,6 +19,8 @@ import { safeDeployment } from '../runtime/redaction.js';
 import { RunStore, readLatestStageStatuses } from './run-store.js';
 import { AsyncPermitPool, BoundedWorkerPool } from './streaming-coordinator.js';
 import { createGeoIpLookup } from './geoip.js';
+import { FingerprintStore, resolveFingerprintCacheDir } from './brand-cache.js';
+import { createFingerprintMemo, credentialBrandKey } from './credential-source.js';
 const STAGES = ['doctor', 'extract', 'dedupe', 'speedtest', 'availability', 'postprocess', 'render', 'obfuscate', 'deploy', 'verify'];
 const RETRYABLE_STAGES = ['speedtest', 'availability', 'postprocess', 'render', 'obfuscate', 'deploy', 'verify'];
 function formatTimestamp(date) {
@@ -45,12 +48,23 @@ function asProfile(payload) {
 async function readProfile(projectRoot, env) {
     const profilePath = resolveProfilePath(projectRoot, env);
     const text = await readFile(profilePath, 'utf8');
-    return asProfile(parse(text));
+    const profile = asProfile(parse(text));
+    return { ...profile, sources: normalizeOfficialSources(profile.sources ?? {}) };
 }
 function enabledSources(profile) {
-    return Object.entries(profile.sources ?? {}).filter(([, source]) => (source.enabled !== false
-        && String(source.url ?? '').trim()
-        && String(source.key ?? '').trim()));
+    return Object.entries(profile.sources ?? {}).filter(([, source]) => {
+        if (source.enabled === false) {
+            return false;
+        }
+        // Credential sources need only a location. The account is leased from the
+        // pool at extract time; an empty pool is a reported source failure, not a
+        // silently disabled source.
+        if (source.mode === 'credential') {
+            return [source.official_site, source.apk_url, source.apk_path]
+                .some((value) => String(value ?? '').trim().length > 0);
+        }
+        return Boolean(String(source.url ?? '').trim()) && Boolean(String(source.key ?? '').trim());
+    });
 }
 function linesText(lines) {
     return lines.length > 0 ? `${lines.join('\n')}\n` : '';
@@ -136,6 +150,9 @@ function renderHumanEvent(event) {
     if (event.type === 'run_failed') {
         return `[run_failed] ${String(event.error ?? 'unknown error')}`;
     }
+    if (event.type === 'extract_platform_started' || event.type === 'extract_platform_completed') {
+        return `[${event.type}] ${String(event.source_name ?? '')} platform=${String(event.platform ?? '')}`;
+    }
     return JSON.stringify(event);
 }
 function eventLogLine(event) {
@@ -443,6 +460,24 @@ export async function runNodePipeline(options, context = {}) {
             await setStage('doctor', 'success');
             const profile = await readProfile(projectRoot, env);
             const sourcesToRun = enabledSources(profile);
+            // Brand fingerprints are shared by every source of the same brand so one
+            // run downloads and inspects each APK at most once.
+            const fingerprintStore = new FingerprintStore({
+                cacheDir: resolveFingerprintCacheDir(resolveUserRuntimeRoot(env)),
+                now: context.now ?? (() => new Date())
+            });
+            // Keyed by brand, not by profile entry name: two sources of one brand must
+            // share both the in-run memo and the on-disk fingerprint.
+            const brandKeyFor = (sourceName, source) => (source.mode === 'credential' || String(source.account ?? '').trim() ? credentialBrandKey(sourceName, source) : sourceName);
+            const fingerprintMemos = new Map();
+            const fingerprintMemoFor = (brand) => {
+                let memo = fingerprintMemos.get(brand);
+                if (!memo) {
+                    memo = createFingerprintMemo(fingerprintStore, brand);
+                    fingerprintMemos.set(brand, memo);
+                }
+                return memo;
+            };
             const runtimePath = path.join(artifactDir, 'runtime');
             const speedConfig = normalizeSpeedTestConfig(profile.speed_test);
             // Preserve cross-stage streaming even for concurrency=1 while preventing
@@ -576,7 +611,9 @@ export async function runNodePipeline(options, context = {}) {
                             cwd: projectRoot,
                             env: runtimeStageEnv,
                             eventCallback: (type, payload) => emit(type, payload),
-                            linksCallback: stream?.onLinks
+                            linksCallback: stream?.onLinks,
+                            fingerprintStore,
+                            fingerprintMemo: fingerprintMemoFor(brandKeyFor(sourceName, source))
                         });
                 }
                 catch (error) {
@@ -594,10 +631,20 @@ export async function runNodePipeline(options, context = {}) {
                     successful_iterations: result.successful_iterations,
                     failed_iterations: result.failed_iterations
                 };
+                // A credential source reports its own failures (rejected login, APK not
+                // located) as zero links rather than an exception. Recording that as
+                // `success` hides the reason, so the source is marked failed and the
+                // text is emitted where an operator can read it.
+                const sourceErrors = (result.errors ?? []).filter((line) => String(line ?? '').trim());
+                const sourceFailed = result.links.length === 0 && sourceErrors.length > 0;
+                if (sourceFailed) {
+                    emit('log', { message: `[extract] ${result.source_name} produced no links: ${sourceErrors.join('; ')}` });
+                }
                 runStore.recordSourceProgress(sourceName, {
                     processed: result.requested_iterations,
                     total: result.requested_iterations,
-                    status: 'success'
+                    status: sourceFailed ? 'failed' : 'success',
+                    ...(sourceFailed ? { error: sourceErrors.join('; ') } : {})
                 });
                 return result;
             }));
@@ -1261,6 +1308,22 @@ export async function resumeNodePipeline(options, context = {}) {
     const report = await readJson(path.join(artifactDir, 'pipeline_report.json'), {});
     const summary = pipelineSummaryFromReport(artifactDir, report);
     const runtimePath = path.join(artifactDir, 'runtime');
+    // A resumed run must reuse the fingerprint cache too: re-extracting a
+    // credential source would otherwise re-download and re-parse the full APK.
+    const resumeFingerprintStore = new FingerprintStore({
+        cacheDir: resolveFingerprintCacheDir(resolveUserRuntimeRoot(env)),
+        now: context.now ?? (() => new Date())
+    });
+    const resumeMemos = new Map();
+    const resumeMemoFor = (brand) => {
+        let memo = resumeMemos.get(brand);
+        if (!memo) {
+            memo = createFingerprintMemo(resumeFingerprintStore, brand);
+            resumeMemos.set(brand, memo);
+        }
+        return memo;
+    };
+    const resumeBrandKeyFor = (sourceName, source) => (source.mode === 'credential' || String(source.account ?? '').trim() ? credentialBrandKey(sourceName, source) : sourceName);
     const emit = (type, payload = {}) => {
         const event = { type, ...payload };
         appendTextFile(eventLog, eventLogLine(event));
@@ -1375,9 +1438,25 @@ export async function resumeNodePipeline(options, context = {}) {
                         try {
                             const result = context.stages?.extract
                                 ? await context.stages.extract({ source_name: sourceName, source }, { onLinks })
-                                : await fetchSourceLinksWithBackend({ source_name: sourceName, source }, { cwd: projectRoot, env: runtimeStageEnv, linksCallback: onLinks });
+                                : await fetchSourceLinksWithBackend({ source_name: sourceName, source }, {
+                                    cwd: projectRoot,
+                                    env: runtimeStageEnv,
+                                    linksCallback: onLinks,
+                                    fingerprintStore: resumeFingerprintStore,
+                                    fingerprintMemo: resumeMemoFor(resumeBrandKeyFor(sourceName, source))
+                                });
                             await onLinks(result.links.filter((link) => !streamed.has(link)));
-                            runStore.recordSourceProgress(sourceName, { processed: result.requested_iterations, total: result.requested_iterations, status: 'success' });
+                            const resumeErrors = (result.errors ?? []).filter((line) => String(line ?? '').trim());
+                            const resumeFailed = result.links.length === 0 && resumeErrors.length > 0;
+                            if (resumeFailed) {
+                                emit('log', { message: `[extract] ${result.source_name} produced no links: ${resumeErrors.join('; ')}` });
+                            }
+                            runStore.recordSourceProgress(sourceName, {
+                                processed: result.requested_iterations,
+                                total: result.requested_iterations,
+                                status: resumeFailed ? 'failed' : 'success',
+                                ...(resumeFailed ? { error: resumeErrors.join('; ') } : {})
+                            });
                             summary.source_counts[sourceName] = { raw_links: result.links.length, successful_iterations: result.successful_iterations, failed_iterations: result.failed_iterations };
                         }
                         catch (error) {