@swimmingliu/autovpn 1.7.0 → 1.8.0

package/dist/pipeline/country-codes.js

@@ -0,0 +1,12 @@
+const ISO_ALPHA_2_CODES = new Set(`
+AD AE AF AG AI AL AM AO AQ AR AS AT AU AW AX AZ BA BB BD BE BF BG BH BI BJ BL BM BN BO BQ BR BS BT BV BW BY BZ
+CA CC CD CF CG CH CI CK CL CM CN CO CR CU CV CW CX CY CZ DE DJ DK DM DO DZ EC EE EG EH ER ES ET FI FJ FK FM FO FR
+GA GB GD GE GF GG GH GI GL GM GN GP GQ GR GS GT GU GW GY HK HM HN HR HT HU ID IE IL IM IN IO IQ IR IS IT JE JM JO JP
+KE KG KH KI KM KN KP KR KW KY KZ LA LB LC LI LK LR LS LT LU LV LY MA MC MD ME MF MG MH MK ML MM MN MO MP MQ MR MS MT
+MU MV MW MX MY MZ NA NC NE NF NG NI NL NO NP NR NU NZ OM PA PE PF PG PH PK PL PM PN PR PS PT PW PY QA RE RO RS RU RW
+SA SB SC SD SE SG SH SI SJ SK SL SM SN SO SR SS ST SV SX SY SZ TC TD TF TG TH TJ TK TL TM TN TO TR TT TV TW TZ UA UG
+UM US UY UZ VA VC VE VG VI VN VU WF WS YE YT ZA ZM ZW
+`.trim().split(/\s+/));
+export function isIsoAlpha2CountryCode(value) {
+    return typeof value === 'string' && ISO_ALPHA_2_CODES.has(value.trim().toUpperCase());
+}

package/dist/pipeline/geoip.js

@@ -0,0 +1,232 @@
+import { isIP } from 'node:net';
+import { lookup as dnsLookup } from 'node:dns/promises';
+import { isIsoAlpha2CountryCode } from './country-codes.js';
+function countryCode(value) {
+    const normalized = typeof value === 'string' ? value.trim().toUpperCase() : '';
+    return isIsoAlpha2CountryCode(normalized) ? normalized : null;
+}
+function retryAfterMilliseconds(value, now, maximum) {
+    if (!value)
+        return 0;
+    const seconds = Number(value);
+    const delay = Number.isFinite(seconds)
+        ? seconds * 1000
+        : Date.parse(value) - now;
+    return Number.isFinite(delay) && delay > 0 ? Math.min(delay, maximum) : 0;
+}
+function ipv4Number(address) {
+    const octets = address.split('.').map(Number);
+    if (octets.length !== 4 || octets.some((octet) => !Number.isInteger(octet) || octet < 0 || octet > 255))
+        return null;
+    return (((octets[0] * 256 + octets[1]) * 256 + octets[2]) * 256 + octets[3]) >>> 0;
+}
+function ipv4InCidr(value, base, prefix) {
+    const baseValue = ipv4Number(base);
+    const size = 2 ** (32 - prefix);
+    return Math.floor(value / size) === Math.floor(baseValue / size);
+}
+function isRejectedIpv4(value) {
+    return [
+        ['0.0.0.0', 8], ['10.0.0.0', 8], ['100.64.0.0', 10], ['127.0.0.0', 8],
+        ['169.254.0.0', 16], ['172.16.0.0', 12], ['192.0.0.0', 24], ['192.0.2.0', 24], ['192.88.99.0', 24],
+        ['192.168.0.0', 16], ['198.18.0.0', 15], ['198.51.100.0', 24], ['203.0.113.0', 24],
+        ['224.0.0.0', 4], ['240.0.0.0', 4]
+    ].some(([base, prefix]) => ipv4InCidr(value, base, prefix));
+}
+function ipv6Number(address) {
+    const pieces = address.toLowerCase().split('::');
+    if (pieces.length > 2)
+        return null;
+    const parseSide = (side) => {
+        if (!side)
+            return [];
+        const groups = side.split(':');
+        const last = groups.at(-1) ?? '';
+        if (last.includes('.')) {
+            const mapped = ipv4Number(last);
+            if (mapped === null)
+                return null;
+            groups.splice(-1, 1, ((mapped >>> 16) & 0xffff).toString(16), (mapped & 0xffff).toString(16));
+        }
+        const parsed = groups.map((group) => /^[0-9a-f]{1,4}$/.test(group) ? Number.parseInt(group, 16) : -1);
+        return parsed.some((group) => group < 0) ? null : parsed;
+    };
+    const left = parseSide(pieces[0]);
+    const right = parseSide(pieces[1] ?? '');
+    if (!left || !right)
+        return null;
+    const missing = 8 - left.length - right.length;
+    if ((pieces.length === 1 && missing !== 0) || (pieces.length === 2 && missing < 1))
+        return null;
+    const groups = [...left, ...Array(missing).fill(0), ...right];
+    return groups.reduce((value, group) => (value << 16n) | BigInt(group), 0n);
+}
+function canonicalIpv6(value) {
+    const groups = Array.from({ length: 8 }, (_, index) => Number((value >> BigInt((7 - index) * 16)) & 0xffffn));
+    let bestStart = -1;
+    let bestLength = 0;
+    for (let index = 0; index < groups.length;) {
+        if (groups[index] !== 0) {
+            index += 1;
+            continue;
+        }
+        let end = index;
+        while (end < groups.length && groups[end] === 0)
+            end += 1;
+        if (end - index > bestLength) {
+            bestStart = index;
+            bestLength = end - index;
+        }
+        index = end;
+    }
+    const hex = groups.map((group) => group.toString(16));
+    if (bestLength < 2)
+        return hex.join(':');
+    const before = hex.slice(0, bestStart).join(':');
+    const after = hex.slice(bestStart + bestLength).join(':');
+    return `${before}::${after}`;
+}
+function normalizeGlobalAddress(address) {
+    if (isIP(address) === 4) {
+        const value = ipv4Number(address);
+        return value !== null && !isRejectedIpv4(value) ? address : null;
+    }
+    const value = ipv6Number(address);
+    if (value === null)
+        return null;
+    if ((value >> 32n) === 0xffffn) {
+        const mapped = Number(value & 0xffffffffn) >>> 0;
+        if (isRejectedIpv4(mapped))
+            return null;
+        return `${mapped >>> 24}.${(mapped >>> 16) & 255}.${(mapped >>> 8) & 255}.${mapped & 255}`;
+    }
+    const globalUnicast = value >= 0x20000000000000000000000000000000n && value <= 0x3fffffffffffffffffffffffffffffffn;
+    const ietfProtocolAssignments = value >= 0x20010000000000000000000000000000n && value <= 0x200101ffffffffffffffffffffffffffn;
+    const documentation = value >= 0x20010db8000000000000000000000000n && value <= 0x20010db8ffffffffffffffffffffffffn;
+    const extendedDocumentation = value >= 0x3fff0000000000000000000000000000n && value <= 0x3fff0fffffffffffffffffffffffffffn;
+    return globalUnicast && !ietfProtocolAssignments && !documentation && !extendedDocumentation ? canonicalIpv6(value) : null;
+}
+export function createGeoIpLookup(options = {}) {
+    const fetchFn = options.fetch ?? globalThis.fetch;
+    const resolve = options.resolve ?? ((hostname) => dnsLookup(hostname, { all: true, verbatim: true }));
+    const now = options.now ?? Date.now;
+    const sleep = options.sleep ?? ((milliseconds) => new Promise((done) => globalThis.setTimeout(done, milliseconds)));
+    const setTimer = options.setTimeout ?? ((callback, milliseconds) => globalThis.setTimeout(callback, milliseconds));
+    const clearTimer = options.clearTimeout ?? ((handle) => globalThis.clearTimeout(handle));
+    const timeoutMs = options.timeoutMs ?? 3000;
+    const successTtlMs = options.successTtlMs ?? 24 * 60 * 60 * 1000;
+    const negativeTtlMs = options.negativeTtlMs ?? 60 * 1000;
+    const maxRetryAfterMs = options.maxRetryAfterMs ?? 2000;
+    const providerConcurrency = Math.max(1, Math.floor(options.providerConcurrency ?? 4));
+    const primaryUrl = options.primaryUrl ?? ((ip) => `https://ipwho.is/${encodeURIComponent(ip)}`);
+    const fallbackUrl = options.fallbackUrl ?? ((ip) => `https://ipapi.co/${encodeURIComponent(ip)}/json/`);
+    const cache = new Map();
+    const pending = new Map();
+    const providerWaiters = [];
+    let activeProviderRequests = 0;
+    let providerCooldownUntil = 0;
+    async function acquireProviderSlot() {
+        if (activeProviderRequests >= providerConcurrency) {
+            await new Promise((resolve) => providerWaiters.push(resolve));
+        }
+        else {
+            activeProviderRequests += 1;
+        }
+        return () => {
+            const next = providerWaiters.shift();
+            if (next)
+                next();
+            else
+                activeProviderRequests -= 1;
+        };
+    }
+    async function request(url, provider) {
+        try {
+            const parsed = new URL(url);
+            const expectedHost = provider === 'primary' ? 'ipwho.is' : 'ipapi.co';
+            if (parsed.protocol !== 'https:' || parsed.hostname !== expectedHost || parsed.port || parsed.username || parsed.password) {
+                return { country: null, retryAfterMs: 0 };
+            }
+        }
+        catch {
+            return { country: null, retryAfterMs: 0 };
+        }
+        const release = await acquireProviderSlot();
+        const cooldownMs = providerCooldownUntil - now();
+        if (cooldownMs > 0)
+            await sleep(cooldownMs);
+        const controller = new AbortController();
+        const timer = setTimer(() => controller.abort(), timeoutMs);
+        try {
+            const response = await fetchFn(url, { signal: controller.signal, headers: { accept: 'application/json' } });
+            const retryAfterMs = response.status === 429
+                ? retryAfterMilliseconds(response.headers.get('retry-after'), now(), maxRetryAfterMs)
+                : 0;
+            if (retryAfterMs > 0)
+                providerCooldownUntil = Math.max(providerCooldownUntil, now() + retryAfterMs);
+            if (!response.ok)
+                return { country: null, retryAfterMs };
+            const payload = await response.json();
+            if (provider === 'primary' && payload.success !== true)
+                return { country: null, retryAfterMs: 0 };
+            if (provider === 'fallback' && payload.error === true)
+                return { country: null, retryAfterMs: 0 };
+            return { country: countryCode(payload.country_code), retryAfterMs: 0 };
+        }
+        catch {
+            return { country: null, retryAfterMs: 0 };
+        }
+        finally {
+            clearTimer(timer);
+            release();
+        }
+    }
+    async function lookupIp(ip) {
+        const cached = cache.get(ip);
+        if (cached && cached.expiresAt > now())
+            return cached;
+        const existing = pending.get(ip);
+        if (existing)
+            return existing;
+        const operation = (async () => {
+            const primary = await request(primaryUrl(ip), 'primary');
+            if (primary.country)
+                return { country: primary.country, detected: true };
+            const fallback = await request(fallbackUrl(ip), 'fallback');
+            return fallback.country
+                ? { country: fallback.country, detected: true }
+                : { country: 'US', detected: false };
+        })();
+        pending.set(ip, operation);
+        try {
+            const result = await operation;
+            cache.set(ip, { ...result, expiresAt: now() + (result.detected ? successTtlMs : negativeTtlMs) });
+            return result;
+        }
+        finally {
+            pending.delete(ip);
+        }
+    }
+    return async (rawAddress) => {
+        const address = String(rawAddress ?? '').trim();
+        if (!address)
+            return 'US';
+        try {
+            const results = isIP(address) ? [{ address, family: isIP(address) }] : await resolve(address);
+            const seen = new Set();
+            for (const result of results) {
+                const globalAddress = normalizeGlobalAddress(result.address);
+                if (!globalAddress || seen.has(globalAddress))
+                    continue;
+                seen.add(globalAddress);
+                const resultCountry = await lookupIp(globalAddress);
+                if (resultCountry.detected)
+                    return resultCountry.country;
+            }
+            return 'US';
+        }
+        catch {
+            return 'US';
+        }
+    };
+}

package/dist/pipeline/orchestrator.js

@@ -7,6 +7,7 @@ import { resolveArtifactsRoot, resolveProfilePath } from '../runtime/paths.js';
 import { redactText } from '../runtime/redaction.js';
 import { resolveWorkerTemplatePath } from '../runtime/templates.js';
 import { fetchSourceLinksWithBackend } from './extract.js';
+import { parseVmessLink } from './dedupe.js';
 import { normalizeSpeedTestConfig, probeSpeedtestLinksInNode, speedtestLinksWithBackend, testSpeedtestLinkInNode } from './speedtest.js';
 import { checkLinkAvailabilityBatchWithBackend } from './availability.js';
 import { decorateLinkWithCountry, postprocessLinksWithBackend } from './postprocess.js';
@@ -16,6 +17,7 @@ import { deployPagesWithBackend, isVerifySuccess, verifyDeploymentWithBackend }
 import { safeDeployment } from '../runtime/redaction.js';
 import { RunStore, readLatestStageStatuses } from './run-store.js';
 import { AsyncPermitPool, BoundedWorkerPool } from './streaming-coordinator.js';
+import { createGeoIpLookup } from './geoip.js';
 const STAGES = ['doctor', 'extract', 'dedupe', 'speedtest', 'availability', 'postprocess', 'render', 'obfuscate', 'deploy', 'verify'];
 const RETRYABLE_STAGES = ['speedtest', 'availability', 'postprocess', 'render', 'obfuscate', 'deploy', 'verify'];
 function formatTimestamp(date) {
@@ -151,8 +153,18 @@ function errorMessage(error) {
 function defaultRuntimeStageEnv(env) {
     return { ...env };
 }
-function defaultCountryFor(_link, _speedResult, _availabilityResult) {
-    return 'US';
+function createCountryLookup(context) {
+    if (context.stages?.countryLookup)
+        return context.stages.countryLookup;
+    const lookup = createGeoIpLookup(context.geoIp);
+    return async (link) => {
+        try {
+            return lookup(String(parseVmessLink(link).add ?? ''));
+        }
+        catch {
+            return 'US';
+        }
+    };
 }
 function normalizeRetryStage(stage) {
     if (RETRYABLE_STAGES.includes(stage)) {
@@ -222,6 +234,26 @@ function pipelineSummaryFromReport(artifactDir, report) {
         error: String(report.error ?? '')
     };
 }
+export function refreshSourceCounts(summary, store) {
+    const rawCounts = store.sourceRawCounts();
+    const dedupedCounts = store.sourceDedupedCounts();
+    const completeOwnership = store.hasCompleteSourceOwnership();
+    const progressSources = store.sourceProgress().map((progress) => progress.source);
+    for (const source of new Set([...Object.keys(summary.source_counts), ...progressSources, ...Object.keys(rawCounts), ...Object.keys(dedupedCounts)])) {
+        const previous = { ...(summary.source_counts[source] ?? {}) };
+        if (!completeOwnership && !Object.hasOwn(dedupedCounts, source))
+            delete previous.deduped_links;
+        summary.source_counts[source] = {
+            ...previous,
+            raw_links: rawCounts[source] ?? 0,
+            ...(Object.hasOwn(dedupedCounts, source)
+                ? { deduped_links: dedupedCounts[source] }
+                : completeOwnership
+                    ? { deduped_links: 0 }
+                    : {})
+        };
+    }
+}
 function passedSpeedResults(allResults, passedLinks) {
     const byLink = new Map(allResults.map((result) => [result.link, result]));
     return passedLinks
@@ -523,7 +555,7 @@ export async function runNodePipeline(options, context = {}) {
                 emit('speedtest_runtime', { runtime_core: requestedRuntime === 'direct' ? 'direct' : 'mihomo', probe_url: speedConfig.probe_url, urls: [...speedConfig.urls] });
                 emit('log', { message: `[speedtest] runtime_core=${requestedRuntime === 'direct' ? 'direct' : 'mihomo'} probe_url=${speedConfig.probe_url}` });
             }
-            const extractResults = await Promise.all(sourcesToRun.map(async ([sourceName, source]) => {
+            const extractSettled = await Promise.allSettled(sourcesToRun.map(async ([sourceName, source]) => {
                 runStore.recordSourceProgress(sourceName, { processed: 0, total: 0, status: 'running' });
                 const streamedLinks = new Set();
                 const stream = useStreamingStages
@@ -569,6 +601,10 @@ export async function runNodePipeline(options, context = {}) {
                 });
                 return result;
             }));
+            const failedExtraction = extractSettled.find((result) => result.status === 'rejected');
+            if (failedExtraction)
+                throw failedExtraction.reason;
+            const extractResults = extractSettled.map((result) => result.value);
             if (!useStreamingStages) {
                 rawLinks = extractResults.flatMap((result) => result.links);
                 for (const result of extractResults) {
@@ -595,6 +631,7 @@ export async function runNodePipeline(options, context = {}) {
                 await setStage('dedupe', 'skipped', false);
             dedupedLinks = runStore.dedupedLinks();
             summary.counts.deduped_links = runStore.counts().deduped;
+            refreshSourceCounts(summary, runStore);
             await writeLines(artifactDir, 'vpn_node_deduped.txt', dedupedLinks);
             if (summary.stage_status.dedupe !== 'skipped')
                 await setStage('dedupe', 'success', !useStreamingStages);
@@ -689,12 +726,12 @@ export async function runNodePipeline(options, context = {}) {
             if (summary.stage_status.availability !== 'skipped')
                 await setStage('availability', 'success', !useStreamingStages);
             await setStage('postprocess', 'running');
-            const countryLookup = context.stages?.countryLookup ?? defaultCountryFor;
+            const countryLookup = createCountryLookup(context);
             const availabilityByLink = new Map(availabilityResults.map((result) => [result.link, result]));
-            const rankedLinks = availableLinks.map((link) => ({
+            const rankedLinks = await Promise.all(availableLinks.map(async (link) => ({
                 link,
-                country_code: countryLookup(link, speedResultByLink.get(link), availabilityByLink.get(link))
-            }));
+                country_code: await countryLookup(link, speedResultByLink.get(link), availabilityByLink.get(link))
+            })));
             const postprocessed = context.stages?.countryLookup
                 ? { links: rankedLinks.map((item) => decorateLinkWithCountry(item.link, item.country_code)) }
                 : await postprocessLinksWithBackend({ ranked_links: rankedLinks, filters: profile.filters }, { cwd: projectRoot, env });
@@ -753,6 +790,10 @@ export async function runNodePipeline(options, context = {}) {
             activeSpeedPool?.abort(error);
             activeAvailabilityPool?.abort(error);
             await Promise.allSettled([activeSpeedPool?.drain(), activeAvailabilityPool?.drain()].filter((task) => Boolean(task)));
+            const failedCounts = runStore.counts();
+            summary.counts.raw_links = failedCounts.raw;
+            summary.counts.deduped_links = failedCounts.deduped;
+            refreshSourceCounts(summary, runStore);
             summary.run_status = 'failed';
             summary.error = errorMessage(error);
             runStore.setRunStatus('failed', summary.error);
@@ -799,6 +840,7 @@ export async function retryNodePipelineStage(options, context = {}) {
     };
     const summary = await seedRetryArtifact(sourceArtifactDir, retryArtifactDir, stage, retryContext, String(profile.worker_build?.bundle_subdir ?? 'pages_bundle'));
     const retryStore = RunStore.seedRetry(sourceArtifactDir, retryArtifactDir, stage);
+    refreshSourceCounts(summary, retryStore);
     try {
         const emit = (type, payload = {}) => {
             const event = { type, ...payload };
@@ -910,11 +952,11 @@ export async function retryNodePipelineStage(options, context = {}) {
                 }
                 await setStage('postprocess', 'running');
                 const speedResultByLink = new Map(speedResults.map((result) => [result.link, result]));
-                const countryLookup = context.stages?.countryLookup ?? defaultCountryFor;
-                const rankedLinks = availabilityResults.map((availabilityResult) => ({
+                const countryLookup = createCountryLookup(context);
+                const rankedLinks = await Promise.all(availabilityResults.map(async (availabilityResult) => ({
                     link: availabilityResult.link,
-                    country_code: countryLookup(availabilityResult.link, speedResultByLink.get(availabilityResult.link) ?? availabilityResult, availabilityResult)
-                }));
+                    country_code: await countryLookup(availabilityResult.link, speedResultByLink.get(availabilityResult.link) ?? availabilityResult, availabilityResult)
+                })));
                 const postprocessed = context.stages?.countryLookup
                     ? { links: rankedLinks.map((item) => decorateLinkWithCountry(item.link, item.country_code)) }
                     : await postprocessLinksWithBackend({ ranked_links: rankedLinks, filters: profile.filters }, { cwd: projectRoot, env });
@@ -1044,6 +1086,7 @@ async function resumeNodeSpeedtest(options, context = {}) {
     try {
         runStore.reopenForResume();
         runStore.resetInterruptedRunning();
+        refreshSourceCounts(summary, runStore);
         const emit = (type, payload = {}) => {
             const event = { type, ...payload };
             appendTextFile(eventLog, eventLogLine(event));
@@ -1244,6 +1287,7 @@ export async function resumeNodePipeline(options, context = {}) {
         const storeCounts = runStore.counts();
         summary.counts.raw_links = storeCounts.raw;
         summary.counts.deduped_links = storeCounts.deduped;
+        refreshSourceCounts(summary, runStore);
         summary.counts.speedtest_links = speedResults.length;
         try {
             runStore.resetInterruptedRunning();
@@ -1357,6 +1401,7 @@ export async function resumeNodePipeline(options, context = {}) {
             }
             summary.counts.raw_links = runStore.counts().raw;
             summary.counts.deduped_links = runStore.counts().deduped;
+            refreshSourceCounts(summary, runStore);
             await writeLines(artifactDir, 'vpn_node_raw.txt', runStore.rawLinks());
             await writeLines(artifactDir, 'vpn_node_deduped.txt', runStore.dedupedLinks());
             const storedPassedSpeedResults = runStore.speedResults().filter((result) => result.status === 'speed_passed');
@@ -1398,11 +1443,11 @@ export async function resumeNodePipeline(options, context = {}) {
             await setStage('postprocess', 'running');
             const speedResultByLink = new Map(speedResults.map((result) => [result.link, result]));
             const availabilityByLink = new Map(availabilityResults.map((result) => [result.link, result]));
-            const countryLookup = context.stages?.countryLookup ?? defaultCountryFor;
-            const rankedLinks = availableLinks.map((link) => ({
+            const countryLookup = createCountryLookup(context);
+            const rankedLinks = await Promise.all(availableLinks.map(async (link) => ({
                 link,
-                country_code: countryLookup(link, speedResultByLink.get(link), availabilityByLink.get(link))
-            }));
+                country_code: await countryLookup(link, speedResultByLink.get(link), availabilityByLink.get(link))
+            })));
             const postprocessed = context.stages?.countryLookup
                 ? { links: rankedLinks.map((item) => decorateLinkWithCountry(item.link, item.country_code)) }
                 : await postprocessLinksWithBackend({ ranked_links: rankedLinks, filters: profile.filters }, { cwd: projectRoot, env });

package/dist/pipeline/postprocess.js

@@ -1,3 +1,4 @@
+import { isIsoAlpha2CountryCode } from './country-codes.js';
 const EMOJI_MAP = {
     AE: '🇦🇪',
     AR: '🇦🇷',
@@ -37,7 +38,6 @@ const EMOJI_MAP = {
     ZA: '🇿🇦'
 };
 const FALLBACK_COUNTRY_CODE = 'US';
-const UNKNOWN_COUNTRY_CODE = 'ZZ';
 const DEFAULT_FILTERS = {
     excluded_country_codes: ['CN'],
     per_country_limit: {}
@@ -55,7 +55,7 @@ export function generateVmessLink(payload) {
 }
 export function normalizeCountryCode(countryCode) {
     const normalized = String(countryCode || '').trim().toUpperCase();
-    if (normalized.length !== 2 || !/^[A-Z]{2}$/.test(normalized) || normalized === UNKNOWN_COUNTRY_CODE) {
+    if (!isIsoAlpha2CountryCode(normalized)) {
         return FALLBACK_COUNTRY_CODE;
     }
     return normalized;

package/dist/pipeline/run-store.js

@@ -119,6 +119,7 @@ export class RunStore {
         canonical_key TEXT NOT NULL,
         link TEXT NOT NULL,
         sequence INTEGER NOT NULL,
+        first_source TEXT,
         UNIQUE (run_id, canonical_key),
         UNIQUE (run_id, sequence)
       );
@@ -198,12 +199,12 @@ export class RunStore {
                 const inserted = destination.statement('INSERT INTO runs(status) VALUES (?)').run('running');
                 destination.runId = Number(inserted.lastInsertRowid);
                 const runId = destination.currentRunId();
-                for (const link of source.rawLinks()) {
-                    destination.statement('INSERT INTO raw_observations(run_id, source, link) VALUES (?, ?, ?)').run(runId, 'retry-seed', link);
+                for (const observation of source.rawObservations()) {
+                    destination.statement('INSERT INTO raw_observations(run_id, source, link) VALUES (?, ?, ?)').run(runId, observation.source, observation.link);
                 }
-                source.dedupedLinks().forEach((link, index) => {
-                    destination.statement('INSERT INTO pipeline_nodes(run_id, canonical_key, link, sequence) VALUES (?, ?, ?, ?)')
-                        .run(runId, canonicalVmessKey(parseVmessLink(link)), link, index + 1);
+                source.dedupedNodeOwnership().forEach(({ link, first_source }, index) => {
+                    destination.statement('INSERT INTO pipeline_nodes(run_id, canonical_key, link, sequence, first_source) VALUES (?, ?, ?, ?, ?)')
+                        .run(runId, canonicalVmessKey(parseVmessLink(link)), link, index + 1, first_source);
                 });
                 const preserveSpeed = boundary !== 'speedtest';
                 const preserveAvailability = !['speedtest', 'availability'].includes(boundary);
@@ -368,8 +369,8 @@ export class RunStore {
                 return { inserted: false, sequence: existing.sequence };
             const row = this.statement('SELECT COALESCE(MAX(sequence), 0) + 1 AS sequence FROM pipeline_nodes WHERE run_id = ?')
                 .get(runId);
-            this.statement('INSERT INTO pipeline_nodes(run_id, canonical_key, link, sequence) VALUES (?, ?, ?, ?)')
-                .run(runId, key, link, row.sequence);
+            this.statement('INSERT INTO pipeline_nodes(run_id, canonical_key, link, sequence, first_source) VALUES (?, ?, ?, ?, ?)')
+                .run(runId, key, link, row.sequence, source);
             return { inserted: true, sequence: row.sequence };
         });
     }
@@ -383,6 +384,24 @@ export class RunStore {
     dedupedLinks() {
         return this.statement('SELECT link FROM pipeline_nodes WHERE run_id = ? ORDER BY sequence').all(this.currentRunId()).map((row) => row.link);
     }
+    sourceDedupedCounts() {
+        const rows = this.statement(`SELECT first_source AS source, COUNT(*) AS count FROM pipeline_nodes
+      WHERE run_id = ? AND first_source IS NOT NULL GROUP BY first_source ORDER BY MIN(sequence)`)
+            .all(this.currentRunId());
+        return Object.fromEntries(rows.map((row) => [row.source, Number(row.count)]));
+    }
+    hasCompleteSourceOwnership() {
+        const row = this.statement(`SELECT NOT EXISTS(
+      SELECT 1 FROM pipeline_nodes WHERE run_id = ? AND first_source IS NULL
+    ) AS complete`).get(this.currentRunId());
+        return Boolean(row.complete);
+    }
+    sourceRawCounts() {
+        const rows = this.statement(`SELECT source, COUNT(*) AS count FROM raw_observations
+      WHERE run_id = ? GROUP BY source ORDER BY MIN(observation_id)`)
+            .all(this.currentRunId());
+        return Object.fromEntries(rows.map((row) => [row.source, Number(row.count)]));
+    }
     markSpeedRunning(link) {
         const { runId, key } = this.nodeIdentity(link);
         this.statement(`INSERT INTO speed_results(run_id, canonical_key, status) VALUES (?, ?, 'running')
@@ -483,8 +502,9 @@ export class RunStore {
         const speedRows = readLegacyReport(path.join(artifactDir, 'vpn_node_speedtest_report.json'));
         const availabilityRows = readLegacyReport(path.join(artifactDir, 'vpn_node_availability_report.json'));
         this.transaction(() => {
+            const rawCanonicalKeys = new Set();
             for (const link of rawLinks) {
-                parseVmessLink(link);
+                rawCanonicalKeys.add(canonicalVmessKey(parseVmessLink(link)));
                 this.statement('INSERT INTO raw_observations(run_id, source, link) VALUES (?, ?, ?)').run(runId, 'legacy', link);
             }
             const nodeLinks = dedupedLinks.length > 0 ? dedupedLinks : rawLinks;
@@ -494,8 +514,8 @@ export class RunStore {
                 if (seen.has(key))
                     continue;
                 seen.add(key);
-                this.statement('INSERT INTO pipeline_nodes(run_id, canonical_key, link, sequence) VALUES (?, ?, ?, ?)')
-                    .run(runId, key, link, seen.size);
+                this.statement('INSERT INTO pipeline_nodes(run_id, canonical_key, link, sequence, first_source) VALUES (?, ?, ?, ?, ?)')
+                    .run(runId, key, link, seen.size, rawCanonicalKeys.has(key) ? 'legacy' : null);
             }
             for (const row of speedRows) {
                 const link = String(row.link ?? '');
@@ -528,6 +548,14 @@ export class RunStore {
             throw new Error('unknown pipeline node');
         return { runId, key };
     }
+    rawObservations() {
+        return this.statement('SELECT source, link FROM raw_observations WHERE run_id = ? ORDER BY observation_id')
+            .all(this.currentRunId());
+    }
+    dedupedNodeOwnership() {
+        return this.statement('SELECT link, first_source FROM pipeline_nodes WHERE run_id = ? ORDER BY sequence')
+            .all(this.currentRunId());
+    }
     migrateSchema() {
         const columns = (table) => new Set(this.db.prepare(`PRAGMA table_info(${table})`).all().map((row) => row.name));
         this.db.exec('PRAGMA foreign_keys=OFF; PRAGMA legacy_alter_table=ON;');
@@ -535,6 +563,30 @@ export class RunStore {
         try {
             if (!columns('runs').has('error'))
                 this.db.exec("ALTER TABLE runs ADD COLUMN error TEXT NOT NULL DEFAULT ''");
+            if (!columns('pipeline_nodes').has('first_source')) {
+                this.db.exec('ALTER TABLE pipeline_nodes ADD COLUMN first_source TEXT');
+                const earliestSources = new Map();
+                const observations = this.db.prepare('SELECT run_id, source, link FROM raw_observations ORDER BY run_id, observation_id')
+                    .all();
+                for (const observation of observations) {
+                    try {
+                        const identity = `${observation.run_id}\0${canonicalVmessKey(parseVmessLink(observation.link))}`;
+                        if (!earliestSources.has(identity))
+                            earliestSources.set(identity, observation.source);
+                    }
+                    catch {
+                        // Historical malformed observations cannot establish canonical ownership.
+                    }
+                }
+                const nodes = this.db.prepare('SELECT node_id, run_id, canonical_key FROM pipeline_nodes ORDER BY run_id, sequence')
+                    .all();
+                const update = this.db.prepare('UPDATE pipeline_nodes SET first_source = ? WHERE node_id = ? AND first_source IS NULL');
+                for (const node of nodes) {
+                    const source = earliestSources.get(`${node.run_id}\0${node.canonical_key}`);
+                    if (source !== undefined)
+                        update.run(source, node.node_id);
+                }
+            }
             const stageColumns = columns('stage_events');
             if (!stageColumns.has('run_id')) {
                 this.db.exec('ALTER TABLE stage_events ADD COLUMN run_id INTEGER');
@@ -565,7 +617,7 @@ export class RunStore {
           DROP TABLE runs_v1;
         `);
             }
-            this.db.exec('PRAGMA user_version=2; COMMIT');
+            this.db.exec('PRAGMA user_version=3; COMMIT');
             this.db.exec('PRAGMA legacy_alter_table=OFF; PRAGMA foreign_keys=ON;');
         }
         catch (error) {