@swimmingliu/autovpn 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +102 -0
- package/bin/autovpn.mjs +5 -0
- package/dist/artifacts/list.js +99 -0
- package/dist/artifacts/preview.js +85 -0
- package/dist/backend/node-backend.js +194 -0
- package/dist/backend/python-backend.js +242 -0
- package/dist/backend/select-backend.js +12 -0
- package/dist/backend/types.js +1 -0
- package/dist/cli/commands/index.js +143 -0
- package/dist/cli/errors.js +7 -0
- package/dist/cli/global-options.js +29 -0
- package/dist/cli/main.js +231 -0
- package/dist/cli/native-commands.js +215 -0
- package/dist/cli/output.js +31 -0
- package/dist/config/profile.js +80 -0
- package/dist/doctor/checks.js +184 -0
- package/dist/events/schema.js +45 -0
- package/dist/jobs/commands.js +159 -0
- package/dist/jobs/logs.js +48 -0
- package/dist/jobs/process.js +75 -0
- package/dist/jobs/read.js +282 -0
- package/dist/jobs/store.js +115 -0
- package/dist/pipeline/availability.js +679 -0
- package/dist/pipeline/dedupe.js +104 -0
- package/dist/pipeline/deploy.js +1103 -0
- package/dist/pipeline/extract.js +259 -0
- package/dist/pipeline/obfuscate.js +203 -0
- package/dist/pipeline/orchestrator.js +1014 -0
- package/dist/pipeline/postprocess.js +214 -0
- package/dist/pipeline/proxy-runtime.js +228 -0
- package/dist/pipeline/render.js +91 -0
- package/dist/pipeline/speedtest.js +579 -0
- package/dist/runtime/env.js +35 -0
- package/dist/runtime/paths.js +66 -0
- package/dist/runtime/redaction.js +56 -0
- package/lib/cache.mjs +14 -0
- package/lib/errors.mjs +11 -0
- package/lib/install-python-cli.mjs +63 -0
- package/lib/runner.mjs +113 -0
- package/package.json +39 -0
|
@@ -0,0 +1,1014 @@
|
|
|
1
|
+
import { copyFile, mkdir, readFile, writeFile } from 'node:fs/promises';
|
|
2
|
+
import fs from 'node:fs';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
import { parse } from '@iarna/toml';
|
|
5
|
+
import { mergeProjectEnv } from '../runtime/env.js';
|
|
6
|
+
import { resolveArtifactsRoot, resolveProfilePath } from '../runtime/paths.js';
|
|
7
|
+
import { redactText } from '../runtime/redaction.js';
|
|
8
|
+
import { fetchSourceLinksWithBackend } from './extract.js';
|
|
9
|
+
import { dedupeVmessLinksWithBackend } from './dedupe.js';
|
|
10
|
+
import { normalizeSpeedTestConfig, probeSpeedtestLinksInNode, selectSpeedtestCandidates, speedtestLinksWithBackend, testSpeedtestLinkInNode } from './speedtest.js';
|
|
11
|
+
import { checkLinkAvailabilityBatchWithBackend } from './availability.js';
|
|
12
|
+
import { decorateLinkWithCountry, postprocessLinksWithBackend } from './postprocess.js';
|
|
13
|
+
import { renderMainDataWithBackend } from './render.js';
|
|
14
|
+
import { buildWorkerArtifactsWithBackend } from './obfuscate.js';
|
|
15
|
+
import { deployPagesWithBackend, isVerifySuccess, verifyDeploymentWithBackend } from './deploy.js';
|
|
16
|
+
import { safeDeployment } from '../runtime/redaction.js';
|
|
17
|
+
const STAGES = ['doctor', 'extract', 'dedupe', 'speedtest', 'availability', 'postprocess', 'render', 'obfuscate', 'deploy', 'verify'];
|
|
18
|
+
const RETRYABLE_STAGES = ['speedtest', 'availability', 'postprocess', 'render', 'obfuscate', 'deploy', 'verify'];
|
|
19
|
+
function formatTimestamp(date) {
|
|
20
|
+
const pad = (value) => String(value).padStart(2, '0');
|
|
21
|
+
return (`${date.getFullYear()}${pad(date.getMonth() + 1)}${pad(date.getDate())}-` +
|
|
22
|
+
`${pad(date.getHours())}${pad(date.getMinutes())}${pad(date.getSeconds())}`);
|
|
23
|
+
}
|
|
24
|
+
async function uniqueArtifactDir(root, timestamp) {
|
|
25
|
+
await mkdir(root, { recursive: true });
|
|
26
|
+
for (let index = 0; index < 1000; index += 1) {
|
|
27
|
+
const candidate = path.join(root, index === 0 ? timestamp : `${timestamp}-${index}`);
|
|
28
|
+
if (!fs.existsSync(candidate)) {
|
|
29
|
+
await mkdir(candidate, { recursive: true });
|
|
30
|
+
return candidate;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
throw new Error(`Unable to allocate artifact directory under ${root}`);
|
|
34
|
+
}
|
|
35
|
+
function stageStatus() {
|
|
36
|
+
return Object.fromEntries(STAGES.map((stage) => [stage, 'pending']));
|
|
37
|
+
}
|
|
38
|
+
function asProfile(payload) {
|
|
39
|
+
return (payload ?? {});
|
|
40
|
+
}
|
|
41
|
+
async function readProfile(projectRoot, env) {
|
|
42
|
+
const profilePath = resolveProfilePath(projectRoot, env);
|
|
43
|
+
const text = await readFile(profilePath, 'utf8');
|
|
44
|
+
return asProfile(parse(text));
|
|
45
|
+
}
|
|
46
|
+
function enabledSources(profile) {
|
|
47
|
+
return Object.entries(profile.sources ?? {}).filter(([, source]) => (source.enabled !== false
|
|
48
|
+
&& String(source.url ?? '').trim()
|
|
49
|
+
&& String(source.key ?? '').trim()));
|
|
50
|
+
}
|
|
51
|
+
function linesText(lines) {
|
|
52
|
+
return lines.length > 0 ? `${lines.join('\n')}\n` : '';
|
|
53
|
+
}
|
|
54
|
+
async function writeLines(artifactDir, filename, lines) {
|
|
55
|
+
await writeFile(path.join(artifactDir, filename), linesText(lines), 'utf8');
|
|
56
|
+
}
|
|
57
|
+
async function writeJson(artifactDir, filename, payload) {
|
|
58
|
+
await writeFile(path.join(artifactDir, filename), `${JSON.stringify(payload, null, 2)}\n`, 'utf8');
|
|
59
|
+
}
|
|
60
|
+
async function readJson(filePath, fallback) {
|
|
61
|
+
if (!fs.existsSync(filePath)) {
|
|
62
|
+
return fallback;
|
|
63
|
+
}
|
|
64
|
+
return JSON.parse(await readFile(filePath, 'utf8'));
|
|
65
|
+
}
|
|
66
|
+
async function readLines(filePath) {
|
|
67
|
+
if (!fs.existsSync(filePath)) {
|
|
68
|
+
return [];
|
|
69
|
+
}
|
|
70
|
+
return (await readFile(filePath, 'utf8')).split(/\r?\n/).map((line) => line.trim()).filter(Boolean);
|
|
71
|
+
}
|
|
72
|
+
async function copyIfExists(source, destination) {
|
|
73
|
+
if (!fs.existsSync(source)) {
|
|
74
|
+
return;
|
|
75
|
+
}
|
|
76
|
+
await mkdir(path.dirname(destination), { recursive: true });
|
|
77
|
+
await copyFile(source, destination);
|
|
78
|
+
}
|
|
79
|
+
function copyDirectoryIfExists(source, destination) {
|
|
80
|
+
if (!fs.existsSync(source)) {
|
|
81
|
+
return;
|
|
82
|
+
}
|
|
83
|
+
fs.mkdirSync(path.dirname(destination), { recursive: true });
|
|
84
|
+
fs.cpSync(source, destination, { recursive: true });
|
|
85
|
+
}
|
|
86
|
+
function appendTextFile(filePath, text) {
|
|
87
|
+
if (!filePath) {
|
|
88
|
+
return;
|
|
89
|
+
}
|
|
90
|
+
fs.mkdirSync(path.dirname(filePath), { recursive: true });
|
|
91
|
+
fs.appendFileSync(filePath, text, 'utf8');
|
|
92
|
+
}
|
|
93
|
+
function renderHumanEvent(event) {
|
|
94
|
+
if (event.type === 'run_started') {
|
|
95
|
+
return `[run_started] artifact_dir=${String(event.artifact_dir ?? '')} skip_deploy=${String(Boolean(event.skip_deploy))} skip_verify=${String(Boolean(event.skip_verify))}`;
|
|
96
|
+
}
|
|
97
|
+
if (event.type === 'log') {
|
|
98
|
+
return String(event.message ?? '');
|
|
99
|
+
}
|
|
100
|
+
if (event.type === 'stage') {
|
|
101
|
+
return `[stage] ${String(event.stage ?? '')}=${String(event.status ?? '')}`;
|
|
102
|
+
}
|
|
103
|
+
if (event.type === 'summary') {
|
|
104
|
+
const counts = Object.entries((event.counts ?? {}))
|
|
105
|
+
.sort(([left], [right]) => left.localeCompare(right))
|
|
106
|
+
.map(([name, value]) => `${name}=${String(value)}`)
|
|
107
|
+
.join(' ');
|
|
108
|
+
const suffix = counts ? ` ${counts}` : '';
|
|
109
|
+
return `[summary] run_status=${String(event.run_status ?? 'unknown')} artifact_dir=${String(event.artifact_dir ?? '')}${suffix}`;
|
|
110
|
+
}
|
|
111
|
+
if (event.type === 'run_failed') {
|
|
112
|
+
return `[run_failed] ${String(event.error ?? 'unknown error')}`;
|
|
113
|
+
}
|
|
114
|
+
return JSON.stringify(event);
|
|
115
|
+
}
|
|
116
|
+
function eventLogLine(event) {
|
|
117
|
+
return `${JSON.stringify(event)}\n`;
|
|
118
|
+
}
|
|
119
|
+
function humanLogLine(event) {
|
|
120
|
+
return `${renderHumanEvent(event)}\n`;
|
|
121
|
+
}
|
|
122
|
+
function errorMessage(error) {
|
|
123
|
+
if (error instanceof Error) {
|
|
124
|
+
return redactText(`${error.constructor.name}: ${error.message}`);
|
|
125
|
+
}
|
|
126
|
+
return redactText(String(error));
|
|
127
|
+
}
|
|
128
|
+
function defaultRuntimeStageEnv(env) {
|
|
129
|
+
return { ...env };
|
|
130
|
+
}
|
|
131
|
+
function orderPreservingUnique(values) {
|
|
132
|
+
const seen = new Set();
|
|
133
|
+
const result = [];
|
|
134
|
+
for (const value of values) {
|
|
135
|
+
if (seen.has(value)) {
|
|
136
|
+
continue;
|
|
137
|
+
}
|
|
138
|
+
seen.add(value);
|
|
139
|
+
result.push(value);
|
|
140
|
+
}
|
|
141
|
+
return result;
|
|
142
|
+
}
|
|
143
|
+
function defaultCountryFor(_link, _speedResult, _availabilityResult) {
|
|
144
|
+
return 'US';
|
|
145
|
+
}
|
|
146
|
+
function normalizeRetryStage(stage) {
|
|
147
|
+
if (RETRYABLE_STAGES.includes(stage)) {
|
|
148
|
+
return stage;
|
|
149
|
+
}
|
|
150
|
+
throw new Error(`Unsupported retry stage: ${stage}`);
|
|
151
|
+
}
|
|
152
|
+
function isStageAtOrAfter(candidate, target) {
|
|
153
|
+
return STAGES.indexOf(candidate) >= STAGES.indexOf(target);
|
|
154
|
+
}
|
|
155
|
+
function seedCompletedStages(summary, stage) {
|
|
156
|
+
for (const name of STAGES) {
|
|
157
|
+
if (name === stage) {
|
|
158
|
+
return;
|
|
159
|
+
}
|
|
160
|
+
summary.stage_status[name] = 'success';
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
async function seedRetryArtifact(sourceArtifactDir, retryArtifactDir, stage, retryContext, bundleSubdir) {
|
|
164
|
+
const sourceReport = await readJson(path.join(sourceArtifactDir, 'pipeline_report.json'), {});
|
|
165
|
+
const summary = {
|
|
166
|
+
artifact_dir: retryArtifactDir,
|
|
167
|
+
stage_status: stageStatus(),
|
|
168
|
+
counts: { ...(sourceReport.counts ?? {}) },
|
|
169
|
+
source_counts: { ...(sourceReport.source_counts ?? {}) },
|
|
170
|
+
deployment: { ...(sourceReport.deployment ?? {}) },
|
|
171
|
+
retry_context: retryContext,
|
|
172
|
+
run_status: 'success',
|
|
173
|
+
error: ''
|
|
174
|
+
};
|
|
175
|
+
await copyIfExists(path.join(sourceArtifactDir, 'vpn_node_raw.txt'), path.join(retryArtifactDir, 'vpn_node_raw.txt'));
|
|
176
|
+
await copyIfExists(path.join(sourceArtifactDir, 'vpn_node_deduped.txt'), path.join(retryArtifactDir, 'vpn_node_deduped.txt'));
|
|
177
|
+
if (isStageAtOrAfter(stage, 'availability')) {
|
|
178
|
+
await copyIfExists(path.join(sourceArtifactDir, 'vpn_node_speedtest.txt'), path.join(retryArtifactDir, 'vpn_node_speedtest.txt'));
|
|
179
|
+
await copyIfExists(path.join(sourceArtifactDir, 'vpn_node_speedtest_report.json'), path.join(retryArtifactDir, 'vpn_node_speedtest_report.json'));
|
|
180
|
+
}
|
|
181
|
+
if (isStageAtOrAfter(stage, 'postprocess')) {
|
|
182
|
+
await copyIfExists(path.join(sourceArtifactDir, 'vpn_node_availability.txt'), path.join(retryArtifactDir, 'vpn_node_availability.txt'));
|
|
183
|
+
await copyIfExists(path.join(sourceArtifactDir, 'vpn_node_availability_report.json'), path.join(retryArtifactDir, 'vpn_node_availability_report.json'));
|
|
184
|
+
}
|
|
185
|
+
if (isStageAtOrAfter(stage, 'render')) {
|
|
186
|
+
await copyIfExists(path.join(sourceArtifactDir, 'vpn_node_emoji.txt'), path.join(retryArtifactDir, 'vpn_node_emoji.txt'));
|
|
187
|
+
}
|
|
188
|
+
if (isStageAtOrAfter(stage, 'obfuscate')) {
|
|
189
|
+
await copyIfExists(path.join(sourceArtifactDir, 'vmess_node.js'), path.join(retryArtifactDir, 'vmess_node.js'));
|
|
190
|
+
}
|
|
191
|
+
if (isStageAtOrAfter(stage, 'deploy')) {
|
|
192
|
+
await copyIfExists(path.join(sourceArtifactDir, 'worker_transformed.js'), path.join(retryArtifactDir, 'worker_transformed.js'));
|
|
193
|
+
await copyIfExists(path.join(sourceArtifactDir, '_worker.js'), path.join(retryArtifactDir, '_worker.js'));
|
|
194
|
+
copyDirectoryIfExists(path.join(sourceArtifactDir, bundleSubdir), path.join(retryArtifactDir, bundleSubdir));
|
|
195
|
+
}
|
|
196
|
+
seedCompletedStages(summary, stage);
|
|
197
|
+
await writeJson(retryArtifactDir, 'pipeline_report.json', summary);
|
|
198
|
+
return summary;
|
|
199
|
+
}
|
|
200
|
+
function pipelineSummaryFromReport(artifactDir, report) {
|
|
201
|
+
return {
|
|
202
|
+
artifact_dir: artifactDir,
|
|
203
|
+
stage_status: { ...stageStatus(), ...(report.stage_status ?? {}) },
|
|
204
|
+
counts: { ...(report.counts ?? {}) },
|
|
205
|
+
source_counts: { ...(report.source_counts ?? {}) },
|
|
206
|
+
deployment: { ...(report.deployment ?? {}) },
|
|
207
|
+
retry_context: { ...(report.retry_context ?? {}) },
|
|
208
|
+
run_status: String(report.run_status ?? 'success') === 'failed' ? 'failed' : 'success',
|
|
209
|
+
error: String(report.error ?? '')
|
|
210
|
+
};
|
|
211
|
+
}
|
|
212
|
+
function passedSpeedResults(allResults, passedLinks) {
|
|
213
|
+
const byLink = new Map(allResults.map((result) => [result.link, result]));
|
|
214
|
+
return passedLinks
|
|
215
|
+
.map((link) => byLink.get(link))
|
|
216
|
+
.filter((result) => Boolean(result))
|
|
217
|
+
.sort((left, right) => right.average_download_mb_s - left.average_download_mb_s);
|
|
218
|
+
}
|
|
219
|
+
async function speedResultsFromEventLog(eventLog, passedLinks) {
|
|
220
|
+
if (!fs.existsSync(eventLog)) {
|
|
221
|
+
return [];
|
|
222
|
+
}
|
|
223
|
+
const passed = new Set(passedLinks);
|
|
224
|
+
const byLink = new Map();
|
|
225
|
+
for (const line of (await readFile(eventLog, 'utf8')).split(/\r?\n/)) {
|
|
226
|
+
if (!line.trim()) {
|
|
227
|
+
continue;
|
|
228
|
+
}
|
|
229
|
+
const payload = JSON.parse(line);
|
|
230
|
+
if (payload.type !== 'speedtest_result') {
|
|
231
|
+
continue;
|
|
232
|
+
}
|
|
233
|
+
const link = String(payload.link ?? '').trim();
|
|
234
|
+
if (!link || !passed.has(link)) {
|
|
235
|
+
continue;
|
|
236
|
+
}
|
|
237
|
+
byLink.set(link, {
|
|
238
|
+
link,
|
|
239
|
+
reachable: Boolean(payload.reachable),
|
|
240
|
+
average_download_mb_s: Number(payload.average_download_mb_s ?? 0) || 0,
|
|
241
|
+
latency_ms: Number(payload.latency_ms ?? 0) || 0,
|
|
242
|
+
error: String(payload.error ?? '')
|
|
243
|
+
});
|
|
244
|
+
}
|
|
245
|
+
return Array.from(byLink.values()).sort((left, right) => right.average_download_mb_s - left.average_download_mb_s);
|
|
246
|
+
}
|
|
247
|
+
async function restoreResumeSpeedResults(artifactDir, eventLog, passedLinks) {
|
|
248
|
+
const artifactResults = passedSpeedResults(await readJson(path.join(artifactDir, 'vpn_node_speedtest_report.json'), []), passedLinks);
|
|
249
|
+
if (artifactResults.length > 0) {
|
|
250
|
+
return artifactResults;
|
|
251
|
+
}
|
|
252
|
+
return speedResultsFromEventLog(eventLog, passedLinks);
|
|
253
|
+
}
|
|
254
|
+
async function forEachWithConcurrency(items, concurrency, mapper) {
|
|
255
|
+
let nextIndex = 0;
|
|
256
|
+
const workerCount = Math.max(1, Math.min(Math.max(1, Math.floor(concurrency)), items.length));
|
|
257
|
+
await Promise.all(Array.from({ length: workerCount }, async () => {
|
|
258
|
+
while (nextIndex < items.length) {
|
|
259
|
+
const index = nextIndex;
|
|
260
|
+
nextIndex += 1;
|
|
261
|
+
await mapper(items[index]);
|
|
262
|
+
}
|
|
263
|
+
}));
|
|
264
|
+
}
|
|
265
|
+
async function speedtestResumeStateFromEventLog(eventLog) {
|
|
266
|
+
const probes = new Map();
|
|
267
|
+
const fullResults = new Map();
|
|
268
|
+
if (!fs.existsSync(eventLog)) {
|
|
269
|
+
return { probes, fullResults };
|
|
270
|
+
}
|
|
271
|
+
for (const line of (await readFile(eventLog, 'utf8')).split(/\r?\n/)) {
|
|
272
|
+
if (!line.trim()) {
|
|
273
|
+
continue;
|
|
274
|
+
}
|
|
275
|
+
const payload = JSON.parse(line);
|
|
276
|
+
const link = String(payload.link ?? '').trim();
|
|
277
|
+
if (!link) {
|
|
278
|
+
continue;
|
|
279
|
+
}
|
|
280
|
+
if (payload.type === 'speedtest_probe_result') {
|
|
281
|
+
probes.set(link, {
|
|
282
|
+
link,
|
|
283
|
+
reachable: Boolean(payload.reachable),
|
|
284
|
+
latency_ms: Number(payload.latency_ms ?? 0) || 0,
|
|
285
|
+
error: String(payload.error ?? '')
|
|
286
|
+
});
|
|
287
|
+
}
|
|
288
|
+
else if (payload.type === 'speedtest_result') {
|
|
289
|
+
fullResults.set(link, {
|
|
290
|
+
link,
|
|
291
|
+
reachable: Boolean(payload.reachable),
|
|
292
|
+
average_download_mb_s: Number(payload.average_download_mb_s ?? 0) || 0,
|
|
293
|
+
latency_ms: Number(payload.latency_ms ?? 0) || 0,
|
|
294
|
+
error: String(payload.error ?? '')
|
|
295
|
+
});
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
return { probes, fullResults };
|
|
299
|
+
}
|
|
300
|
+
async function writeWorkerArtifacts(artifactDir, profile, renderedSource, artifacts) {
|
|
301
|
+
const workerBuild = profile.worker_build ?? {};
|
|
302
|
+
const entryFilename = String(workerBuild.entry_filename ?? 'unknown.js');
|
|
303
|
+
const bundleSubdir = String(workerBuild.bundle_subdir ?? 'pages_bundle');
|
|
304
|
+
const manifestFilename = String(workerBuild.manifest_filename ?? 'manifest.json');
|
|
305
|
+
const bundleDir = path.join(artifactDir, bundleSubdir);
|
|
306
|
+
await mkdir(bundleDir, { recursive: true });
|
|
307
|
+
await writeFile(path.join(artifactDir, 'vmess_node.js'), renderedSource, 'utf8');
|
|
308
|
+
await writeFile(path.join(artifactDir, 'worker_transformed.js'), artifacts.transformed_source, 'utf8');
|
|
309
|
+
await writeFile(path.join(artifactDir, entryFilename), artifacts.transformed_source, 'utf8');
|
|
310
|
+
await writeFile(path.join(bundleDir, entryFilename), artifacts.transformed_source, 'utf8');
|
|
311
|
+
await writeJson(bundleDir, manifestFilename, artifacts.manifest);
|
|
312
|
+
for (const [modulePath, moduleSource] of Object.entries(artifacts.modules ?? {})) {
|
|
313
|
+
const destination = path.join(bundleDir, modulePath);
|
|
314
|
+
await mkdir(path.dirname(destination), { recursive: true });
|
|
315
|
+
await writeFile(destination, moduleSource, 'utf8');
|
|
316
|
+
}
|
|
317
|
+
return bundleDir;
|
|
318
|
+
}
|
|
319
|
+
export async function runNodePipeline(options, context = {}) {
|
|
320
|
+
const projectRoot = path.resolve(options.projectRoot);
|
|
321
|
+
const env = mergeProjectEnv(projectRoot, { ...process.env, ...(context.env ?? {}) });
|
|
322
|
+
const runtimeStageEnv = defaultRuntimeStageEnv(env);
|
|
323
|
+
const artifactDir = await uniqueArtifactDir(resolveArtifactsRoot(projectRoot, env), formatTimestamp((context.now ?? (() => new Date()))()));
|
|
324
|
+
const summary = {
|
|
325
|
+
artifact_dir: artifactDir,
|
|
326
|
+
stage_status: stageStatus(),
|
|
327
|
+
counts: {},
|
|
328
|
+
source_counts: {},
|
|
329
|
+
deployment: {},
|
|
330
|
+
retry_context: {},
|
|
331
|
+
run_status: 'success',
|
|
332
|
+
error: ''
|
|
333
|
+
};
|
|
334
|
+
const emit = (type, payload = {}) => {
|
|
335
|
+
const event = { type, ...payload };
|
|
336
|
+
appendTextFile(options.eventLog, eventLogLine(event));
|
|
337
|
+
appendTextFile(options.humanLog, humanLogLine(event));
|
|
338
|
+
context.emit?.(event);
|
|
339
|
+
};
|
|
340
|
+
const writeReport = () => writeJson(artifactDir, 'pipeline_report.json', summary);
|
|
341
|
+
let activeStage;
|
|
342
|
+
const setStage = async (stage, status) => {
|
|
343
|
+
summary.stage_status[stage] = status;
|
|
344
|
+
activeStage = status === 'running' ? stage : activeStage === stage ? undefined : activeStage;
|
|
345
|
+
emit('stage', { stage, status });
|
|
346
|
+
await writeReport();
|
|
347
|
+
};
|
|
348
|
+
emit('run_started', {
|
|
349
|
+
artifact_dir: artifactDir,
|
|
350
|
+
skip_deploy: Boolean(options.skipDeploy),
|
|
351
|
+
skip_verify: Boolean(options.skipVerify || options.skipDeploy),
|
|
352
|
+
resume_from: ''
|
|
353
|
+
});
|
|
354
|
+
try {
|
|
355
|
+
await setStage('doctor', 'running');
|
|
356
|
+
await setStage('doctor', 'success');
|
|
357
|
+
const profile = await readProfile(projectRoot, env);
|
|
358
|
+
await setStage('extract', 'running');
|
|
359
|
+
const extractResults = [];
|
|
360
|
+
for (const [sourceName, source] of enabledSources(profile)) {
|
|
361
|
+
const result = context.stages?.extract
|
|
362
|
+
? await context.stages.extract({ source_name: sourceName, source })
|
|
363
|
+
: await fetchSourceLinksWithBackend({ source_name: sourceName, source }, { cwd: projectRoot, env: runtimeStageEnv });
|
|
364
|
+
extractResults.push(result);
|
|
365
|
+
summary.source_counts[result.source_name] = {
|
|
366
|
+
raw_links: result.links.length,
|
|
367
|
+
successful_iterations: result.successful_iterations,
|
|
368
|
+
failed_iterations: result.failed_iterations
|
|
369
|
+
};
|
|
370
|
+
}
|
|
371
|
+
const rawLinks = extractResults.flatMap((result) => result.links);
|
|
372
|
+
summary.counts.raw_links = rawLinks.length;
|
|
373
|
+
await writeLines(artifactDir, 'vpn_node_raw.txt', rawLinks);
|
|
374
|
+
await setStage('extract', 'success');
|
|
375
|
+
await setStage('dedupe', 'running');
|
|
376
|
+
const dedupedLinks = context.stages?.extract
|
|
377
|
+
? orderPreservingUnique(rawLinks)
|
|
378
|
+
: await dedupeVmessLinksWithBackend(rawLinks, { cwd: projectRoot, env });
|
|
379
|
+
summary.counts.deduped_links = dedupedLinks.length;
|
|
380
|
+
await writeLines(artifactDir, 'vpn_node_deduped.txt', dedupedLinks);
|
|
381
|
+
await setStage('dedupe', 'success');
|
|
382
|
+
await setStage('speedtest', 'running');
|
|
383
|
+
const runtimePath = path.join(artifactDir, 'runtime');
|
|
384
|
+
const speedResults = context.stages?.speedtest
|
|
385
|
+
? await context.stages.speedtest(dedupedLinks, profile.speed_test ?? {}, runtimePath)
|
|
386
|
+
: await speedtestLinksWithBackend({ links: dedupedLinks, config: profile.speed_test, runtime_path: runtimePath }, { cwd: projectRoot, env: runtimeStageEnv });
|
|
387
|
+
const passedSpeedLinks = speedResults
|
|
388
|
+
.filter((result) => result.reachable && result.average_download_mb_s >= Number(profile.speed_test?.min_download_mb_s ?? 0))
|
|
389
|
+
.map((result) => result.link);
|
|
390
|
+
summary.counts.speedtest_links = passedSpeedLinks.length;
|
|
391
|
+
await writeLines(artifactDir, 'vpn_node_speedtest.txt', passedSpeedLinks);
|
|
392
|
+
await writeJson(artifactDir, 'vpn_node_speedtest_report.json', speedResults);
|
|
393
|
+
await setStage('speedtest', 'success');
|
|
394
|
+
await setStage('availability', 'running');
|
|
395
|
+
const speedResultByLink = new Map(speedResults.map((result) => [result.link, result]));
|
|
396
|
+
const candidateSpeedResults = passedSpeedLinks.map((link) => speedResultByLink.get(link)).filter((result) => Boolean(result));
|
|
397
|
+
const availabilityResults = context.stages?.availability
|
|
398
|
+
? await context.stages.availability(candidateSpeedResults, profile.speed_test ?? {}, runtimePath, profile.availability_targets)
|
|
399
|
+
: await checkLinkAvailabilityBatchWithBackend({
|
|
400
|
+
results: candidateSpeedResults,
|
|
401
|
+
config: profile.speed_test ?? {},
|
|
402
|
+
runtime_path: runtimePath,
|
|
403
|
+
targets: profile.availability_targets
|
|
404
|
+
}, { cwd: projectRoot, env: runtimeStageEnv });
|
|
405
|
+
const availableLinks = availabilityResults.filter((result) => result.all_passed).map((result) => result.link);
|
|
406
|
+
summary.counts.availability_links = availableLinks.length;
|
|
407
|
+
await writeLines(artifactDir, 'vpn_node_availability.txt', availableLinks);
|
|
408
|
+
await writeJson(artifactDir, 'vpn_node_availability_report.json', availabilityResults);
|
|
409
|
+
await setStage('availability', 'success');
|
|
410
|
+
await setStage('postprocess', 'running');
|
|
411
|
+
const countryLookup = context.stages?.countryLookup ?? defaultCountryFor;
|
|
412
|
+
const availabilityByLink = new Map(availabilityResults.map((result) => [result.link, result]));
|
|
413
|
+
const rankedLinks = availableLinks.map((link) => ({
|
|
414
|
+
link,
|
|
415
|
+
country_code: countryLookup(link, speedResultByLink.get(link), availabilityByLink.get(link))
|
|
416
|
+
}));
|
|
417
|
+
const postprocessed = context.stages?.countryLookup
|
|
418
|
+
? { links: rankedLinks.map((item) => decorateLinkWithCountry(item.link, item.country_code)) }
|
|
419
|
+
: await postprocessLinksWithBackend({ ranked_links: rankedLinks, filters: profile.filters }, { cwd: projectRoot, env });
|
|
420
|
+
summary.counts.final_links = postprocessed.links.length;
|
|
421
|
+
await writeLines(artifactDir, 'vpn_node_emoji.txt', postprocessed.links);
|
|
422
|
+
await setStage('postprocess', 'success');
|
|
423
|
+
await setStage('render', 'running');
|
|
424
|
+
const template = await readFile(path.join(projectRoot, 'templates', 'vmess_node.js'), 'utf8');
|
|
425
|
+
const rendered = await renderMainDataWithBackend({ template, links: postprocessed.links }, { cwd: projectRoot, env });
|
|
426
|
+
await setStage('render', 'success');
|
|
427
|
+
await setStage('obfuscate', 'running');
|
|
428
|
+
const secretQuery = String(profile.deploy?.secret_query ?? '');
|
|
429
|
+
const workerArtifacts = context.stages?.obfuscate
|
|
430
|
+
? await context.stages.obfuscate({ transformedSource: rendered.rendered_source, config: profile.worker_build ?? {}, secretQuery })
|
|
431
|
+
: await buildWorkerArtifactsWithBackend({
|
|
432
|
+
rendered_source: rendered.rendered_source,
|
|
433
|
+
config: profile.worker_build,
|
|
434
|
+
secret_query: secretQuery
|
|
435
|
+
}, { cwd: projectRoot, env });
|
|
436
|
+
const bundleDir = await writeWorkerArtifacts(artifactDir, profile, rendered.rendered_source, workerArtifacts);
|
|
437
|
+
await setStage('obfuscate', 'success');
|
|
438
|
+
if (options.skipDeploy) {
|
|
439
|
+
await setStage('deploy', 'skipped');
|
|
440
|
+
}
|
|
441
|
+
else {
|
|
442
|
+
await setStage('deploy', 'running');
|
|
443
|
+
const deployment = context.stages?.deploy
|
|
444
|
+
? await context.stages.deploy({ projectRoot, bundleDir, profile })
|
|
445
|
+
: await deployPagesWithBackend({ projectRoot, bundleDir, deploy: profile.deploy ?? {} }, { cwd: projectRoot, env });
|
|
446
|
+
if (Number(deployment.returncode ?? 1) !== 0) {
|
|
447
|
+
summary.deployment = safeDeployment(deployment);
|
|
448
|
+
throw new Error(`Cloudflare deployment failed: ${JSON.stringify(summary.deployment)}`);
|
|
449
|
+
}
|
|
450
|
+
summary.deployment = safeDeployment(deployment);
|
|
451
|
+
await setStage('deploy', 'success');
|
|
452
|
+
}
|
|
453
|
+
const effectiveSkipVerify = Boolean(options.skipVerify || options.skipDeploy);
|
|
454
|
+
if (effectiveSkipVerify) {
|
|
455
|
+
await setStage('verify', 'skipped');
|
|
456
|
+
}
|
|
457
|
+
else {
|
|
458
|
+
await setStage('verify', 'running');
|
|
459
|
+
const verification = context.stages?.verify
|
|
460
|
+
? await context.stages.verify({ projectRoot, profile, deployment: summary.deployment })
|
|
461
|
+
: await verifyDeploymentWithBackend({ projectRoot, deploy: profile.deploy ?? {}, deployment: summary.deployment }, { cwd: projectRoot, env });
|
|
462
|
+
summary.deployment = safeDeployment({ ...summary.deployment, ...verification });
|
|
463
|
+
if (!isVerifySuccess(verification)) {
|
|
464
|
+
throw new Error(`Verification failed: ${JSON.stringify(summary.deployment)}`);
|
|
465
|
+
}
|
|
466
|
+
await setStage('verify', 'success');
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
catch (error) {
|
|
470
|
+
summary.run_status = 'failed';
|
|
471
|
+
summary.error = errorMessage(error);
|
|
472
|
+
if (activeStage && summary.stage_status[activeStage] === 'running') {
|
|
473
|
+
summary.stage_status[activeStage] = 'failed';
|
|
474
|
+
emit('stage', { stage: activeStage, status: 'failed' });
|
|
475
|
+
}
|
|
476
|
+
await writeReport();
|
|
477
|
+
emit('summary', summary);
|
|
478
|
+
emit('run_failed', { error: summary.error });
|
|
479
|
+
throw error;
|
|
480
|
+
}
|
|
481
|
+
await writeReport();
|
|
482
|
+
emit('summary', summary);
|
|
483
|
+
return summary;
|
|
484
|
+
}
|
|
485
|
+
export async function retryNodePipelineStage(options, context = {}) {
|
|
486
|
+
const projectRoot = path.resolve(options.projectRoot);
|
|
487
|
+
const sourceArtifactDir = path.resolve(options.artifactDir);
|
|
488
|
+
if (!fs.existsSync(sourceArtifactDir)) {
|
|
489
|
+
throw new Error(`artifact dir not found: ${sourceArtifactDir}`);
|
|
490
|
+
}
|
|
491
|
+
const stage = normalizeRetryStage(options.stage);
|
|
492
|
+
const env = mergeProjectEnv(projectRoot, { ...process.env, ...(context.env ?? {}) });
|
|
493
|
+
const runtimeStageEnv = defaultRuntimeStageEnv(env);
|
|
494
|
+
const profile = await readProfile(projectRoot, env);
|
|
495
|
+
const retryArtifactDir = await uniqueArtifactDir(resolveArtifactsRoot(projectRoot, env), formatTimestamp((context.now ?? (() => new Date()))()));
|
|
496
|
+
const retryContext = {
|
|
497
|
+
source_artifact_dir: sourceArtifactDir,
|
|
498
|
+
source_artifact_name: path.basename(sourceArtifactDir),
|
|
499
|
+
start_stage: stage
|
|
500
|
+
};
|
|
501
|
+
const summary = await seedRetryArtifact(sourceArtifactDir, retryArtifactDir, stage, retryContext, String(profile.worker_build?.bundle_subdir ?? 'pages_bundle'));
|
|
502
|
+
const emit = (type, payload = {}) => {
|
|
503
|
+
const event = { type, ...payload };
|
|
504
|
+
appendTextFile(options.eventLog, eventLogLine(event));
|
|
505
|
+
appendTextFile(options.humanLog, humanLogLine(event));
|
|
506
|
+
context.emit?.(event);
|
|
507
|
+
};
|
|
508
|
+
const writeReport = () => writeJson(retryArtifactDir, 'pipeline_report.json', summary);
|
|
509
|
+
let activeStage;
|
|
510
|
+
const setStage = async (stageName, status) => {
|
|
511
|
+
summary.stage_status[stageName] = status;
|
|
512
|
+
activeStage = status === 'running' ? stageName : activeStage === stageName ? undefined : activeStage;
|
|
513
|
+
emit('stage', { stage: stageName, status });
|
|
514
|
+
await writeReport();
|
|
515
|
+
};
|
|
516
|
+
const runtimePath = path.join(retryArtifactDir, 'runtime');
|
|
517
|
+
let speedResults = [];
|
|
518
|
+
let availabilityResults = [];
|
|
519
|
+
let finalLinks = [];
|
|
520
|
+
let bundleDir = path.join(retryArtifactDir, String(profile.worker_build?.bundle_subdir ?? 'pages_bundle'));
|
|
521
|
+
emit('run_started', {
|
|
522
|
+
artifact_dir: retryArtifactDir,
|
|
523
|
+
skip_deploy: false,
|
|
524
|
+
skip_verify: false,
|
|
525
|
+
retry_stage: stage,
|
|
526
|
+
source_artifact_dir: sourceArtifactDir
|
|
527
|
+
});
|
|
528
|
+
emit('log', { message: `[retry] source=${path.basename(sourceArtifactDir)} stage=${stage}` });
|
|
529
|
+
try {
|
|
530
|
+
if (stage === 'speedtest') {
|
|
531
|
+
const dedupedLinks = await readLines(path.join(sourceArtifactDir, 'vpn_node_deduped.txt'));
|
|
532
|
+
if (dedupedLinks.length === 0) {
|
|
533
|
+
throw new Error('No deduped links available to retry speedtest');
|
|
534
|
+
}
|
|
535
|
+
summary.counts.raw_links = (await readLines(path.join(sourceArtifactDir, 'vpn_node_raw.txt'))).length;
|
|
536
|
+
summary.counts.deduped_links = dedupedLinks.length;
|
|
537
|
+
await setStage('speedtest', 'running');
|
|
538
|
+
speedResults = context.stages?.speedtest
|
|
539
|
+
? await context.stages.speedtest(dedupedLinks, profile.speed_test ?? {}, runtimePath)
|
|
540
|
+
: await speedtestLinksWithBackend({ links: dedupedLinks, config: profile.speed_test, runtime_path: runtimePath }, { cwd: projectRoot, env: runtimeStageEnv });
|
|
541
|
+
const passedSpeedLinks = speedResults
|
|
542
|
+
.filter((result) => result.reachable && result.average_download_mb_s >= Number(profile.speed_test?.min_download_mb_s ?? 0))
|
|
543
|
+
.map((result) => result.link);
|
|
544
|
+
summary.counts.speedtest_links = passedSpeedLinks.length;
|
|
545
|
+
await writeLines(retryArtifactDir, 'vpn_node_speedtest.txt', passedSpeedLinks);
|
|
546
|
+
await writeJson(retryArtifactDir, 'vpn_node_speedtest_report.json', speedResults);
|
|
547
|
+
if (passedSpeedLinks.length === 0) {
|
|
548
|
+
await setStage('speedtest', 'failed');
|
|
549
|
+
summary.run_status = 'failed';
|
|
550
|
+
summary.error = 'Error: No links passed speed test';
|
|
551
|
+
await writeReport();
|
|
552
|
+
throw new Error('No links passed speed test');
|
|
553
|
+
}
|
|
554
|
+
speedResults = speedResults.filter((result) => passedSpeedLinks.includes(result.link));
|
|
555
|
+
await setStage('speedtest', 'success');
|
|
556
|
+
}
|
|
557
|
+
else {
|
|
558
|
+
const passedSpeedLinks = new Set(await readLines(path.join(retryArtifactDir, 'vpn_node_speedtest.txt')));
|
|
559
|
+
speedResults = (await readJson(path.join(retryArtifactDir, 'vpn_node_speedtest_report.json'), []))
|
|
560
|
+
.filter((result) => passedSpeedLinks.has(result.link));
|
|
561
|
+
}
|
|
562
|
+
if (isStageAtOrAfter('availability', stage)) {
|
|
563
|
+
if (speedResults.length === 0) {
|
|
564
|
+
throw new Error('No speedtest results available to retry availability');
|
|
565
|
+
}
|
|
566
|
+
await setStage('availability', 'running');
|
|
567
|
+
availabilityResults = context.stages?.availability
|
|
568
|
+
? await context.stages.availability(speedResults, profile.speed_test ?? {}, runtimePath, profile.availability_targets)
|
|
569
|
+
: await checkLinkAvailabilityBatchWithBackend({
|
|
570
|
+
results: speedResults,
|
|
571
|
+
config: profile.speed_test ?? {},
|
|
572
|
+
runtime_path: runtimePath,
|
|
573
|
+
targets: profile.availability_targets
|
|
574
|
+
}, { cwd: projectRoot, env: runtimeStageEnv });
|
|
575
|
+
const availableLinks = availabilityResults.filter((result) => result.all_passed).map((result) => result.link);
|
|
576
|
+
summary.counts.availability_links = availableLinks.length;
|
|
577
|
+
await writeLines(retryArtifactDir, 'vpn_node_availability.txt', availableLinks);
|
|
578
|
+
await writeJson(retryArtifactDir, 'vpn_node_availability_report.json', availabilityResults);
|
|
579
|
+
if (availableLinks.length === 0) {
|
|
580
|
+
await setStage('availability', 'failed');
|
|
581
|
+
summary.run_status = 'failed';
|
|
582
|
+
summary.error = 'Error: No links passed availability';
|
|
583
|
+
await writeReport();
|
|
584
|
+
throw new Error('No links passed availability');
|
|
585
|
+
}
|
|
586
|
+
await setStage('availability', 'success');
|
|
587
|
+
}
|
|
588
|
+
else if (isStageAtOrAfter(stage, 'postprocess')) {
|
|
589
|
+
const availableLinks = new Set(await readLines(path.join(retryArtifactDir, 'vpn_node_availability.txt')));
|
|
590
|
+
availabilityResults = (await readJson(path.join(retryArtifactDir, 'vpn_node_availability_report.json'), []))
|
|
591
|
+
.filter((result) => availableLinks.has(result.link));
|
|
592
|
+
}
|
|
593
|
+
if (isStageAtOrAfter('postprocess', stage)) {
|
|
594
|
+
if (availabilityResults.length === 0) {
|
|
595
|
+
throw new Error('No availability inputs available to retry postprocess');
|
|
596
|
+
}
|
|
597
|
+
await setStage('postprocess', 'running');
|
|
598
|
+
const speedResultByLink = new Map(speedResults.map((result) => [result.link, result]));
|
|
599
|
+
const countryLookup = context.stages?.countryLookup ?? defaultCountryFor;
|
|
600
|
+
const rankedLinks = availabilityResults.map((availabilityResult) => ({
|
|
601
|
+
link: availabilityResult.link,
|
|
602
|
+
country_code: countryLookup(availabilityResult.link, speedResultByLink.get(availabilityResult.link) ?? availabilityResult, availabilityResult)
|
|
603
|
+
}));
|
|
604
|
+
const postprocessed = context.stages?.countryLookup
|
|
605
|
+
? { links: rankedLinks.map((item) => decorateLinkWithCountry(item.link, item.country_code)) }
|
|
606
|
+
: await postprocessLinksWithBackend({ ranked_links: rankedLinks, filters: profile.filters }, { cwd: projectRoot, env });
|
|
607
|
+
finalLinks = postprocessed.links;
|
|
608
|
+
summary.counts.final_links = finalLinks.length;
|
|
609
|
+
summary.counts.postprocess_links = finalLinks.length;
|
|
610
|
+
await writeLines(retryArtifactDir, 'vpn_node_emoji.txt', finalLinks);
|
|
611
|
+
if (finalLinks.length === 0) {
|
|
612
|
+
await setStage('postprocess', 'failed');
|
|
613
|
+
summary.run_status = 'failed';
|
|
614
|
+
summary.error = 'Error: No links remained after postprocess filters';
|
|
615
|
+
await writeReport();
|
|
616
|
+
throw new Error('No links remained after postprocess filters');
|
|
617
|
+
}
|
|
618
|
+
await setStage('postprocess', 'success');
|
|
619
|
+
}
|
|
620
|
+
else if (isStageAtOrAfter(stage, 'render')) {
|
|
621
|
+
finalLinks = await readLines(path.join(retryArtifactDir, 'vpn_node_emoji.txt'));
|
|
622
|
+
}
|
|
623
|
+
if (isStageAtOrAfter('render', stage)) {
|
|
624
|
+
if (finalLinks.length === 0) {
|
|
625
|
+
throw new Error('No postprocess output available to retry render');
|
|
626
|
+
}
|
|
627
|
+
await setStage('render', 'running');
|
|
628
|
+
const template = await readFile(path.join(projectRoot, 'templates', 'vmess_node.js'), 'utf8');
|
|
629
|
+
const rendered = await renderMainDataWithBackend({ template, links: finalLinks }, { cwd: projectRoot, env });
|
|
630
|
+
await writeFile(path.join(retryArtifactDir, 'vmess_node.js'), rendered.rendered_source, 'utf8');
|
|
631
|
+
await setStage('render', 'success');
|
|
632
|
+
}
|
|
633
|
+
if (isStageAtOrAfter('obfuscate', stage)) {
|
|
634
|
+
const renderedPath = path.join(retryArtifactDir, 'vmess_node.js');
|
|
635
|
+
if (!fs.existsSync(renderedPath)) {
|
|
636
|
+
throw new Error('No rendered template available to retry obfuscate');
|
|
637
|
+
}
|
|
638
|
+
await setStage('obfuscate', 'running');
|
|
639
|
+
const renderedSource = await readFile(renderedPath, 'utf8');
|
|
640
|
+
const secretQuery = String(profile.deploy?.secret_query ?? '');
|
|
641
|
+
const workerArtifacts = context.stages?.obfuscate
|
|
642
|
+
? await context.stages.obfuscate({ transformedSource: renderedSource, config: profile.worker_build ?? {}, secretQuery })
|
|
643
|
+
: await buildWorkerArtifactsWithBackend({
|
|
644
|
+
rendered_source: renderedSource,
|
|
645
|
+
config: profile.worker_build,
|
|
646
|
+
secret_query: secretQuery
|
|
647
|
+
}, { cwd: projectRoot, env });
|
|
648
|
+
bundleDir = await writeWorkerArtifacts(retryArtifactDir, profile, renderedSource, workerArtifacts);
|
|
649
|
+
await setStage('obfuscate', 'success');
|
|
650
|
+
}
|
|
651
|
+
if (isStageAtOrAfter('deploy', stage)) {
|
|
652
|
+
if (!fs.existsSync(bundleDir)) {
|
|
653
|
+
throw new Error('No Pages bundle available to retry deploy');
|
|
654
|
+
}
|
|
655
|
+
await setStage('deploy', 'running');
|
|
656
|
+
const deployment = context.stages?.deploy
|
|
657
|
+
? await context.stages.deploy({ projectRoot, bundleDir, profile })
|
|
658
|
+
: await deployPagesWithBackend({ projectRoot, bundleDir, deploy: profile.deploy ?? {} }, { cwd: projectRoot, env });
|
|
659
|
+
if (Number(deployment.returncode ?? 1) !== 0) {
|
|
660
|
+
summary.deployment = safeDeployment(deployment);
|
|
661
|
+
throw new Error(`Cloudflare deployment failed: ${JSON.stringify(summary.deployment)}`);
|
|
662
|
+
}
|
|
663
|
+
summary.deployment = safeDeployment(deployment);
|
|
664
|
+
await setStage('deploy', 'success');
|
|
665
|
+
}
|
|
666
|
+
if (isStageAtOrAfter('verify', stage)) {
|
|
667
|
+
await setStage('verify', 'running');
|
|
668
|
+
const verification = context.stages?.verify
|
|
669
|
+
? await context.stages.verify({ projectRoot, profile, deployment: summary.deployment })
|
|
670
|
+
: await verifyDeploymentWithBackend({ projectRoot, deploy: profile.deploy ?? {}, deployment: summary.deployment }, { cwd: projectRoot, env });
|
|
671
|
+
summary.deployment = safeDeployment({ ...summary.deployment, ...verification });
|
|
672
|
+
if (!isVerifySuccess(verification)) {
|
|
673
|
+
throw new Error(`Verification failed: ${JSON.stringify(summary.deployment)}`);
|
|
674
|
+
}
|
|
675
|
+
await setStage('verify', 'success');
|
|
676
|
+
}
|
|
677
|
+
}
|
|
678
|
+
catch (error) {
|
|
679
|
+
summary.run_status = 'failed';
|
|
680
|
+
summary.error = errorMessage(error);
|
|
681
|
+
if (activeStage && summary.stage_status[activeStage] === 'running') {
|
|
682
|
+
summary.stage_status[activeStage] = 'failed';
|
|
683
|
+
emit('stage', { stage: activeStage, status: 'failed' });
|
|
684
|
+
}
|
|
685
|
+
await writeReport();
|
|
686
|
+
emit('summary', summary);
|
|
687
|
+
emit('run_failed', { error: summary.error });
|
|
688
|
+
throw error;
|
|
689
|
+
}
|
|
690
|
+
summary.run_status = 'success';
|
|
691
|
+
summary.error = '';
|
|
692
|
+
await writeReport();
|
|
693
|
+
emit('summary', summary);
|
|
694
|
+
return summary;
|
|
695
|
+
}
|
|
696
|
+
async function resumeNodeSpeedtest(options, context = {}) {
|
|
697
|
+
const projectRoot = path.resolve(options.projectRoot);
|
|
698
|
+
const sessionDir = path.resolve(options.session);
|
|
699
|
+
const sessionPath = path.join(sessionDir, 'session.json');
|
|
700
|
+
if (!fs.existsSync(sessionPath)) {
|
|
701
|
+
throw new Error(`session.json not found: ${sessionPath}`);
|
|
702
|
+
}
|
|
703
|
+
const sessionPayload = await readJson(sessionPath, {});
|
|
704
|
+
const artifactDirRaw = String(sessionPayload.artifact_dir ?? '').trim();
|
|
705
|
+
if (!artifactDirRaw) {
|
|
706
|
+
throw new Error(`session artifact_dir is required: ${sessionPath}`);
|
|
707
|
+
}
|
|
708
|
+
const artifactDir = path.resolve(artifactDirRaw);
|
|
709
|
+
if (!artifactDir || !fs.existsSync(artifactDir)) {
|
|
710
|
+
throw new Error(`artifact dir not found: ${artifactDir}`);
|
|
711
|
+
}
|
|
712
|
+
const eventLog = options.eventLog ?? String(sessionPayload.event_log ?? path.join(sessionDir, 'events.jsonl'));
|
|
713
|
+
const resumeEventLog = String(sessionPayload.event_log ?? path.join(sessionDir, 'events.jsonl'));
|
|
714
|
+
const humanLog = options.humanLog ?? String(sessionPayload.human_log ?? path.join(sessionDir, 'human.log'));
|
|
715
|
+
const env = mergeProjectEnv(projectRoot, { ...process.env, ...(context.env ?? {}) });
|
|
716
|
+
const profile = await readProfile(projectRoot, env);
|
|
717
|
+
const report = await readJson(path.join(artifactDir, 'pipeline_report.json'), {});
|
|
718
|
+
const summary = pipelineSummaryFromReport(artifactDir, report);
|
|
719
|
+
const runtimePath = path.join(artifactDir, 'runtime');
|
|
720
|
+
const emit = (type, payload = {}) => {
|
|
721
|
+
const event = { type, ...payload };
|
|
722
|
+
appendTextFile(eventLog, eventLogLine(event));
|
|
723
|
+
appendTextFile(humanLog, humanLogLine(event));
|
|
724
|
+
context.emit?.(event);
|
|
725
|
+
};
|
|
726
|
+
const writeReport = () => writeJson(artifactDir, 'pipeline_report.json', summary);
|
|
727
|
+
const setStage = async (stageName, status) => {
|
|
728
|
+
summary.stage_status[stageName] = status;
|
|
729
|
+
emit('stage', { stage: stageName, status });
|
|
730
|
+
await writeReport();
|
|
731
|
+
};
|
|
732
|
+
const rawLinks = await readLines(path.join(artifactDir, 'vpn_node_raw.txt'));
|
|
733
|
+
const dedupedLinks = await readLines(path.join(artifactDir, 'vpn_node_deduped.txt'));
|
|
734
|
+
summary.counts.raw_links = rawLinks.length;
|
|
735
|
+
summary.counts.deduped_links = dedupedLinks.length;
|
|
736
|
+
for (const baseStage of ['doctor', 'extract', 'dedupe']) {
|
|
737
|
+
if (summary.stage_status[baseStage] === 'pending') {
|
|
738
|
+
summary.stage_status[baseStage] = 'success';
|
|
739
|
+
}
|
|
740
|
+
}
|
|
741
|
+
const { probes, fullResults } = await speedtestResumeStateFromEventLog(resumeEventLog);
|
|
742
|
+
emit('speedtest_resume_state', {
|
|
743
|
+
resumed_probe_count: probes.size,
|
|
744
|
+
resumed_full_count: fullResults.size,
|
|
745
|
+
total_links: dedupedLinks.length
|
|
746
|
+
});
|
|
747
|
+
emit('log', { message: `[resume] speedtest resume from probe=${probes.size}/${dedupedLinks.length} full=${fullResults.size}` });
|
|
748
|
+
try {
|
|
749
|
+
await setStage('speedtest', 'running');
|
|
750
|
+
const speedConfig = normalizeSpeedTestConfig(profile.speed_test);
|
|
751
|
+
const requestedRuntime = String(env.AUTOVPN_SPEEDTEST_RUNTIME ?? '').trim().toLowerCase();
|
|
752
|
+
const usingInjectedSpeedtestStages = Boolean(context.stages?.speedtestProbe && context.stages?.speedtestLink);
|
|
753
|
+
if (requestedRuntime !== 'mihomo' && !usingInjectedSpeedtestStages) {
|
|
754
|
+
throw new Error('Node resume speedtest requires AUTOVPN_SPEEDTEST_RUNTIME=mihomo');
|
|
755
|
+
}
|
|
756
|
+
const remainingProbeLinks = dedupedLinks.filter((link) => !probes.has(link));
|
|
757
|
+
if (remainingProbeLinks.length > 0) {
|
|
758
|
+
const probeResults = context.stages?.speedtestProbe
|
|
759
|
+
? await context.stages.speedtestProbe(remainingProbeLinks, profile.speed_test ?? {}, runtimePath)
|
|
760
|
+
: await probeSpeedtestLinksInNode({ links: remainingProbeLinks, config: profile.speed_test, runtime_path: runtimePath }, { cwd: projectRoot, env });
|
|
761
|
+
for (let index = 0; index < probeResults.length; index += 1) {
|
|
762
|
+
const result = probeResults[index];
|
|
763
|
+
probes.set(result.link, result);
|
|
764
|
+
const completed = probes.size;
|
|
765
|
+
emit('log', { message: `[speedtest:probe] ${completed}/${dedupedLinks.length} reachable=${result.reachable} latency=${result.latency_ms}ms` });
|
|
766
|
+
emit('speedtest_probe_result', {
|
|
767
|
+
completed,
|
|
768
|
+
total: dedupedLinks.length,
|
|
769
|
+
link: result.link,
|
|
770
|
+
reachable: result.reachable,
|
|
771
|
+
latency_ms: result.latency_ms,
|
|
772
|
+
error: result.error ?? ''
|
|
773
|
+
});
|
|
774
|
+
}
|
|
775
|
+
}
|
|
776
|
+
const orderedProbes = dedupedLinks.map((link) => probes.get(link)).filter((result) => Boolean(result));
|
|
777
|
+
const candidateLinks = selectSpeedtestCandidates(orderedProbes, speedConfig.max_download_candidates);
|
|
778
|
+
const reachableCount = orderedProbes.filter((probe) => probe.reachable).length;
|
|
779
|
+
emit('log', { message: `[speedtest] selected ${candidateLinks.length}/${reachableCount} reachable links for full download test` });
|
|
780
|
+
emit('speedtest_selected', {
|
|
781
|
+
total_links: dedupedLinks.length,
|
|
782
|
+
reachable_count: reachableCount,
|
|
783
|
+
candidate_count: candidateLinks.length
|
|
784
|
+
});
|
|
785
|
+
const remainingFullLinks = candidateLinks.filter((link) => !fullResults.has(link));
|
|
786
|
+
await forEachWithConcurrency(remainingFullLinks, speedConfig.concurrency, async (link) => {
|
|
787
|
+
const result = context.stages?.speedtestLink
|
|
788
|
+
? await context.stages.speedtestLink(link, profile.speed_test ?? {}, runtimePath)
|
|
789
|
+
: await testSpeedtestLinkInNode({ link, config: profile.speed_test, runtime_path: runtimePath }, { cwd: projectRoot, env });
|
|
790
|
+
if (result.reachable && result.latency_ms <= 0) {
|
|
791
|
+
result.latency_ms = probes.get(result.link)?.latency_ms ?? 0;
|
|
792
|
+
}
|
|
793
|
+
fullResults.set(result.link, result);
|
|
794
|
+
const completed = fullResults.size;
|
|
795
|
+
const passedThreshold = result.reachable && result.average_download_mb_s >= speedConfig.min_download_mb_s;
|
|
796
|
+
emit('log', { message: `[speedtest] ${completed}/${candidateLinks.length} reachable=${result.reachable} speed=${result.average_download_mb_s}MB/s` });
|
|
797
|
+
emit('speedtest_result', {
|
|
798
|
+
completed,
|
|
799
|
+
total: candidateLinks.length,
|
|
800
|
+
link: result.link,
|
|
801
|
+
reachable: result.reachable,
|
|
802
|
+
average_download_mb_s: result.average_download_mb_s,
|
|
803
|
+
latency_ms: result.latency_ms,
|
|
804
|
+
passed_threshold: passedThreshold,
|
|
805
|
+
error: result.error ?? ''
|
|
806
|
+
});
|
|
807
|
+
});
|
|
808
|
+
const orderedFullResults = candidateLinks.map((link) => fullResults.get(link)).filter((result) => Boolean(result));
|
|
809
|
+
const fastResults = orderedFullResults
|
|
810
|
+
.filter((result) => result.reachable && result.average_download_mb_s >= speedConfig.min_download_mb_s)
|
|
811
|
+
.sort((left, right) => right.average_download_mb_s - left.average_download_mb_s);
|
|
812
|
+
await writeLines(artifactDir, 'vpn_node_speedtest.txt', fastResults.map((result) => result.link));
|
|
813
|
+
await writeJson(artifactDir, 'vpn_node_speedtest_report.json', fastResults);
|
|
814
|
+
summary.counts.speedtest_links = fastResults.length;
|
|
815
|
+
emit('log', { message: `[speedtest] kept ${fastResults.length} links above threshold` });
|
|
816
|
+
if (fastResults.length === 0) {
|
|
817
|
+
await setStage('speedtest', 'failed');
|
|
818
|
+
summary.run_status = 'failed';
|
|
819
|
+
summary.error = 'Error: No links passed speed test';
|
|
820
|
+
await writeReport();
|
|
821
|
+
throw new Error('No links passed speed test');
|
|
822
|
+
}
|
|
823
|
+
await setStage('speedtest', 'success');
|
|
824
|
+
summary.run_status = 'success';
|
|
825
|
+
summary.error = '';
|
|
826
|
+
await writeReport();
|
|
827
|
+
emit('summary', summary);
|
|
828
|
+
return summary;
|
|
829
|
+
}
|
|
830
|
+
catch (error) {
|
|
831
|
+
summary.run_status = 'failed';
|
|
832
|
+
summary.error = errorMessage(error);
|
|
833
|
+
if (summary.stage_status.speedtest === 'running') {
|
|
834
|
+
summary.stage_status.speedtest = 'failed';
|
|
835
|
+
emit('stage', { stage: 'speedtest', status: 'failed' });
|
|
836
|
+
}
|
|
837
|
+
await writeReport();
|
|
838
|
+
emit('summary', summary);
|
|
839
|
+
emit('run_failed', { error: summary.error });
|
|
840
|
+
throw error;
|
|
841
|
+
}
|
|
842
|
+
}
|
|
843
|
+
export async function resumeNodePipeline(options, context = {}) {
|
|
844
|
+
if (options.mode === 'speedtest') {
|
|
845
|
+
return resumeNodeSpeedtest(options, context);
|
|
846
|
+
}
|
|
847
|
+
const projectRoot = path.resolve(options.projectRoot);
|
|
848
|
+
const sessionDir = path.resolve(options.session);
|
|
849
|
+
const sessionPath = path.join(sessionDir, 'session.json');
|
|
850
|
+
if (!fs.existsSync(sessionPath)) {
|
|
851
|
+
throw new Error(`session.json not found: ${sessionPath}`);
|
|
852
|
+
}
|
|
853
|
+
const sessionPayload = await readJson(sessionPath, {});
|
|
854
|
+
const artifactDirRaw = String(sessionPayload.artifact_dir ?? '').trim();
|
|
855
|
+
if (!artifactDirRaw) {
|
|
856
|
+
throw new Error(`session artifact_dir is required: ${sessionPath}`);
|
|
857
|
+
}
|
|
858
|
+
const artifactDir = path.resolve(artifactDirRaw);
|
|
859
|
+
if (!artifactDir || !fs.existsSync(artifactDir)) {
|
|
860
|
+
throw new Error(`artifact dir not found: ${artifactDir}`);
|
|
861
|
+
}
|
|
862
|
+
const eventLog = options.eventLog ?? String(sessionPayload.event_log ?? path.join(sessionDir, 'events.jsonl'));
|
|
863
|
+
const humanLog = options.humanLog ?? String(sessionPayload.human_log ?? path.join(sessionDir, 'human.log'));
|
|
864
|
+
const env = mergeProjectEnv(projectRoot, { ...process.env, ...(context.env ?? {}) });
|
|
865
|
+
const runtimeStageEnv = defaultRuntimeStageEnv(env);
|
|
866
|
+
const profile = await readProfile(projectRoot, env);
|
|
867
|
+
const report = await readJson(path.join(artifactDir, 'pipeline_report.json'), {});
|
|
868
|
+
const summary = pipelineSummaryFromReport(artifactDir, report);
|
|
869
|
+
const runtimePath = path.join(artifactDir, 'runtime');
|
|
870
|
+
const emit = (type, payload = {}) => {
|
|
871
|
+
const event = { type, ...payload };
|
|
872
|
+
appendTextFile(eventLog, eventLogLine(event));
|
|
873
|
+
appendTextFile(humanLog, humanLogLine(event));
|
|
874
|
+
context.emit?.(event);
|
|
875
|
+
};
|
|
876
|
+
const writeReport = () => writeJson(artifactDir, 'pipeline_report.json', summary);
|
|
877
|
+
let activeStage;
|
|
878
|
+
const setStage = async (stageName, status) => {
|
|
879
|
+
summary.stage_status[stageName] = status;
|
|
880
|
+
activeStage = status === 'running' ? stageName : activeStage === stageName ? undefined : activeStage;
|
|
881
|
+
emit('stage', { stage: stageName, status });
|
|
882
|
+
await writeReport();
|
|
883
|
+
};
|
|
884
|
+
const rawLinks = await readLines(path.join(artifactDir, 'vpn_node_raw.txt'));
|
|
885
|
+
const dedupedLinks = await readLines(path.join(artifactDir, 'vpn_node_deduped.txt'));
|
|
886
|
+
const speedtestLinks = await readLines(path.join(artifactDir, 'vpn_node_speedtest.txt'));
|
|
887
|
+
const speedResults = await restoreResumeSpeedResults(artifactDir, eventLog, speedtestLinks);
|
|
888
|
+
summary.counts.raw_links = rawLinks.length;
|
|
889
|
+
summary.counts.deduped_links = dedupedLinks.length;
|
|
890
|
+
summary.counts.speedtest_links = speedResults.length;
|
|
891
|
+
try {
|
|
892
|
+
if (speedResults.length === 0) {
|
|
893
|
+
summary.stage_status.speedtest = 'failed';
|
|
894
|
+
throw new Error('No speedtest results available to continue pipeline');
|
|
895
|
+
}
|
|
896
|
+
for (const baseStage of ['doctor', 'extract', 'dedupe', 'speedtest']) {
|
|
897
|
+
if (summary.stage_status[baseStage] === 'pending' || summary.stage_status[baseStage] === 'failed') {
|
|
898
|
+
summary.stage_status[baseStage] = 'success';
|
|
899
|
+
}
|
|
900
|
+
}
|
|
901
|
+
emit('resume_pipeline_state', {
|
|
902
|
+
speedtest_links: speedResults.length,
|
|
903
|
+
artifact_dir: artifactDir
|
|
904
|
+
});
|
|
905
|
+
emit('log', { message: `[resume] continue pipeline from speedtest_links=${speedResults.length}` });
|
|
906
|
+
await writeReport();
|
|
907
|
+
await setStage('availability', 'running');
|
|
908
|
+
const availabilityResults = context.stages?.availability
|
|
909
|
+
? await context.stages.availability(speedResults, profile.speed_test ?? {}, runtimePath, profile.availability_targets)
|
|
910
|
+
: await checkLinkAvailabilityBatchWithBackend({
|
|
911
|
+
results: speedResults,
|
|
912
|
+
config: profile.speed_test ?? {},
|
|
913
|
+
runtime_path: runtimePath,
|
|
914
|
+
targets: profile.availability_targets
|
|
915
|
+
}, { cwd: projectRoot, env: runtimeStageEnv });
|
|
916
|
+
const availableLinks = availabilityResults.filter((result) => result.all_passed).map((result) => result.link);
|
|
917
|
+
summary.counts.availability_links = availableLinks.length;
|
|
918
|
+
await writeLines(artifactDir, 'vpn_node_availability.txt', availableLinks);
|
|
919
|
+
await writeJson(artifactDir, 'vpn_node_availability_report.json', availabilityResults);
|
|
920
|
+
if (availableLinks.length === 0) {
|
|
921
|
+
await setStage('availability', 'failed');
|
|
922
|
+
summary.run_status = 'failed';
|
|
923
|
+
summary.error = 'Error: No links passed availability';
|
|
924
|
+
await writeReport();
|
|
925
|
+
throw new Error('No links passed availability');
|
|
926
|
+
}
|
|
927
|
+
await setStage('availability', 'success');
|
|
928
|
+
await setStage('postprocess', 'running');
|
|
929
|
+
const speedResultByLink = new Map(speedResults.map((result) => [result.link, result]));
|
|
930
|
+
const availabilityByLink = new Map(availabilityResults.map((result) => [result.link, result]));
|
|
931
|
+
const countryLookup = context.stages?.countryLookup ?? defaultCountryFor;
|
|
932
|
+
const rankedLinks = availableLinks.map((link) => ({
|
|
933
|
+
link,
|
|
934
|
+
country_code: countryLookup(link, speedResultByLink.get(link), availabilityByLink.get(link))
|
|
935
|
+
}));
|
|
936
|
+
const postprocessed = context.stages?.countryLookup
|
|
937
|
+
? { links: rankedLinks.map((item) => decorateLinkWithCountry(item.link, item.country_code)) }
|
|
938
|
+
: await postprocessLinksWithBackend({ ranked_links: rankedLinks, filters: profile.filters }, { cwd: projectRoot, env });
|
|
939
|
+
summary.counts.postprocess_links = postprocessed.links.length;
|
|
940
|
+
summary.counts.final_links = postprocessed.links.length;
|
|
941
|
+
await writeLines(artifactDir, 'vpn_node_emoji.txt', postprocessed.links);
|
|
942
|
+
if (postprocessed.links.length === 0) {
|
|
943
|
+
await setStage('postprocess', 'failed');
|
|
944
|
+
summary.run_status = 'failed';
|
|
945
|
+
summary.error = 'Error: No links remained after postprocess filters';
|
|
946
|
+
await writeReport();
|
|
947
|
+
throw new Error('No links remained after postprocess filters');
|
|
948
|
+
}
|
|
949
|
+
await setStage('postprocess', 'success');
|
|
950
|
+
await setStage('render', 'running');
|
|
951
|
+
const template = await readFile(path.join(projectRoot, 'templates', 'vmess_node.js'), 'utf8');
|
|
952
|
+
const rendered = await renderMainDataWithBackend({ template, links: postprocessed.links }, { cwd: projectRoot, env });
|
|
953
|
+
await writeFile(path.join(artifactDir, 'vmess_node.js'), rendered.rendered_source, 'utf8');
|
|
954
|
+
await setStage('render', 'success');
|
|
955
|
+
await setStage('obfuscate', 'running');
|
|
956
|
+
const secretQuery = String(profile.deploy?.secret_query ?? '');
|
|
957
|
+
const workerArtifacts = context.stages?.obfuscate
|
|
958
|
+
? await context.stages.obfuscate({ transformedSource: rendered.rendered_source, config: profile.worker_build ?? {}, secretQuery })
|
|
959
|
+
: await buildWorkerArtifactsWithBackend({
|
|
960
|
+
rendered_source: rendered.rendered_source,
|
|
961
|
+
config: profile.worker_build,
|
|
962
|
+
secret_query: secretQuery
|
|
963
|
+
}, { cwd: projectRoot, env });
|
|
964
|
+
const bundleDir = await writeWorkerArtifacts(artifactDir, profile, rendered.rendered_source, workerArtifacts);
|
|
965
|
+
await setStage('obfuscate', 'success');
|
|
966
|
+
if (options.skipDeploy) {
|
|
967
|
+
await setStage('deploy', 'skipped');
|
|
968
|
+
await setStage('verify', 'skipped');
|
|
969
|
+
}
|
|
970
|
+
else {
|
|
971
|
+
await setStage('deploy', 'running');
|
|
972
|
+
const deployment = context.stages?.deploy
|
|
973
|
+
? await context.stages.deploy({ projectRoot, bundleDir, profile })
|
|
974
|
+
: await deployPagesWithBackend({ projectRoot, bundleDir, deploy: profile.deploy ?? {} }, { cwd: projectRoot, env });
|
|
975
|
+
if (Number(deployment.returncode ?? 1) !== 0) {
|
|
976
|
+
summary.deployment = safeDeployment(deployment);
|
|
977
|
+
throw new Error(`Cloudflare deployment failed: ${JSON.stringify(summary.deployment)}`);
|
|
978
|
+
}
|
|
979
|
+
summary.deployment = safeDeployment(deployment);
|
|
980
|
+
await setStage('deploy', 'success');
|
|
981
|
+
if (options.skipVerify) {
|
|
982
|
+
await setStage('verify', 'skipped');
|
|
983
|
+
}
|
|
984
|
+
else {
|
|
985
|
+
await setStage('verify', 'running');
|
|
986
|
+
const verification = context.stages?.verify
|
|
987
|
+
? await context.stages.verify({ projectRoot, profile, deployment: summary.deployment })
|
|
988
|
+
: await verifyDeploymentWithBackend({ projectRoot, deploy: profile.deploy ?? {}, deployment: summary.deployment }, { cwd: projectRoot, env });
|
|
989
|
+
summary.deployment = safeDeployment({ ...summary.deployment, ...verification });
|
|
990
|
+
if (!isVerifySuccess(verification)) {
|
|
991
|
+
throw new Error(`Verification failed: ${JSON.stringify(summary.deployment)}`);
|
|
992
|
+
}
|
|
993
|
+
await setStage('verify', 'success');
|
|
994
|
+
}
|
|
995
|
+
}
|
|
996
|
+
}
|
|
997
|
+
catch (error) {
|
|
998
|
+
summary.run_status = 'failed';
|
|
999
|
+
summary.error = errorMessage(error);
|
|
1000
|
+
if (activeStage && summary.stage_status[activeStage] === 'running') {
|
|
1001
|
+
summary.stage_status[activeStage] = 'failed';
|
|
1002
|
+
emit('stage', { stage: activeStage, status: 'failed' });
|
|
1003
|
+
}
|
|
1004
|
+
await writeReport();
|
|
1005
|
+
emit('summary', summary);
|
|
1006
|
+
emit('run_failed', { error: summary.error });
|
|
1007
|
+
throw error;
|
|
1008
|
+
}
|
|
1009
|
+
summary.run_status = 'success';
|
|
1010
|
+
summary.error = '';
|
|
1011
|
+
await writeReport();
|
|
1012
|
+
emit('summary', summary);
|
|
1013
|
+
return summary;
|
|
1014
|
+
}
|