@swimmingliu/autovpn 1.6.1 → 1.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backend/node-backend.js +1 -1
- package/dist/backend/select-backend.js +1 -2
- package/dist/cli/main.js +23 -14
- package/dist/cli/native-commands.js +12 -12
- package/dist/jobs/commands.js +8 -19
- package/dist/pipeline/availability.js +25 -12
- package/dist/pipeline/dedupe.js +3 -5
- package/dist/pipeline/deploy.js +3 -5
- package/dist/pipeline/extract.js +35 -7
- package/dist/pipeline/obfuscate.js +3 -5
- package/dist/pipeline/orchestrator.js +240 -39
- package/dist/pipeline/postprocess.js +3 -5
- package/dist/pipeline/proxy-runtime.js +42 -6
- package/dist/pipeline/render.js +3 -5
- package/dist/pipeline/speedtest.js +72 -36
- package/dist/runtime/managed-tools.js +9 -2
- package/dist/server/http.js +11 -8
- package/dist/server/runtime.js +40 -3
- package/dist/web/renderer/app.js +62 -4
- package/dist/web/renderer/i18n.js +1 -1
- package/dist/web/renderer/styles.css +321 -1
- package/dist/web/renderer/views.js +8 -2
- package/package.json +1 -1
|
@@ -5,7 +5,7 @@ import { resumeNodePipeline, retryNodePipelineStage, runNodePipeline } from '../
|
|
|
5
5
|
import { resolveArtifactsRoot } from '../runtime/paths.js';
|
|
6
6
|
const require = createRequire(import.meta.url);
|
|
7
7
|
function unsupported(method) {
|
|
8
|
-
return new Error(`Node backend ${method} is not available
|
|
8
|
+
return new Error(`Node backend ${method} is not available in this execution path`);
|
|
9
9
|
}
|
|
10
10
|
function latestIncompleteRunArtifact(projectRoot, env) {
|
|
11
11
|
const artifactsRoot = resolveArtifactsRoot(projectRoot, env);
|
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
import { NodeBackend } from './node-backend.js';
|
|
2
|
-
import { PythonBackend } from './python-backend.js';
|
|
3
2
|
export function selectBackend(options = {}) {
|
|
4
3
|
const backend = String(options.env?.AUTOVPN_BACKEND ?? '').trim().toLowerCase();
|
|
5
4
|
if (backend === 'python') {
|
|
6
|
-
|
|
5
|
+
throw new Error('AUTOVPN_BACKEND=python is no longer supported; AutoVPN now runs on the NodeJS engine');
|
|
7
6
|
}
|
|
8
7
|
if (backend && backend !== 'node') {
|
|
9
8
|
throw new Error(`Unsupported AUTOVPN_BACKEND: ${backend}`);
|
package/dist/cli/main.js
CHANGED
|
@@ -35,6 +35,10 @@ async function defaultReadStdin() {
|
|
|
35
35
|
function isEnabled(value) {
|
|
36
36
|
return ['1', 'true', 'yes', 'on'].includes(String(value ?? '').trim().toLowerCase());
|
|
37
37
|
}
|
|
38
|
+
function wantsDeprecatedPythonShell(env) {
|
|
39
|
+
return String(env.AUTOVPN_CLI_SHELL ?? '').trim().toLowerCase() === 'python'
|
|
40
|
+
|| String(env.AUTOVPN_BACKEND ?? '').trim().toLowerCase() === 'python';
|
|
41
|
+
}
|
|
38
42
|
async function resolvePackageVersion(options) {
|
|
39
43
|
if (options.packageVersion) {
|
|
40
44
|
return options.packageVersion;
|
|
@@ -86,6 +90,20 @@ function resolveResumeSessionDir(job) {
|
|
|
86
90
|
}
|
|
87
91
|
return '';
|
|
88
92
|
}
|
|
93
|
+
async function waitForServeShutdown(server) {
|
|
94
|
+
let closing;
|
|
95
|
+
const closeOnce = () => {
|
|
96
|
+
closing ??= server.close();
|
|
97
|
+
return closing;
|
|
98
|
+
};
|
|
99
|
+
await new Promise((resolve) => {
|
|
100
|
+
const shutdown = () => {
|
|
101
|
+
void closeOnce().finally(resolve);
|
|
102
|
+
};
|
|
103
|
+
process.once('SIGINT', shutdown);
|
|
104
|
+
process.once('SIGTERM', shutdown);
|
|
105
|
+
});
|
|
106
|
+
}
|
|
89
107
|
async function runForegroundPipeline(argv, backend, io, cwd) {
|
|
90
108
|
if (backend.kind !== 'node') {
|
|
91
109
|
return undefined;
|
|
@@ -183,8 +201,9 @@ export async function runCliShell(argv, options = {}) {
|
|
|
183
201
|
const io = options.io ?? defaultIo();
|
|
184
202
|
const runForwarder = options.runForwarder ?? defaultRunForwarder;
|
|
185
203
|
const cwd = options.cwd ?? process.cwd();
|
|
186
|
-
if (env
|
|
187
|
-
|
|
204
|
+
if (wantsDeprecatedPythonShell(env)) {
|
|
205
|
+
io.writeStderr('autovpn: Python backend is no longer supported; AutoVPN now runs on the NodeJS engine\n');
|
|
206
|
+
return 2;
|
|
188
207
|
}
|
|
189
208
|
if (isEnabled(env.AUTOVPN_WRAPPER_PROBE) && argv.length === 1 && argv[0] === '--version') {
|
|
190
209
|
return 42;
|
|
@@ -233,27 +252,17 @@ export async function runCliShell(argv, options = {}) {
|
|
|
233
252
|
await server.close();
|
|
234
253
|
return 0;
|
|
235
254
|
}
|
|
236
|
-
await
|
|
255
|
+
await waitForServeShutdown(server);
|
|
237
256
|
return 0;
|
|
238
257
|
}
|
|
239
|
-
let pythonFallbackBackend;
|
|
240
|
-
const runExplicitPythonFallback = (fallbackArgv) => {
|
|
241
|
-
pythonFallbackBackend ??= createBackend({
|
|
242
|
-
env: { ...env, AUTOVPN_BACKEND: 'python' },
|
|
243
|
-
cwd,
|
|
244
|
-
runForwarder
|
|
245
|
-
});
|
|
246
|
-
return pythonFallbackBackend.executeCli(fallbackArgv);
|
|
247
|
-
};
|
|
248
258
|
if (isPipelineProxyCommand(normalizedArgv)) {
|
|
249
|
-
|
|
259
|
+
throw new Error('--proxy is now handled by serve and Node runtime proxy settings; Python proxy mode is no longer supported');
|
|
250
260
|
}
|
|
251
261
|
const nativeResult = await runNativeCommand(normalizedArgv, {
|
|
252
262
|
cwd,
|
|
253
263
|
env,
|
|
254
264
|
io,
|
|
255
265
|
readStdin: options.readStdin ?? defaultReadStdin,
|
|
256
|
-
pythonFallback: runExplicitPythonFallback,
|
|
257
266
|
spawn: options.spawn,
|
|
258
267
|
now: options.now,
|
|
259
268
|
jobId: options.jobId,
|
|
@@ -8,12 +8,18 @@ import { publicStartedPayload, startDetachedResume, startDetachedRetry, startDet
|
|
|
8
8
|
import { followLog } from '../jobs/logs.js';
|
|
9
9
|
import { latestJobId, listJobs, loadJob, publicJobPayload, singleActiveJobId, tailLog } from '../jobs/read.js';
|
|
10
10
|
import { readOptionValue, resolveProjectRoot } from '../runtime/paths.js';
|
|
11
|
-
function wantsPython(env, key) {
|
|
12
|
-
return String(env[key] ?? '').trim().toLowerCase() === 'python';
|
|
13
|
-
}
|
|
14
11
|
function jsonLine(payload) {
|
|
15
12
|
return `${JSON.stringify(payload)}\n`;
|
|
16
13
|
}
|
|
14
|
+
function renderDoctorHuman(payload) {
|
|
15
|
+
const checks = Array.isArray(payload.checks) ? payload.checks : [];
|
|
16
|
+
const lines = [`doctor: ${payload.ok ? 'ok' : 'failed'}`];
|
|
17
|
+
for (const item of checks) {
|
|
18
|
+
const check = item;
|
|
19
|
+
lines.push(`[${String(check.status ?? 'unknown')}] ${String(check.name ?? 'check')}: ${String(check.message ?? '')}`);
|
|
20
|
+
}
|
|
21
|
+
return `${lines.join('\n')}\n`;
|
|
22
|
+
}
|
|
17
23
|
function positionalAfter(argv, start) {
|
|
18
24
|
for (let index = start; index < argv.length; index += 1) {
|
|
19
25
|
const value = argv[index];
|
|
@@ -66,17 +72,13 @@ export async function runNativeCommand(argv, context) {
|
|
|
66
72
|
const projectRoot = resolveProjectRoot(argv, context.cwd);
|
|
67
73
|
const command = argv[0];
|
|
68
74
|
if (command === 'doctor') {
|
|
69
|
-
if (wantsPython(context.env, 'AUTOVPN_DOCTOR_BACKEND'))
|
|
70
|
-
return context.pythonFallback(argv);
|
|
71
|
-
if (readOptionValue(argv, '--output') !== 'json')
|
|
72
|
-
return undefined;
|
|
73
75
|
const result = await runDoctor(projectRoot, argv, context.env);
|
|
74
|
-
context.io.writeStdout(
|
|
76
|
+
context.io.writeStdout(readOptionValue(argv, '--output') === 'json'
|
|
77
|
+
? jsonLine(result.payload)
|
|
78
|
+
: renderDoctorHuman(result.payload));
|
|
75
79
|
return result.code;
|
|
76
80
|
}
|
|
77
81
|
if (command === 'profile') {
|
|
78
|
-
if (wantsPython(context.env, 'AUTOVPN_PROFILE_BACKEND'))
|
|
79
|
-
return context.pythonFallback(argv);
|
|
80
82
|
if (argv[1] === 'show') {
|
|
81
83
|
context.io.writeStdout(jsonLine(profilePayload(projectRoot, context.env)));
|
|
82
84
|
return 0;
|
|
@@ -92,8 +94,6 @@ export async function runNativeCommand(argv, context) {
|
|
|
92
94
|
}
|
|
93
95
|
}
|
|
94
96
|
if (command === 'artifacts') {
|
|
95
|
-
if (wantsPython(context.env, 'AUTOVPN_ARTIFACTS_BACKEND'))
|
|
96
|
-
return context.pythonFallback(argv);
|
|
97
97
|
const subcommand = argv[1];
|
|
98
98
|
if (subcommand === 'latest') {
|
|
99
99
|
context.io.writeStdout(jsonLine(artifactLatest(projectRoot, context.env)));
|
package/dist/jobs/commands.js
CHANGED
|
@@ -5,23 +5,12 @@ import { fileURLToPath } from 'node:url';
|
|
|
5
5
|
import { publicJobPayload } from './read.js';
|
|
6
6
|
import { createJobStore } from './store.js';
|
|
7
7
|
import { processMatchesJob as defaultProcessMatchesJob, terminateProcessGroup } from './process.js';
|
|
8
|
-
async function defaultResolvePythonCli(env) {
|
|
9
|
-
// @ts-expect-error Phase 1 runner remains plain ESM JavaScript.
|
|
10
|
-
const runner = await import('../../lib/runner.mjs');
|
|
11
|
-
return runner.resolveOrInstallPythonCli({ env });
|
|
12
|
-
}
|
|
13
8
|
function defaultResolveNodeCli() {
|
|
14
9
|
const packageRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..');
|
|
15
10
|
return { command: process.execPath, args: [path.join(packageRoot, 'bin', 'autovpn.mjs')] };
|
|
16
11
|
}
|
|
17
|
-
function
|
|
18
|
-
return
|
|
19
|
-
}
|
|
20
|
-
async function resolveDetachedWorker(env, options) {
|
|
21
|
-
if (wantsNodeWorker(env)) {
|
|
22
|
-
return defaultResolveNodeCli();
|
|
23
|
-
}
|
|
24
|
-
return options.resolvePythonCli ? await options.resolvePythonCli() : await defaultResolvePythonCli(env);
|
|
12
|
+
async function resolveDetachedWorker(_env, _options) {
|
|
13
|
+
return defaultResolveNodeCli();
|
|
25
14
|
}
|
|
26
15
|
function pushFlag(argv, enabled, flag) {
|
|
27
16
|
if (enabled)
|
|
@@ -117,7 +106,7 @@ export async function startDetachedResume(command, options = {}) {
|
|
|
117
106
|
options: { source_job_id: command.sourceJobId, session_dir: command.sessionDir, output_format: outputFormat },
|
|
118
107
|
resumeFrom: command.sessionDir
|
|
119
108
|
});
|
|
120
|
-
const
|
|
109
|
+
const resumeArgs = [
|
|
121
110
|
'resume',
|
|
122
111
|
'pipeline',
|
|
123
112
|
'--project-root',
|
|
@@ -131,8 +120,8 @@ export async function startDetachedResume(command, options = {}) {
|
|
|
131
120
|
'--human-log',
|
|
132
121
|
String(job.human_log)
|
|
133
122
|
];
|
|
134
|
-
job.command = [resolved.command, ...resolved.args, ...
|
|
135
|
-
const child = spawnDetached(resolved.command, [...resolved.args, ...
|
|
123
|
+
job.command = [resolved.command, ...resolved.args, ...resumeArgs];
|
|
124
|
+
const child = spawnDetached(resolved.command, [...resolved.args, ...resumeArgs], job, options);
|
|
136
125
|
job.pid = Number(child.pid ?? 0);
|
|
137
126
|
job.pgid = Number(child.pid ?? 0);
|
|
138
127
|
return jobStore.writeJob(job);
|
|
@@ -149,7 +138,7 @@ export async function startDetachedRetry(command, options = {}) {
|
|
|
149
138
|
options: { artifact_dir: command.artifactDir, stage: command.stage, output_format: outputFormat },
|
|
150
139
|
retry
|
|
151
140
|
});
|
|
152
|
-
const
|
|
141
|
+
const retryArgs = [
|
|
153
142
|
'retry-stage',
|
|
154
143
|
'--project-root',
|
|
155
144
|
command.projectRoot,
|
|
@@ -164,8 +153,8 @@ export async function startDetachedRetry(command, options = {}) {
|
|
|
164
153
|
'--human-log',
|
|
165
154
|
String(job.human_log)
|
|
166
155
|
];
|
|
167
|
-
job.command = [resolved.command, ...resolved.args, ...
|
|
168
|
-
const child = spawnDetached(resolved.command, [...resolved.args, ...
|
|
156
|
+
job.command = [resolved.command, ...resolved.args, ...retryArgs];
|
|
157
|
+
const child = spawnDetached(resolved.command, [...resolved.args, ...retryArgs], job, options);
|
|
169
158
|
job.pid = Number(child.pid ?? 0);
|
|
170
159
|
job.pgid = Number(child.pid ?? 0);
|
|
171
160
|
return jobStore.writeJob(job);
|
|
@@ -199,6 +199,24 @@ function emitEvent(callback, eventType, payload) {
|
|
|
199
199
|
callback(eventType, payload);
|
|
200
200
|
}
|
|
201
201
|
}
|
|
202
|
+
async function mapWithConcurrency(items, concurrency, worker, onComplete) {
|
|
203
|
+
const limit = Math.max(1, Math.trunc(Number(concurrency) || 1));
|
|
204
|
+
const results = new Array(items.length);
|
|
205
|
+
let nextIndex = 0;
|
|
206
|
+
let completed = 0;
|
|
207
|
+
async function runWorker() {
|
|
208
|
+
while (nextIndex < items.length) {
|
|
209
|
+
const index = nextIndex;
|
|
210
|
+
nextIndex += 1;
|
|
211
|
+
const result = await worker(items[index], index);
|
|
212
|
+
results[index] = result;
|
|
213
|
+
completed += 1;
|
|
214
|
+
onComplete?.(result, index, completed);
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, () => runWorker()));
|
|
218
|
+
return results;
|
|
219
|
+
}
|
|
202
220
|
function numberOrDefault(value, fallback) {
|
|
203
221
|
const parsed = Number(value);
|
|
204
222
|
return Number.isFinite(parsed) ? parsed : fallback;
|
|
@@ -584,9 +602,7 @@ async function checkBatchInNode(input, options) {
|
|
|
584
602
|
runtimePath: input.runtime_path ?? ''
|
|
585
603
|
})
|
|
586
604
|
: checkLinkAvailabilityDirect(speedResult, config, { targets, fetch: options.fetch })));
|
|
587
|
-
|
|
588
|
-
for (let index = 0; index < input.results.length; index += 1) {
|
|
589
|
-
const speedResult = input.results[index];
|
|
605
|
+
return mapWithConcurrency(input.results, numberOrDefault(input.config.concurrency, 1), async (speedResult) => {
|
|
590
606
|
let availability;
|
|
591
607
|
try {
|
|
592
608
|
availability = availabilityResultToDict(await checkLinkAvailability(speedResult, input.config, {
|
|
@@ -597,8 +613,8 @@ async function checkBatchInNode(input, options) {
|
|
|
597
613
|
catch (error) {
|
|
598
614
|
availability = buildRuntimeErrorResult(speedResult, error instanceof Error ? error.message : String(error), targets);
|
|
599
615
|
}
|
|
600
|
-
|
|
601
|
-
|
|
616
|
+
return availability;
|
|
617
|
+
}, (availability, _index, completed) => {
|
|
602
618
|
if (options.progressCallback) {
|
|
603
619
|
const statuses = Object.entries(availability.provider_results)
|
|
604
620
|
.map(([name, provider]) => `${name}=${provider.passed ? 'ok' : provider.reason}`)
|
|
@@ -612,15 +628,12 @@ async function checkBatchInNode(input, options) {
|
|
|
612
628
|
all_passed: availability.all_passed,
|
|
613
629
|
provider_results: availability.provider_results
|
|
614
630
|
});
|
|
615
|
-
}
|
|
616
|
-
return output;
|
|
631
|
+
});
|
|
617
632
|
}
|
|
618
633
|
export function selectPipelineStageBackend(stage, env = process.env) {
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
const selected = stageOverride || pipelineOverride || 'node';
|
|
623
|
-
return selected === 'python' ? 'python' : 'node';
|
|
634
|
+
void stage;
|
|
635
|
+
void env;
|
|
636
|
+
return 'node';
|
|
624
637
|
}
|
|
625
638
|
async function defaultResolvePythonCli(env) {
|
|
626
639
|
// @ts-expect-error Phase 1 runner remains plain ESM JavaScript.
|
package/dist/pipeline/dedupe.js
CHANGED
|
@@ -41,11 +41,9 @@ export function dedupeVmessLinks(links) {
|
|
|
41
41
|
return result;
|
|
42
42
|
}
|
|
43
43
|
export function selectPipelineStageBackend(stage, env = process.env) {
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
const selected = stageOverride || pipelineOverride || 'node';
|
|
48
|
-
return selected === 'python' ? 'python' : 'node';
|
|
44
|
+
void stage;
|
|
45
|
+
void env;
|
|
46
|
+
return 'node';
|
|
49
47
|
}
|
|
50
48
|
async function defaultResolvePythonCli(env) {
|
|
51
49
|
// @ts-expect-error Phase 1 runner remains plain ESM JavaScript.
|
package/dist/pipeline/deploy.js
CHANGED
|
@@ -720,11 +720,9 @@ export async function cleanupBlockedPagesProjects(deploy, deployment, client) {
|
|
|
720
720
|
return { cleanup_deleted: deletedAny, cleanup_errors: errors };
|
|
721
721
|
}
|
|
722
722
|
export function selectPipelineStageBackend(stage, env = process.env) {
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
const selected = stageOverride || pipelineOverride || 'node';
|
|
727
|
-
return selected === 'python' ? 'python' : 'node';
|
|
723
|
+
void stage;
|
|
724
|
+
void env;
|
|
725
|
+
return 'node';
|
|
728
726
|
}
|
|
729
727
|
async function defaultResolvePythonCli(env) {
|
|
730
728
|
// @ts-expect-error Phase 1 runner remains plain ESM JavaScript.
|
package/dist/pipeline/extract.js
CHANGED
|
@@ -65,6 +65,27 @@ function generateVmessLink(payload) {
|
|
|
65
65
|
const encoded = Buffer.from(json, 'utf8').toString('base64').replaceAll('+', '-').replaceAll('/', '_');
|
|
66
66
|
return `vmess://${encoded}`;
|
|
67
67
|
}
|
|
68
|
+
function linkFingerprint(link) {
|
|
69
|
+
try {
|
|
70
|
+
const encoded = String(link).replace(/^vmess:\/\//, '');
|
|
71
|
+
const padded = encoded + '='.repeat((4 - (encoded.length % 4)) % 4);
|
|
72
|
+
const payload = JSON.parse(Buffer.from(padded, 'base64url').toString('utf8'));
|
|
73
|
+
const canonical = JSON.stringify([
|
|
74
|
+
payload.add ?? '',
|
|
75
|
+
payload.port ?? '',
|
|
76
|
+
payload.id ?? '',
|
|
77
|
+
payload.net ?? '',
|
|
78
|
+
payload.host ?? '',
|
|
79
|
+
payload.path ?? '',
|
|
80
|
+
payload.tls ?? '',
|
|
81
|
+
payload.sni ?? ''
|
|
82
|
+
].map((value) => String(value)));
|
|
83
|
+
return crypto.createHash('sha256').update(canonical).digest('hex');
|
|
84
|
+
}
|
|
85
|
+
catch {
|
|
86
|
+
return crypto.createHash('sha256').update(String(link)).digest('hex');
|
|
87
|
+
}
|
|
88
|
+
}
|
|
68
89
|
function payloadFromOutboundConfig(psName, jsonText) {
|
|
69
90
|
const config = JSON.parse(jsonText);
|
|
70
91
|
const outbound = (config.outbounds ?? []).find((item) => item.protocol === 'vmess');
|
|
@@ -108,11 +129,9 @@ export function extractLinksFromPlaintext(sourceName, plaintext) {
|
|
|
108
129
|
return [generateVmessLink(payloadFromOutboundConfig(psName, jsonText))];
|
|
109
130
|
}
|
|
110
131
|
export function selectPipelineStageBackend(stage, env = process.env) {
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
const selected = stageOverride || pipelineOverride || 'node';
|
|
115
|
-
return selected === 'python' ? 'python' : 'node';
|
|
132
|
+
void stage;
|
|
133
|
+
void env;
|
|
134
|
+
return 'node';
|
|
116
135
|
}
|
|
117
136
|
async function defaultResolvePythonCli(env) {
|
|
118
137
|
// @ts-expect-error Phase 1 runner remains plain ESM JavaScript.
|
|
@@ -257,7 +276,7 @@ async function fetchSourceLinksInNode(input, options) {
|
|
|
257
276
|
const maxIterations = Math.max(0, Math.trunc(numberOrDefault(source.max_iterations, 0)));
|
|
258
277
|
const configuredMinIterations = Math.max(0, Math.trunc(numberOrDefault(source.min_iterations, 0)));
|
|
259
278
|
const minIterations = configuredMinIterations > maxIterations ? 0 : configuredMinIterations;
|
|
260
|
-
const plateauLimit = Math.max(1, Math.trunc(numberOrDefault(source.plateau_limit,
|
|
279
|
+
const plateauLimit = Math.max(1, Math.trunc(numberOrDefault(source.plateau_limit, 20)));
|
|
261
280
|
const failureLimit = Math.max(1, Math.trunc(numberOrDefault(source.failure_limit, 1)));
|
|
262
281
|
const maxRuntimeSeconds = Math.max(0, numberOrDefault(source.max_runtime_seconds, 0));
|
|
263
282
|
const startIteration = Math.max(1, Math.trunc(numberOrDefault(source.resume_from_iteration, 1)));
|
|
@@ -317,13 +336,17 @@ async function fetchSourceLinksInNode(input, options) {
|
|
|
317
336
|
successes += 1;
|
|
318
337
|
failures = 0;
|
|
319
338
|
let newItems = 0;
|
|
339
|
+
const newLinks = [];
|
|
340
|
+
const newItemFingerprints = [];
|
|
320
341
|
for (const link of extracted) {
|
|
321
342
|
if (seen.has(link)) {
|
|
322
343
|
continue;
|
|
323
344
|
}
|
|
324
345
|
seen.add(link);
|
|
325
346
|
links.push(link);
|
|
347
|
+
newLinks.push(link);
|
|
326
348
|
newItems += 1;
|
|
349
|
+
newItemFingerprints.push(linkFingerprint(link));
|
|
327
350
|
}
|
|
328
351
|
emitExtractEvent(options, 'extract_iteration', {
|
|
329
352
|
source_name: input.source_name,
|
|
@@ -331,8 +354,13 @@ async function fetchSourceLinksInNode(input, options) {
|
|
|
331
354
|
requested_iterations: maxIterations,
|
|
332
355
|
new_items: newItems,
|
|
333
356
|
extracted_links: extracted.length,
|
|
334
|
-
total_links: links.length
|
|
357
|
+
total_links: links.length,
|
|
358
|
+
deduped_links: links.length,
|
|
359
|
+
new_item_fingerprints: newItemFingerprints
|
|
335
360
|
});
|
|
361
|
+
if (newItems > 0) {
|
|
362
|
+
await options.linksCallback?.(newLinks);
|
|
363
|
+
}
|
|
336
364
|
plateau = newItems === 0 ? plateau + 1 : 0;
|
|
337
365
|
if (plateau >= plateauLimit && attempt >= minIterations) {
|
|
338
366
|
break;
|
|
@@ -140,11 +140,9 @@ export function buildWorkerArtifacts(renderedSource, configInput = {}, secretQue
|
|
|
140
140
|
};
|
|
141
141
|
}
|
|
142
142
|
export function selectPipelineStageBackend(stage, env = process.env) {
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
const selected = stageOverride || pipelineOverride || 'node';
|
|
147
|
-
return selected === 'python' ? 'python' : 'node';
|
|
143
|
+
void stage;
|
|
144
|
+
void env;
|
|
145
|
+
return 'node';
|
|
148
146
|
}
|
|
149
147
|
async function defaultResolvePythonCli(env) {
|
|
150
148
|
// @ts-expect-error Phase 1 runner remains plain ESM JavaScript.
|