@qvac/harness 0.3.1 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +79 -1
- package/dist/harness.bundle +359 -175
- package/dist/prebuilds-manifest.json +8 -0
- package/dist-lib/lib/capabilities/completion.d.ts +2 -1
- package/dist-lib/lib/capabilities/completion.js +5 -6
- package/dist-lib/lib/tools/exec/adapters/adapter.d.ts +3 -1
- package/dist-lib/lib/tools/exec/adapters/python/guarded-fetch.d.ts +5 -0
- package/dist-lib/lib/tools/exec/adapters/python/guarded-fetch.js +28 -2
- package/dist-lib/lib/tools/exec/adapters/python/interrupt-timer.d.ts +5 -0
- package/dist-lib/lib/tools/exec/adapters/python/interrupt-timer.js +29 -0
- package/dist-lib/lib/tools/exec/adapters/python/named-hosts.d.ts +2 -0
- package/dist-lib/lib/tools/exec/adapters/python/named-hosts.js +15 -0
- package/dist-lib/lib/tools/exec/adapters/python/pyodide.d.ts +3 -1
- package/dist-lib/lib/tools/exec/adapters/python/pyodide.js +41 -7
- package/dist-lib/lib/tools/exec/adapters/python/python-adapter.js +9 -3
- package/dist-lib/lib/tools/exec/adapters/python/run-session.d.ts +2 -0
- package/dist-lib/lib/tools/exec/adapters/python/run-session.js +19 -11
- package/dist-lib/lib/tools/exec/adapters/python/worker/client.d.ts +2 -1
- package/dist-lib/lib/tools/exec/adapters/python/worker/client.js +16 -3
- package/dist-lib/lib/tools/exec/adapters/python/worker/entry.js +15 -2
- package/dist-lib/lib/tools/exec/adapters/python/worker/protocol.d.ts +6 -0
- package/dist-lib/lib/tools/exec/adapters/python/worker/protocol.js +6 -2
- package/dist-lib/lib/tools/exec/adapters/python/worker/spawn-bare.d.ts +1 -0
- package/dist-lib/lib/tools/exec/adapters/python/worker/spawn-bare.js +8 -0
- package/dist-lib/lib/tools/exec/adapters/python/worker/spawn-node.d.ts +1 -0
- package/dist-lib/lib/tools/exec/adapters/python/worker/spawn-node.js +8 -0
- package/dist-lib/lib/tools/exec.js +36 -3
- package/package.json +16 -7
- package/dist-lib/lib/capabilities/vision-history.d.ts +0 -3
- package/dist-lib/lib/capabilities/vision-history.js +0 -21
|
@@ -272,6 +272,14 @@
|
|
|
272
272
|
"win32-arm64",
|
|
273
273
|
"win32-x64"
|
|
274
274
|
],
|
|
275
|
+
"bare-tty/node_modules/bare-signals": [
|
|
276
|
+
"darwin-arm64",
|
|
277
|
+
"darwin-x64",
|
|
278
|
+
"linux-arm64",
|
|
279
|
+
"linux-x64",
|
|
280
|
+
"win32-arm64",
|
|
281
|
+
"win32-x64"
|
|
282
|
+
],
|
|
275
283
|
"bare-tty": [
|
|
276
284
|
"darwin-arm64",
|
|
277
285
|
"darwin-x64",
|
|
@@ -142,6 +142,7 @@ export interface ToolExecuteContext {
|
|
|
142
142
|
uncappedResponse?: boolean | undefined;
|
|
143
143
|
sandboxMode?: boolean | undefined;
|
|
144
144
|
toolNames?: readonly string[] | undefined;
|
|
145
|
+
approval?: ToolDecision | undefined;
|
|
145
146
|
}
|
|
146
147
|
export interface RunBackendUse {
|
|
147
148
|
tool: string;
|
|
@@ -181,7 +182,7 @@ export interface ToolDefinition {
|
|
|
181
182
|
toolset: Toolset | null;
|
|
182
183
|
validate?: (args: Record<string, unknown>, ctx?: ToolExecuteContext) => string | null;
|
|
183
184
|
effect: (args: Record<string, unknown>, ctx?: ToolExecuteContext) => ToolEffect;
|
|
184
|
-
resource?: (args: Record<string, unknown
|
|
185
|
+
resource?: (args: Record<string, unknown>, ctx?: ToolExecuteContext) => ResourceDescriptor | null;
|
|
185
186
|
compiled?: true;
|
|
186
187
|
execute: (args: Record<string, unknown>, ctx?: ToolExecuteContext) => Promise<unknown>;
|
|
187
188
|
}
|
|
@@ -29,7 +29,6 @@ import { NETWORK_TOOLSETS, permittedTools } from '../tools/toolsets.js';
|
|
|
29
29
|
import { promptPrefixHash } from './prompt-prefix.js';
|
|
30
30
|
import { logToolPrompt } from './tool-prompt-log.js';
|
|
31
31
|
import { DIRECT_CALL_ID, loadedSkillFilesFrom, loadedSkillsFrom, TOOL_CONTINUATION_NUDGE, toolResultTurn, withoutSkillLoadTurns } from './tool-turns.js';
|
|
32
|
-
import { projectVisionHistory, visionHistoryCacheKey } from './vision-history.js';
|
|
33
32
|
export function imageSettingsFrom(agent) {
|
|
34
33
|
if (!agent)
|
|
35
34
|
return undefined;
|
|
@@ -273,7 +272,7 @@ async function executeCall(tool, args, ctx, { approve, signal, denied, sources =
|
|
|
273
272
|
type: 'tool-activity',
|
|
274
273
|
name: tool.name,
|
|
275
274
|
args,
|
|
276
|
-
resource: tool.resource?.(args) ?? null,
|
|
275
|
+
resource: tool.resource?.(args, ctx) ?? null,
|
|
277
276
|
effect: tool.effect(args, ctx),
|
|
278
277
|
decision: 'blocked'
|
|
279
278
|
});
|
|
@@ -285,7 +284,7 @@ async function executeCall(tool, args, ctx, { approve, signal, denied, sources =
|
|
|
285
284
|
return deniedCall();
|
|
286
285
|
}
|
|
287
286
|
const effect = classified ?? tool.effect(args, ctx);
|
|
288
|
-
const resource = tool.resource?.(args) ?? null;
|
|
287
|
+
const resource = tool.resource?.(args, ctx) ?? null;
|
|
289
288
|
const autoRun = effect === 'read' || tool.autoApprove === true || skipApprovals === true;
|
|
290
289
|
hlog('tool decision', {
|
|
291
290
|
tool: tool.name,
|
|
@@ -327,7 +326,7 @@ async function executeCall(tool, args, ctx, { approve, signal, denied, sources =
|
|
|
327
326
|
let output;
|
|
328
327
|
let outcome = { kind: 'success' };
|
|
329
328
|
try {
|
|
330
|
-
output = await tool.execute(args, { ...ctx, signal });
|
|
329
|
+
output = await tool.execute(args, { ...ctx, signal, approval: decision });
|
|
331
330
|
hlog('tool done', { tool: tool.name });
|
|
332
331
|
}
|
|
333
332
|
catch (err) {
|
|
@@ -545,7 +544,7 @@ async function* runRecoveredRoundWith(sdk, args, modelId) {
|
|
|
545
544
|
const cacheable = candidate.history === args.history;
|
|
546
545
|
const params = {
|
|
547
546
|
modelId,
|
|
548
|
-
history:
|
|
547
|
+
history: candidate.history,
|
|
549
548
|
stream: true,
|
|
550
549
|
tools: args.tools,
|
|
551
550
|
generationParams: args.generationParams,
|
|
@@ -798,7 +797,7 @@ export async function* completion(sdk, { modelId, acquireRound, model, agent, ct
|
|
|
798
797
|
}
|
|
799
798
|
logSelection();
|
|
800
799
|
let history = annotateLastUserTime(systemContent ? [{ role: 'system', content: systemContent }, ...turns] : turns);
|
|
801
|
-
let roundCacheKey = stubDropped ? undefined :
|
|
800
|
+
let roundCacheKey = stubDropped ? undefined : cacheKey;
|
|
802
801
|
const runUsage = () => usageBreakdown(promptParts, stats.cacheTokens, model);
|
|
803
802
|
const denied = new Set();
|
|
804
803
|
const failed = new Map();
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { AbortSignal as BareAbortSignal } from 'bare-abort-controller';
|
|
2
|
-
import type { AttachFn, FetchAttachmentFn, RunBudget } from '../../../capabilities/completion.ts';
|
|
2
|
+
import type { AttachFn, FetchAttachmentFn, RunBudget, ToolDecision } from '../../../capabilities/completion.ts';
|
|
3
3
|
import type { ExecResult } from '../runner.ts';
|
|
4
4
|
export type ExecutionOptions = {
|
|
5
5
|
readonly cwd?: string;
|
|
@@ -17,6 +17,8 @@ export type ExecutionContext = {
|
|
|
17
17
|
readonly runBudget?: RunBudget;
|
|
18
18
|
readonly sandboxMode?: boolean;
|
|
19
19
|
readonly toolNames?: readonly string[];
|
|
20
|
+
readonly approval?: ToolDecision;
|
|
21
|
+
readonly confirmConnect?: (host: string) => Promise<boolean>;
|
|
20
22
|
};
|
|
21
23
|
export type ExecutionPlan = {
|
|
22
24
|
run(options: ExecutionOptions): Promise<ExecResult> | ExecResult;
|
|
@@ -3,6 +3,11 @@ import type { NoRedirectFetchInit, NoRedirectFetchResponse } from '../../../../n
|
|
|
3
3
|
export declare const MAX_RESPONSE_BYTES: number;
|
|
4
4
|
type NoRedirectFetch = (url: URL, init?: NoRedirectFetchInit) => Promise<NoRedirectFetchResponse>;
|
|
5
5
|
declare const GUARDED: unique symbol;
|
|
6
|
+
export type ConnectPolicy = {
|
|
7
|
+
readonly named: readonly string[];
|
|
8
|
+
readonly ask: (host: string) => Promise<boolean>;
|
|
9
|
+
};
|
|
10
|
+
export declare function setConnectPolicy(next: ConnectPolicy | null): void;
|
|
6
11
|
export declare function isGuardedFetch(fetch: unknown): boolean;
|
|
7
12
|
type SignalLike = {
|
|
8
13
|
readonly aborted: boolean;
|
|
@@ -2,12 +2,27 @@ import { AbortController } from 'bare-abort-controller';
|
|
|
2
2
|
import b4a from 'b4a';
|
|
3
3
|
import defaultFetch from '#no-redirect-fetch';
|
|
4
4
|
import { lookupHost } from '../../../../dns-lookup.js';
|
|
5
|
-
import {
|
|
5
|
+
import { hostIsNamed } from './named-hosts.js';
|
|
6
|
+
import { checkOutboundHost, resolveOutboundUrl } from '../../../url-guard.js';
|
|
6
7
|
export const MAX_RESPONSE_BYTES = 100 * 1024 * 1024;
|
|
7
8
|
const MAX_REDIRECTS = 10;
|
|
8
9
|
const TIMEOUT_MS = 120_000;
|
|
9
10
|
const CHUNK_BYTES = 1024 * 1024;
|
|
10
11
|
const GUARDED = Symbol.for('qvac.harness.guardedFetch');
|
|
12
|
+
let policy = null;
|
|
13
|
+
const granted = new Set();
|
|
14
|
+
export function setConnectPolicy(next) {
|
|
15
|
+
policy = next;
|
|
16
|
+
granted.clear();
|
|
17
|
+
}
|
|
18
|
+
async function mayConnect(host) {
|
|
19
|
+
if (!policy || granted.has(host) || hostIsNamed(host, policy.named))
|
|
20
|
+
return true;
|
|
21
|
+
const allowed = await policy.ask(host);
|
|
22
|
+
if (allowed)
|
|
23
|
+
granted.add(host);
|
|
24
|
+
return allowed;
|
|
25
|
+
}
|
|
11
26
|
export function isGuardedFetch(fetch) {
|
|
12
27
|
return typeof fetch === 'function' && fetch[GUARDED] === true;
|
|
13
28
|
}
|
|
@@ -154,6 +169,12 @@ export function createGuardedFetch(options = {}) {
|
|
|
154
169
|
if (url.protocol !== 'http:' && url.protocol !== 'https:') {
|
|
155
170
|
throw new TypeError(`fetch refused: unsupported protocol ${url.protocol}`);
|
|
156
171
|
}
|
|
172
|
+
const literal = checkOutboundHost(url);
|
|
173
|
+
if (literal)
|
|
174
|
+
throw new TypeError(`fetch refused: ${literal}`);
|
|
175
|
+
if (!(await mayConnect(url.hostname))) {
|
|
176
|
+
throw new TypeError(`fetch refused: ${url.hostname} was not approved for this run`);
|
|
177
|
+
}
|
|
157
178
|
const check = await resolveOutboundUrl(url, resolveHost, controller.signal);
|
|
158
179
|
if (check.blocked)
|
|
159
180
|
throw new TypeError(`fetch refused: ${check.blocked}`);
|
|
@@ -192,5 +213,10 @@ export function createGuardedFetch(options = {}) {
|
|
|
192
213
|
clearTimeout(timer);
|
|
193
214
|
}
|
|
194
215
|
};
|
|
195
|
-
|
|
216
|
+
const fetch = (input, init) => {
|
|
217
|
+
const response = guardedFetch(input, init);
|
|
218
|
+
response.catch(() => { });
|
|
219
|
+
return response;
|
|
220
|
+
};
|
|
221
|
+
return Object.assign(fetch, { [GUARDED]: true });
|
|
196
222
|
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { spawnInlineThread } from '#pyodide-worker-spawn';
|
|
2
|
+
const ARMED = 0;
|
|
3
|
+
const DISARMED = 1;
|
|
4
|
+
const FIRED = 2;
|
|
5
|
+
const SIGINT = 2;
|
|
6
|
+
const RESEND_MS = 100;
|
|
7
|
+
const SOURCE = `
|
|
8
|
+
const state = new Int32Array(data.state)
|
|
9
|
+
const interrupt = new Uint8Array(data.interrupt)
|
|
10
|
+
Atomics.wait(state, 0, ${ARMED}, data.ms)
|
|
11
|
+
if (Atomics.compareExchange(state, 0, ${ARMED}, ${FIRED}) === ${ARMED}) {
|
|
12
|
+
do Atomics.store(interrupt, 0, ${SIGINT})
|
|
13
|
+
while (Atomics.wait(state, 0, ${FIRED}, ${RESEND_MS}) === 'timed-out')
|
|
14
|
+
}
|
|
15
|
+
`;
|
|
16
|
+
export function startInterruptTimer(ms) {
|
|
17
|
+
const state = new Int32Array(new SharedArrayBuffer(4));
|
|
18
|
+
const interrupt = new Uint8Array(new SharedArrayBuffer(1));
|
|
19
|
+
const reap = spawnInlineThread(SOURCE, { state: state.buffer, interrupt: interrupt.buffer, ms });
|
|
20
|
+
return {
|
|
21
|
+
interrupt,
|
|
22
|
+
disarm() {
|
|
23
|
+
const fired = Atomics.exchange(state, 0, DISARMED) === FIRED;
|
|
24
|
+
Atomics.notify(state, 0);
|
|
25
|
+
reap();
|
|
26
|
+
return fired;
|
|
27
|
+
}
|
|
28
|
+
};
|
|
29
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
const URL_LITERAL = /https?:\/\/[^\s'"`\\<>(){}[\]]+/gi;
|
|
2
|
+
export function namedHosts(source) {
|
|
3
|
+
const hosts = new Set();
|
|
4
|
+
for (const [match] of source.matchAll(URL_LITERAL)) {
|
|
5
|
+
try {
|
|
6
|
+
hosts.add(new URL(match).hostname.toLowerCase());
|
|
7
|
+
}
|
|
8
|
+
catch { }
|
|
9
|
+
}
|
|
10
|
+
return [...hosts];
|
|
11
|
+
}
|
|
12
|
+
export function hostIsNamed(host, named) {
|
|
13
|
+
const bare = (value) => value.toLowerCase().replace(/^www\./, '');
|
|
14
|
+
return named.some((name) => bare(name) === bare(host));
|
|
15
|
+
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type
|
|
1
|
+
import { type AbortSignal as BareAbortSignal } from 'bare-abort-controller';
|
|
2
2
|
import { pyodideLoadOptions } from './loader.ts';
|
|
3
3
|
import { type LoadPyodideFn, type PyodideLike, type PythonRunResult } from './run-session.ts';
|
|
4
4
|
export { pyodideLoadOptions };
|
|
@@ -16,5 +16,7 @@ export type RunPythonOptions = {
|
|
|
16
16
|
readonly workdir?: string;
|
|
17
17
|
readonly loadPyodide?: LoadPyodideFn;
|
|
18
18
|
readonly network?: boolean;
|
|
19
|
+
readonly hosts?: readonly string[];
|
|
20
|
+
readonly askConnect?: (host: string) => Promise<boolean>;
|
|
19
21
|
};
|
|
20
22
|
export declare function runPython(code: string, options?: RunPythonOptions): Promise<PythonRunResult>;
|
|
@@ -1,8 +1,10 @@
|
|
|
1
|
+
import { AbortController } from 'bare-abort-controller';
|
|
1
2
|
import process from 'process';
|
|
2
3
|
import ScopeLock from 'scope-lock';
|
|
3
4
|
import { AbortedError, raceAbort } from '../../../../helpers/race-abort.js';
|
|
4
5
|
import { hlog } from '../../../../log.js';
|
|
5
6
|
import { MAX_OUTPUT_CHARS } from '../../runner.js';
|
|
7
|
+
import { startInterruptTimer } from './interrupt-timer.js';
|
|
6
8
|
import { defaultLoad, pyodideLoadOptions } from './loader.js';
|
|
7
9
|
import { isPyodideFatal, performRun } from './run-session.js';
|
|
8
10
|
import { runPythonInWorker } from './worker/client.js';
|
|
@@ -32,7 +34,11 @@ function getPyodide(load) {
|
|
|
32
34
|
}
|
|
33
35
|
const runLock = new ScopeLock();
|
|
34
36
|
async function runInProcess(load, spec, options) {
|
|
37
|
+
const deadline = Date.now() + options.timeoutMs;
|
|
38
|
+
const timeoutMessage = `Command timed out after ${options.timeoutMs}ms`;
|
|
35
39
|
let timedOut = false;
|
|
40
|
+
const stop = new AbortController();
|
|
41
|
+
const ask = options.askConnect;
|
|
36
42
|
const run = (async () => {
|
|
37
43
|
await runLock.lock();
|
|
38
44
|
try {
|
|
@@ -40,12 +46,26 @@ async function runInProcess(load, spec, options) {
|
|
|
40
46
|
if (isCancelled())
|
|
41
47
|
throw new AbortedError();
|
|
42
48
|
const py = await getPyodide(load);
|
|
43
|
-
const
|
|
44
|
-
|
|
49
|
+
const timer = startInterruptTimer(Math.max(0, deadline - Date.now()));
|
|
50
|
+
py.setInterruptBuffer(timer.interrupt);
|
|
51
|
+
let outcome;
|
|
52
|
+
try {
|
|
53
|
+
outcome = await performRun(py, spec, {
|
|
54
|
+
isCancelled,
|
|
55
|
+
askConnect: ask && ((host) => raceAbort(ask(host), stop.signal).catch(() => false))
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
finally {
|
|
59
|
+
if (timer.disarm())
|
|
60
|
+
timedOut = true;
|
|
61
|
+
}
|
|
62
|
+
if (outcome.evict) {
|
|
45
63
|
cache.delete(load);
|
|
46
64
|
hlog('python: interpreter evicted; next run boots fresh');
|
|
47
65
|
}
|
|
48
|
-
|
|
66
|
+
if (timedOut)
|
|
67
|
+
throw new Error(timeoutMessage);
|
|
68
|
+
return outcome.result;
|
|
49
69
|
}
|
|
50
70
|
catch (error) {
|
|
51
71
|
if (isPyodideFatal(error))
|
|
@@ -60,7 +80,7 @@ async function runInProcess(load, spec, options) {
|
|
|
60
80
|
const timeout = new Promise((_, reject) => {
|
|
61
81
|
timer = setTimeout(() => {
|
|
62
82
|
timedOut = true;
|
|
63
|
-
reject(new Error(
|
|
83
|
+
reject(new Error(timeoutMessage));
|
|
64
84
|
}, options.timeoutMs);
|
|
65
85
|
});
|
|
66
86
|
try {
|
|
@@ -69,6 +89,7 @@ async function runInProcess(load, spec, options) {
|
|
|
69
89
|
finally {
|
|
70
90
|
if (timer)
|
|
71
91
|
clearTimeout(timer);
|
|
92
|
+
stop.abort(undefined);
|
|
72
93
|
}
|
|
73
94
|
}
|
|
74
95
|
function workerAvailable() {
|
|
@@ -89,20 +110,33 @@ export async function runPython(code, options = {}) {
|
|
|
89
110
|
workdir: options.workdir,
|
|
90
111
|
maxOutputChars: options.maxOutputChars ?? MAX_OUTPUT_CHARS,
|
|
91
112
|
network: options.network === true,
|
|
113
|
+
hosts: options.hosts,
|
|
92
114
|
installTimeoutMs: timeoutMs
|
|
93
115
|
};
|
|
94
116
|
if (options.loadPyodide) {
|
|
95
|
-
return runInProcess(options.loadPyodide, spec, {
|
|
117
|
+
return runInProcess(options.loadPyodide, spec, {
|
|
118
|
+
timeoutMs,
|
|
119
|
+
signal: options.signal,
|
|
120
|
+
askConnect: options.askConnect
|
|
121
|
+
});
|
|
96
122
|
}
|
|
97
123
|
if (!workerAvailable()) {
|
|
98
124
|
hlog('python: worker entry unavailable in this bundle; running in-process');
|
|
99
|
-
return runInProcess(defaultLoad, spec, {
|
|
125
|
+
return runInProcess(defaultLoad, spec, {
|
|
126
|
+
timeoutMs,
|
|
127
|
+
signal: options.signal,
|
|
128
|
+
askConnect: options.askConnect
|
|
129
|
+
});
|
|
100
130
|
}
|
|
101
131
|
await runLock.lock();
|
|
102
132
|
try {
|
|
103
133
|
if (options.signal?.aborted)
|
|
104
134
|
throw new AbortedError();
|
|
105
|
-
return await runPythonInWorker(spec, {
|
|
135
|
+
return await runPythonInWorker(spec, {
|
|
136
|
+
timeoutMs,
|
|
137
|
+
signal: options.signal,
|
|
138
|
+
askConnect: options.askConnect
|
|
139
|
+
});
|
|
106
140
|
}
|
|
107
141
|
finally {
|
|
108
142
|
runLock.unlock();
|
|
@@ -4,6 +4,7 @@ import { checkFileName, checkOutputName, MAX_INPUTS, MAX_OUTPUTS } from './outpu
|
|
|
4
4
|
import { runPython } from './pyodide.js';
|
|
5
5
|
import { isTranscoder, TRANSCODE_ERROR, VIDEO_TOOLS_ERROR, videoTool } from '../../video-tools.js';
|
|
6
6
|
import { moduleDistribution } from './distributions.js';
|
|
7
|
+
import { namedHosts } from './named-hosts.js';
|
|
7
8
|
import { normalizeDistribution, planVendoredInstall, vendoredSpec } from './wheels.js';
|
|
8
9
|
export function isPythonLanguage(language) {
|
|
9
10
|
return typeof language === 'string' && language.trim().toLowerCase() === 'python';
|
|
@@ -1063,7 +1064,7 @@ async function collect(result, declared, attach, runBudget, source) {
|
|
|
1063
1064
|
}
|
|
1064
1065
|
}
|
|
1065
1066
|
}
|
|
1066
|
-
const { attachments, missingOutputs, rejectedOutputs } = await attachOutputs(attach, result.files
|
|
1067
|
+
const { attachments, missingOutputs, rejectedOutputs } = await attachOutputs(attach, (result.exitCode === 0 && result.files) || new Map(), declared, runBudget);
|
|
1067
1068
|
const rejected = [...(result.rejected ?? []), ...rejectedOutputs];
|
|
1068
1069
|
const stop = deckDeliveredStop(attachments, result.exitCode, source);
|
|
1069
1070
|
if (stop && result.exitCode === 0) {
|
|
@@ -1098,7 +1099,10 @@ export function createPythonAdapter(disabled, loadPyodide, network = false) {
|
|
|
1098
1099
|
const runBudget = ctx?.runBudget;
|
|
1099
1100
|
const sandbox = ctx?.sandboxMode === true;
|
|
1100
1101
|
const toolNames = ctx?.toolNames;
|
|
1101
|
-
const
|
|
1102
|
+
const approved = ctx?.approval === 'allowed' || ctx?.approval === 'skip-all';
|
|
1103
|
+
const reach = network && !sandbox && approved;
|
|
1104
|
+
const gated = reach && ctx?.approval === 'allowed';
|
|
1105
|
+
const confirmConnect = ctx?.confirmConnect;
|
|
1102
1106
|
return {
|
|
1103
1107
|
async run(options) {
|
|
1104
1108
|
if (!command)
|
|
@@ -1153,7 +1157,9 @@ export function createPythonAdapter(disabled, loadPyodide, network = false) {
|
|
|
1153
1157
|
outputs: prepared.outputs,
|
|
1154
1158
|
workdir: prepared.workdir,
|
|
1155
1159
|
loadPyodide,
|
|
1156
|
-
network: reach
|
|
1160
|
+
network: reach,
|
|
1161
|
+
hosts: gated ? namedHosts(source) : undefined,
|
|
1162
|
+
askConnect: gated ? confirmConnect : undefined
|
|
1157
1163
|
});
|
|
1158
1164
|
const soft = softFailedDeckHint(result.stdout, result.exitCode, source);
|
|
1159
1165
|
if (soft)
|
|
@@ -65,6 +65,7 @@ export type RunSpec = {
|
|
|
65
65
|
readonly maxOutputChars: number;
|
|
66
66
|
readonly installTimeoutMs: number;
|
|
67
67
|
readonly network?: boolean | undefined;
|
|
68
|
+
readonly hosts?: readonly string[] | undefined;
|
|
68
69
|
};
|
|
69
70
|
export type RunOutcome = {
|
|
70
71
|
readonly result: PythonRunResult;
|
|
@@ -75,4 +76,5 @@ export declare function isPyodideFatal(error: unknown): boolean;
|
|
|
75
76
|
export declare function repairDoubleEscapedSource(code: string, message: string): string | null;
|
|
76
77
|
export declare function performRun(py: PyodideLike, spec: RunSpec, opts?: {
|
|
77
78
|
readonly isCancelled?: () => boolean;
|
|
79
|
+
readonly askConnect?: ((host: string) => Promise<boolean>) | undefined;
|
|
78
80
|
}): Promise<RunOutcome>;
|
|
@@ -2,7 +2,7 @@ import { AbortedError } from '../../../../helpers/race-abort.js';
|
|
|
2
2
|
import { hlog } from '../../../../log.js';
|
|
3
3
|
import { TRUNCATED_OUTPUT_SUFFIX } from '../../runner.js';
|
|
4
4
|
import { collectOutputs, openScratch, stageInputFiles } from './memfs.js';
|
|
5
|
-
import { isGuardedFetch } from './guarded-fetch.js';
|
|
5
|
+
import { isGuardedFetch, setConnectPolicy } from './guarded-fetch.js';
|
|
6
6
|
import { NET_SHIM, pythonReachesNetwork } from './net-shim.js';
|
|
7
7
|
import { planVendoredInstall, stageVendoredWheels } from './wheels.js';
|
|
8
8
|
export function cap(text, limit) {
|
|
@@ -40,9 +40,9 @@ export function repairDoubleEscapedSource(code, message) {
|
|
|
40
40
|
return repaired === code ? null : repaired;
|
|
41
41
|
}
|
|
42
42
|
const promisingEval = new WeakMap();
|
|
43
|
-
async function enter(py, code,
|
|
43
|
+
async function enter(py, code, access) {
|
|
44
44
|
const guarded = pythonReachesNetwork() && isGuardedFetch(globalThis.fetch);
|
|
45
|
-
if (!
|
|
45
|
+
if (!access || !guarded || py.ffi === undefined || py.pyimport === undefined) {
|
|
46
46
|
py.runPython(code);
|
|
47
47
|
return;
|
|
48
48
|
}
|
|
@@ -52,11 +52,17 @@ async function enter(py, code, network) {
|
|
|
52
52
|
evalCode = py.pyimport('pyodide.code').eval_code;
|
|
53
53
|
promisingEval.set(py, evalCode);
|
|
54
54
|
}
|
|
55
|
-
|
|
55
|
+
setConnectPolicy(access.policy);
|
|
56
|
+
try {
|
|
57
|
+
await evalCode.callPromising(code, py.globals);
|
|
58
|
+
}
|
|
59
|
+
finally {
|
|
60
|
+
setConnectPolicy(null);
|
|
61
|
+
}
|
|
56
62
|
}
|
|
57
|
-
async function attempt(py, code, output, maxOutputChars,
|
|
63
|
+
async function attempt(py, code, output, maxOutputChars, access) {
|
|
58
64
|
try {
|
|
59
|
-
await enter(py, code,
|
|
65
|
+
await enter(py, code, access);
|
|
60
66
|
return {
|
|
61
67
|
result: {
|
|
62
68
|
stdout: cap(output.stdout, maxOutputChars),
|
|
@@ -80,14 +86,14 @@ async function attempt(py, code, output, maxOutputChars, network) {
|
|
|
80
86
|
};
|
|
81
87
|
}
|
|
82
88
|
}
|
|
83
|
-
async function runOnce(py, code, output, maxOutputChars,
|
|
84
|
-
const first = await attempt(py, code, output, maxOutputChars,
|
|
89
|
+
async function runOnce(py, code, output, maxOutputChars, access) {
|
|
90
|
+
const first = await attempt(py, code, output, maxOutputChars, access);
|
|
85
91
|
if (first.error === null)
|
|
86
92
|
return first;
|
|
87
93
|
const repaired = repairDoubleEscapedSource(code, first.error);
|
|
88
94
|
if (repaired === null)
|
|
89
95
|
return first;
|
|
90
|
-
const retry = await attempt(py, repaired, output, maxOutputChars,
|
|
96
|
+
const retry = await attempt(py, repaired, output, maxOutputChars, access);
|
|
91
97
|
if (retry.error !== null && retry.error.includes('SyntaxError'))
|
|
92
98
|
return first;
|
|
93
99
|
return retry;
|
|
@@ -213,6 +219,8 @@ function fitDecks(py, outputs) {
|
|
|
213
219
|
}
|
|
214
220
|
export async function performRun(py, spec, opts = {}) {
|
|
215
221
|
const isCancelled = opts.isCancelled ?? (() => false);
|
|
222
|
+
const ask = opts.askConnect ?? (async () => false);
|
|
223
|
+
const access = spec.network === true ? { policy: spec.hosts ? { named: spec.hosts, ask } : null } : null;
|
|
216
224
|
const output = { stdout: '', stderr: '' };
|
|
217
225
|
py.setStdout({ batched: (chunk) => (output.stdout += chunk + '\n') });
|
|
218
226
|
py.setStderr({ batched: (chunk) => (output.stderr += chunk + '\n') });
|
|
@@ -251,7 +259,7 @@ export async function performRun(py, spec, opts = {}) {
|
|
|
251
259
|
if (!spec.inputs?.length && !spec.outputs?.length && !spec.workdir) {
|
|
252
260
|
const cwd = py.FS.cwd();
|
|
253
261
|
const before = listFiles(py, cwd);
|
|
254
|
-
const { result, fatal } = await runOnce(py, spec.code, output, spec.maxOutputChars,
|
|
262
|
+
const { result, fatal } = await runOnce(py, spec.code, output, spec.maxOutputChars, access);
|
|
255
263
|
if (result.exitCode !== 0)
|
|
256
264
|
return { result, evict: fatal };
|
|
257
265
|
const undeclared = newDeliverables(before, listFiles(py, cwd));
|
|
@@ -262,7 +270,7 @@ export async function performRun(py, spec, opts = {}) {
|
|
|
262
270
|
py.FS.chdir(scratch.cwd);
|
|
263
271
|
stageInputFiles(py.FS, scratch.cwd, spec.inputs ?? []);
|
|
264
272
|
const before = listFiles(py, scratch.cwd);
|
|
265
|
-
const { result, fatal } = await runOnce(py, spec.code, output, spec.maxOutputChars,
|
|
273
|
+
const { result, fatal } = await runOnce(py, spec.code, output, spec.maxOutputChars, access);
|
|
266
274
|
let evict = fatal;
|
|
267
275
|
if (result.exitCode === 0)
|
|
268
276
|
evict = fitDecks(py, spec.outputs ?? []);
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import type
|
|
1
|
+
import { type AbortSignal as BareAbortSignal } from 'bare-abort-controller';
|
|
2
2
|
import type { PythonRunResult, RunSpec } from '../run-session.ts';
|
|
3
3
|
export declare function runPythonInWorker(spec: RunSpec, opts: {
|
|
4
4
|
readonly timeoutMs: number;
|
|
5
5
|
readonly signal?: BareAbortSignal | undefined;
|
|
6
|
+
readonly askConnect?: ((host: string) => Promise<boolean>) | undefined;
|
|
6
7
|
}): Promise<PythonRunResult>;
|
|
7
8
|
export declare function closePythonWorker(): Promise<void>;
|
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
import { spawnPythonWorker } from '#pyodide-worker-spawn';
|
|
2
|
+
import { AbortController } from 'bare-abort-controller';
|
|
2
3
|
import process from 'process';
|
|
3
4
|
import { fileURLToPath } from 'url';
|
|
4
5
|
import { AbortedError, raceAbort } from '../../../../../helpers/race-abort.js';
|
|
5
6
|
import { harnessLogEnabled, hlog } from '../../../../../log.js';
|
|
6
7
|
import { asWorkerMessage, fromWireResult } from './protocol.js';
|
|
7
8
|
const SIGINT = 2;
|
|
9
|
+
const RESEND_MS = 100;
|
|
8
10
|
function interruptGraceMs() {
|
|
9
11
|
const ms = Number(process.env.QVAC_PYODIDE_INTERRUPT_GRACE_MS);
|
|
10
12
|
return ms > 0 ? ms : 30_000;
|
|
@@ -69,8 +71,13 @@ async function drainAndReap(worker) {
|
|
|
69
71
|
timer.clear();
|
|
70
72
|
}
|
|
71
73
|
}
|
|
72
|
-
async function resultFor(worker, id) {
|
|
73
|
-
|
|
74
|
+
async function resultFor(worker, id, askConnect) {
|
|
75
|
+
let message = await readMessage(worker.link);
|
|
76
|
+
while (message?.type === 'connect-ask') {
|
|
77
|
+
const allowed = askConnect ? await askConnect(message.host).catch(() => false) : false;
|
|
78
|
+
await worker.link.write({ type: 'connect-answer', allowed });
|
|
79
|
+
message = await readMessage(worker.link);
|
|
80
|
+
}
|
|
74
81
|
if (message?.type === 'result') {
|
|
75
82
|
if (message.id !== id)
|
|
76
83
|
throw new Error('python worker answered a different run');
|
|
@@ -89,6 +96,7 @@ function grace(ms) {
|
|
|
89
96
|
}
|
|
90
97
|
async function interrupt(worker, outcome) {
|
|
91
98
|
Atomics.store(worker.view, 0, SIGINT);
|
|
99
|
+
const resend = setInterval(() => Atomics.store(worker.view, 0, SIGINT), RESEND_MS);
|
|
92
100
|
const graceMs = interruptGraceMs();
|
|
93
101
|
const timer = grace(graceMs);
|
|
94
102
|
try {
|
|
@@ -105,6 +113,7 @@ async function interrupt(worker, outcome) {
|
|
|
105
113
|
}
|
|
106
114
|
}
|
|
107
115
|
finally {
|
|
116
|
+
clearInterval(resend);
|
|
108
117
|
timer.clear();
|
|
109
118
|
}
|
|
110
119
|
}
|
|
@@ -116,13 +125,15 @@ export async function runPythonInWorker(spec, opts) {
|
|
|
116
125
|
Atomics.store(worker.view, 0, 0);
|
|
117
126
|
let cancelled = false;
|
|
118
127
|
let sent = false;
|
|
128
|
+
const stop = new AbortController();
|
|
129
|
+
const ask = opts.askConnect;
|
|
119
130
|
const outcome = (async () => {
|
|
120
131
|
await worker.ready;
|
|
121
132
|
if (cancelled)
|
|
122
133
|
throw new AbortedError();
|
|
123
134
|
sent = true;
|
|
124
135
|
await worker.link.write({ type: 'run', id, spec });
|
|
125
|
-
return resultFor(worker, id);
|
|
136
|
+
return resultFor(worker, id, ask && ((host) => raceAbort(ask(host), stop.signal)));
|
|
126
137
|
})();
|
|
127
138
|
let timer;
|
|
128
139
|
const timeout = new Promise((resolve) => {
|
|
@@ -135,6 +146,7 @@ export async function runPythonInWorker(spec, opts) {
|
|
|
135
146
|
catch (error) {
|
|
136
147
|
if (error instanceof AbortedError) {
|
|
137
148
|
cancelled = true;
|
|
149
|
+
stop.abort(undefined);
|
|
138
150
|
if (sent)
|
|
139
151
|
await interrupt(worker, outcome);
|
|
140
152
|
else
|
|
@@ -150,6 +162,7 @@ export async function runPythonInWorker(spec, opts) {
|
|
|
150
162
|
}
|
|
151
163
|
if (first === 'timeout') {
|
|
152
164
|
cancelled = true;
|
|
165
|
+
stop.abort(undefined);
|
|
153
166
|
if (sent)
|
|
154
167
|
await interrupt(worker, outcome);
|
|
155
168
|
else
|
|
@@ -90,9 +90,22 @@ try {
|
|
|
90
90
|
const message = asHostMessage(await port.read());
|
|
91
91
|
if (!message || message.type === 'close')
|
|
92
92
|
break;
|
|
93
|
-
|
|
93
|
+
if (message.type === 'connect-answer')
|
|
94
|
+
continue;
|
|
95
|
+
let closing = false;
|
|
96
|
+
const askConnect = async (host) => {
|
|
97
|
+
if (closing)
|
|
98
|
+
return false;
|
|
99
|
+
port.write({ type: 'connect-ask', host });
|
|
100
|
+
const answer = asHostMessage(await port.read());
|
|
101
|
+
if (answer?.type === 'connect-answer')
|
|
102
|
+
return answer.allowed;
|
|
103
|
+
closing = true;
|
|
104
|
+
return false;
|
|
105
|
+
};
|
|
106
|
+
const { result, evict } = await performRun(py, message.spec, { askConnect });
|
|
94
107
|
port.write({ type: 'result', id: message.id, result: toWireResult(result), evict });
|
|
95
|
-
if (evict)
|
|
108
|
+
if (evict || closing)
|
|
96
109
|
break;
|
|
97
110
|
}
|
|
98
111
|
}
|
|
@@ -5,6 +5,9 @@ export type HostMessage = {
|
|
|
5
5
|
readonly spec: RunSpec;
|
|
6
6
|
} | {
|
|
7
7
|
readonly type: 'close';
|
|
8
|
+
} | {
|
|
9
|
+
readonly type: 'connect-answer';
|
|
10
|
+
readonly allowed: boolean;
|
|
8
11
|
};
|
|
9
12
|
export type WireResult = {
|
|
10
13
|
readonly stdout: string;
|
|
@@ -31,6 +34,9 @@ export type WorkerMessage = {
|
|
|
31
34
|
readonly id: number;
|
|
32
35
|
readonly result: WireResult;
|
|
33
36
|
readonly evict: boolean;
|
|
37
|
+
} | {
|
|
38
|
+
readonly type: 'connect-ask';
|
|
39
|
+
readonly host: string;
|
|
34
40
|
};
|
|
35
41
|
export declare function toWireResult(result: PythonRunResult): WireResult;
|
|
36
42
|
export declare function fromWireResult(wire: WireResult): PythonRunResult;
|
|
@@ -28,13 +28,17 @@ function messageType(value) {
|
|
|
28
28
|
}
|
|
29
29
|
export function asHostMessage(value) {
|
|
30
30
|
const type = messageType(value);
|
|
31
|
-
if (type === 'run' || type === 'close')
|
|
31
|
+
if (type === 'run' || type === 'close' || type === 'connect-answer')
|
|
32
32
|
return value;
|
|
33
33
|
return null;
|
|
34
34
|
}
|
|
35
35
|
export function asWorkerMessage(value) {
|
|
36
36
|
const type = messageType(value);
|
|
37
|
-
if (type === 'log' ||
|
|
37
|
+
if (type === 'log' ||
|
|
38
|
+
type === 'ready' ||
|
|
39
|
+
type === 'boot-error' ||
|
|
40
|
+
type === 'result' ||
|
|
41
|
+
type === 'connect-ask') {
|
|
38
42
|
return value;
|
|
39
43
|
}
|
|
40
44
|
return null;
|