@qvac/harness 0.3.0 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +262 -0
- package/dist/harness.bundle +15 -11
- package/dist-lib/index.d.ts +1 -1
- package/dist-lib/lib/capabilities/completion.d.ts +5 -1
- package/dist-lib/lib/capabilities/completion.js +20 -8
- package/dist-lib/lib/helpers/tool-call-markup.js +15 -7
- package/dist-lib/lib/no-redirect-fetch-impl.d.ts +3 -2
- package/dist-lib/lib/no-redirect-fetch-impl.js +4 -1
- package/dist-lib/lib/tools/exec/adapters/adapter.d.ts +5 -1
- package/dist-lib/lib/tools/exec/adapters/python/distributions.d.ts +1 -0
- package/dist-lib/lib/tools/exec/adapters/python/distributions.js +4 -0
- package/dist-lib/lib/tools/exec/adapters/python/guarded-fetch.d.ts +72 -0
- package/dist-lib/lib/tools/exec/adapters/python/guarded-fetch.js +222 -0
- package/dist-lib/lib/tools/exec/adapters/python/interrupt-timer.d.ts +5 -0
- package/dist-lib/lib/tools/exec/adapters/python/interrupt-timer.js +29 -0
- package/dist-lib/lib/tools/exec/adapters/python/loader.js +2 -1
- package/dist-lib/lib/tools/exec/adapters/python/module-distributions.d.ts +1 -0
- package/dist-lib/lib/tools/exec/adapters/python/module-distributions.js +52 -0
- package/dist-lib/lib/tools/exec/adapters/python/named-hosts.d.ts +2 -0
- package/dist-lib/lib/tools/exec/adapters/python/named-hosts.js +15 -0
- package/dist-lib/lib/tools/exec/adapters/python/net-shim.d.ts +2 -0
- package/dist-lib/lib/tools/exec/adapters/python/net-shim.js +148 -0
- package/dist-lib/lib/tools/exec/adapters/python/output-policy.js +13 -1
- package/dist-lib/lib/tools/exec/adapters/python/pyodide.d.ts +4 -1
- package/dist-lib/lib/tools/exec/adapters/python/pyodide.js +42 -7
- package/dist-lib/lib/tools/exec/adapters/python/python-adapter.d.ts +9 -3
- package/dist-lib/lib/tools/exec/adapters/python/python-adapter.js +137 -21
- package/dist-lib/lib/tools/exec/adapters/python/run-session.d.ts +5 -0
- package/dist-lib/lib/tools/exec/adapters/python/run-session.js +33 -8
- package/dist-lib/lib/tools/exec/adapters/python/wheels.d.ts +6 -0
- package/dist-lib/lib/tools/exec/adapters/python/wheels.js +17 -3
- package/dist-lib/lib/tools/exec/adapters/python/worker/client.d.ts +2 -1
- package/dist-lib/lib/tools/exec/adapters/python/worker/client.js +16 -3
- package/dist-lib/lib/tools/exec/adapters/python/worker/entry.js +15 -2
- package/dist-lib/lib/tools/exec/adapters/python/worker/protocol.d.ts +6 -0
- package/dist-lib/lib/tools/exec/adapters/python/worker/protocol.js +6 -2
- package/dist-lib/lib/tools/exec/adapters/python/worker/spawn-bare.d.ts +1 -0
- package/dist-lib/lib/tools/exec/adapters/python/worker/spawn-bare.js +8 -0
- package/dist-lib/lib/tools/exec/adapters/python/worker/spawn-node.d.ts +1 -0
- package/dist-lib/lib/tools/exec/adapters/python/worker/spawn-node.js +8 -0
- package/dist-lib/lib/tools/exec/adapters/shell/shell-adapter.js +7 -1
- package/dist-lib/lib/tools/exec/shell-guidance.d.ts +10 -0
- package/dist-lib/lib/tools/exec/shell-guidance.js +123 -0
- package/dist-lib/lib/tools/exec/video-tools.d.ts +5 -0
- package/dist-lib/lib/tools/exec/video-tools.js +40 -0
- package/dist-lib/lib/tools/exec.d.ts +2 -1
- package/dist-lib/lib/tools/exec.js +93 -33
- package/dist-lib/lib/tools/http-request/truncate-response.d.ts +1 -0
- package/dist-lib/lib/tools/http-request/truncate-response.js +1 -1
- package/dist-lib/lib/tools/http-request.js +4 -1
- package/dist-lib/lib/tools/operations/builders.d.ts +1 -1
- package/dist-lib/lib/tools/operations/builders.js +9 -0
- package/dist-lib/lib/tools/operations/compile.js +41 -18
- package/dist-lib/lib/tools/operations/decoders.d.ts +6 -0
- package/dist-lib/lib/tools/operations/decoders.js +82 -0
- package/dist-lib/lib/tools/operations/manifest.d.ts +16 -2
- package/dist-lib/lib/tools/operations/manifest.js +6 -1
- package/dist-lib/lib/tools/skill.d.ts +3 -0
- package/dist-lib/lib/tools/skill.js +38 -5
- package/dist-lib/lib/tools/skills/manifest-parse.js +43 -1
- package/dist-lib/lib/tools/skills/manifest.d.ts +13 -0
- package/dist-lib/schema/harness.js +20 -1
- package/dist-lib/spec/hrpc/messages.js +93 -5
- package/dist-lib/spec/hyperschema/index.js +93 -5
- package/dist-lib/spec/hyperschema/schema.json +66 -1
- package/dist-lib/spec/hyperschema/types.d.ts +18 -1
- package/package.json +40 -2
- package/wheels/manifest.json +15 -0
|
@@ -2,6 +2,8 @@ import { AbortedError } from '../../../../helpers/race-abort.js';
|
|
|
2
2
|
import { hlog } from '../../../../log.js';
|
|
3
3
|
import { TRUNCATED_OUTPUT_SUFFIX } from '../../runner.js';
|
|
4
4
|
import { collectOutputs, openScratch, stageInputFiles } from './memfs.js';
|
|
5
|
+
import { isGuardedFetch, setConnectPolicy } from './guarded-fetch.js';
|
|
6
|
+
import { NET_SHIM, pythonReachesNetwork } from './net-shim.js';
|
|
5
7
|
import { planVendoredInstall, stageVendoredWheels } from './wheels.js';
|
|
6
8
|
export function cap(text, limit) {
|
|
7
9
|
if (text.length <= limit)
|
|
@@ -37,9 +39,30 @@ export function repairDoubleEscapedSource(code, message) {
|
|
|
37
39
|
const repaired = unescapeSource(code);
|
|
38
40
|
return repaired === code ? null : repaired;
|
|
39
41
|
}
|
|
40
|
-
|
|
41
|
-
|
|
42
|
+
const promisingEval = new WeakMap();
|
|
43
|
+
async function enter(py, code, access) {
|
|
44
|
+
const guarded = pythonReachesNetwork() && isGuardedFetch(globalThis.fetch);
|
|
45
|
+
if (!access || !guarded || py.ffi === undefined || py.pyimport === undefined) {
|
|
42
46
|
py.runPython(code);
|
|
47
|
+
return;
|
|
48
|
+
}
|
|
49
|
+
let evalCode = promisingEval.get(py);
|
|
50
|
+
if (!evalCode) {
|
|
51
|
+
py.runPython(NET_SHIM);
|
|
52
|
+
evalCode = py.pyimport('pyodide.code').eval_code;
|
|
53
|
+
promisingEval.set(py, evalCode);
|
|
54
|
+
}
|
|
55
|
+
setConnectPolicy(access.policy);
|
|
56
|
+
try {
|
|
57
|
+
await evalCode.callPromising(code, py.globals);
|
|
58
|
+
}
|
|
59
|
+
finally {
|
|
60
|
+
setConnectPolicy(null);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
async function attempt(py, code, output, maxOutputChars, access) {
|
|
64
|
+
try {
|
|
65
|
+
await enter(py, code, access);
|
|
43
66
|
return {
|
|
44
67
|
result: {
|
|
45
68
|
stdout: cap(output.stdout, maxOutputChars),
|
|
@@ -63,14 +86,14 @@ function attempt(py, code, output, maxOutputChars) {
|
|
|
63
86
|
};
|
|
64
87
|
}
|
|
65
88
|
}
|
|
66
|
-
function runOnce(py, code, output, maxOutputChars) {
|
|
67
|
-
const first = attempt(py, code, output, maxOutputChars);
|
|
89
|
+
async function runOnce(py, code, output, maxOutputChars, access) {
|
|
90
|
+
const first = await attempt(py, code, output, maxOutputChars, access);
|
|
68
91
|
if (first.error === null)
|
|
69
92
|
return first;
|
|
70
93
|
const repaired = repairDoubleEscapedSource(code, first.error);
|
|
71
94
|
if (repaired === null)
|
|
72
95
|
return first;
|
|
73
|
-
const retry = attempt(py, repaired, output, maxOutputChars);
|
|
96
|
+
const retry = await attempt(py, repaired, output, maxOutputChars, access);
|
|
74
97
|
if (retry.error !== null && retry.error.includes('SyntaxError'))
|
|
75
98
|
return first;
|
|
76
99
|
return retry;
|
|
@@ -168,7 +191,7 @@ async function installPackages(py, packages, timeoutMs, isCancelled) {
|
|
|
168
191
|
clearTimeout(timer);
|
|
169
192
|
}
|
|
170
193
|
}
|
|
171
|
-
const DELIVERABLE = /\.(pptx|docx|xlsx|pdf|png|jpe?g|gif|webp|csv|json|txt|md|zip)$/i;
|
|
194
|
+
const DELIVERABLE = /\.(pptx|docx|xlsx|pdf|png|jpe?g|gif|webp|csv|json|txt|md|zip|mp4|webm|m4a|mp3)$/i;
|
|
172
195
|
function listFiles(py, dir) {
|
|
173
196
|
try {
|
|
174
197
|
return new Set(py.FS.readdir(dir).filter((name) => name !== '.' && name !== '..'));
|
|
@@ -196,6 +219,8 @@ function fitDecks(py, outputs) {
|
|
|
196
219
|
}
|
|
197
220
|
export async function performRun(py, spec, opts = {}) {
|
|
198
221
|
const isCancelled = opts.isCancelled ?? (() => false);
|
|
222
|
+
const ask = opts.askConnect ?? (async () => false);
|
|
223
|
+
const access = spec.network === true ? { policy: spec.hosts ? { named: spec.hosts, ask } : null } : null;
|
|
199
224
|
const output = { stdout: '', stderr: '' };
|
|
200
225
|
py.setStdout({ batched: (chunk) => (output.stdout += chunk + '\n') });
|
|
201
226
|
py.setStderr({ batched: (chunk) => (output.stderr += chunk + '\n') });
|
|
@@ -234,7 +259,7 @@ export async function performRun(py, spec, opts = {}) {
|
|
|
234
259
|
if (!spec.inputs?.length && !spec.outputs?.length && !spec.workdir) {
|
|
235
260
|
const cwd = py.FS.cwd();
|
|
236
261
|
const before = listFiles(py, cwd);
|
|
237
|
-
const { result, fatal } = runOnce(py, spec.code, output, spec.maxOutputChars);
|
|
262
|
+
const { result, fatal } = await runOnce(py, spec.code, output, spec.maxOutputChars, access);
|
|
238
263
|
if (result.exitCode !== 0)
|
|
239
264
|
return { result, evict: fatal };
|
|
240
265
|
const undeclared = newDeliverables(before, listFiles(py, cwd));
|
|
@@ -245,7 +270,7 @@ export async function performRun(py, spec, opts = {}) {
|
|
|
245
270
|
py.FS.chdir(scratch.cwd);
|
|
246
271
|
stageInputFiles(py.FS, scratch.cwd, spec.inputs ?? []);
|
|
247
272
|
const before = listFiles(py, scratch.cwd);
|
|
248
|
-
const { result, fatal } = runOnce(py, spec.code, output, spec.maxOutputChars);
|
|
273
|
+
const { result, fatal } = await runOnce(py, spec.code, output, spec.maxOutputChars, access);
|
|
249
274
|
let evict = fatal;
|
|
250
275
|
if (result.exitCode === 0)
|
|
251
276
|
evict = fitDecks(py, spec.outputs ?? []);
|
|
@@ -3,6 +3,7 @@ declare const ManifestSchema: z.ZodObject<{
|
|
|
3
3
|
pyodide: z.ZodString;
|
|
4
4
|
packages: z.ZodRecord<z.ZodString, z.ZodObject<{
|
|
5
5
|
version: z.ZodOptional<z.ZodString>;
|
|
6
|
+
imports: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
6
7
|
pypi: z.ZodArray<z.ZodString>;
|
|
7
8
|
pyodide: z.ZodArray<z.ZodString>;
|
|
8
9
|
}, z.core.$strip>>;
|
|
@@ -29,6 +30,11 @@ type WheelFs = {
|
|
|
29
30
|
};
|
|
30
31
|
export declare function wheelsDir(): string | undefined;
|
|
31
32
|
export declare function vendoredManifest(dir?: string | undefined): WheelManifest | null;
|
|
33
|
+
export declare function normalizeDistribution(name: string): string;
|
|
34
|
+
export declare function vendoredSpec(distribution: string, opts?: {
|
|
35
|
+
readonly dir?: string | undefined;
|
|
36
|
+
readonly manifest?: WheelManifest | null;
|
|
37
|
+
}): string | null;
|
|
32
38
|
export declare function planVendoredInstall(specs: readonly string[], opts?: {
|
|
33
39
|
readonly dir?: string | undefined;
|
|
34
40
|
readonly manifest?: WheelManifest | null;
|
|
@@ -7,6 +7,7 @@ const ManifestSchema = z.object({
|
|
|
7
7
|
pyodide: z.string(),
|
|
8
8
|
packages: z.record(z.string(), z.object({
|
|
9
9
|
version: z.string().optional(),
|
|
10
|
+
imports: z.array(z.string()).optional(),
|
|
10
11
|
pypi: z.array(z.string()),
|
|
11
12
|
pyodide: z.array(z.string())
|
|
12
13
|
})),
|
|
@@ -42,12 +43,25 @@ export function vendoredManifest(dir = wheelsDir()) {
|
|
|
42
43
|
return null;
|
|
43
44
|
}
|
|
44
45
|
}
|
|
45
|
-
function
|
|
46
|
-
return name
|
|
46
|
+
export function normalizeDistribution(name) {
|
|
47
|
+
return name
|
|
48
|
+
.trim()
|
|
49
|
+
.toLowerCase()
|
|
50
|
+
.replace(/[-_.]+/g, '-');
|
|
47
51
|
}
|
|
48
52
|
function parseSpec(spec) {
|
|
49
53
|
const [name, version] = spec.split('==');
|
|
50
|
-
return { name:
|
|
54
|
+
return { name: normalizeDistribution(name ?? ''), version: version?.trim() };
|
|
55
|
+
}
|
|
56
|
+
export function vendoredSpec(distribution, opts = {}) {
|
|
57
|
+
const name = normalizeDistribution(distribution);
|
|
58
|
+
const dir = opts.dir ?? wheelsDir();
|
|
59
|
+
const manifest = opts.manifest === undefined ? vendoredManifest(dir) : opts.manifest;
|
|
60
|
+
const entry = manifest?.packages[name];
|
|
61
|
+
if (!entry)
|
|
62
|
+
return null;
|
|
63
|
+
const spec = entry.version ? `${name}==${entry.version}` : name;
|
|
64
|
+
return planVendoredInstall([spec], { dir, manifest }) ? spec : null;
|
|
51
65
|
}
|
|
52
66
|
export function planVendoredInstall(specs, opts = {}) {
|
|
53
67
|
const dir = opts.dir ?? wheelsDir();
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import type
|
|
1
|
+
import { type AbortSignal as BareAbortSignal } from 'bare-abort-controller';
|
|
2
2
|
import type { PythonRunResult, RunSpec } from '../run-session.ts';
|
|
3
3
|
export declare function runPythonInWorker(spec: RunSpec, opts: {
|
|
4
4
|
readonly timeoutMs: number;
|
|
5
5
|
readonly signal?: BareAbortSignal | undefined;
|
|
6
|
+
readonly askConnect?: ((host: string) => Promise<boolean>) | undefined;
|
|
6
7
|
}): Promise<PythonRunResult>;
|
|
7
8
|
export declare function closePythonWorker(): Promise<void>;
|
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
import { spawnPythonWorker } from '#pyodide-worker-spawn';
|
|
2
|
+
import { AbortController } from 'bare-abort-controller';
|
|
2
3
|
import process from 'process';
|
|
3
4
|
import { fileURLToPath } from 'url';
|
|
4
5
|
import { AbortedError, raceAbort } from '../../../../../helpers/race-abort.js';
|
|
5
6
|
import { harnessLogEnabled, hlog } from '../../../../../log.js';
|
|
6
7
|
import { asWorkerMessage, fromWireResult } from './protocol.js';
|
|
7
8
|
const SIGINT = 2;
|
|
9
|
+
const RESEND_MS = 100;
|
|
8
10
|
function interruptGraceMs() {
|
|
9
11
|
const ms = Number(process.env.QVAC_PYODIDE_INTERRUPT_GRACE_MS);
|
|
10
12
|
return ms > 0 ? ms : 30_000;
|
|
@@ -69,8 +71,13 @@ async function drainAndReap(worker) {
|
|
|
69
71
|
timer.clear();
|
|
70
72
|
}
|
|
71
73
|
}
|
|
72
|
-
async function resultFor(worker, id) {
|
|
73
|
-
|
|
74
|
+
async function resultFor(worker, id, askConnect) {
|
|
75
|
+
let message = await readMessage(worker.link);
|
|
76
|
+
while (message?.type === 'connect-ask') {
|
|
77
|
+
const allowed = askConnect ? await askConnect(message.host).catch(() => false) : false;
|
|
78
|
+
await worker.link.write({ type: 'connect-answer', allowed });
|
|
79
|
+
message = await readMessage(worker.link);
|
|
80
|
+
}
|
|
74
81
|
if (message?.type === 'result') {
|
|
75
82
|
if (message.id !== id)
|
|
76
83
|
throw new Error('python worker answered a different run');
|
|
@@ -89,6 +96,7 @@ function grace(ms) {
|
|
|
89
96
|
}
|
|
90
97
|
async function interrupt(worker, outcome) {
|
|
91
98
|
Atomics.store(worker.view, 0, SIGINT);
|
|
99
|
+
const resend = setInterval(() => Atomics.store(worker.view, 0, SIGINT), RESEND_MS);
|
|
92
100
|
const graceMs = interruptGraceMs();
|
|
93
101
|
const timer = grace(graceMs);
|
|
94
102
|
try {
|
|
@@ -105,6 +113,7 @@ async function interrupt(worker, outcome) {
|
|
|
105
113
|
}
|
|
106
114
|
}
|
|
107
115
|
finally {
|
|
116
|
+
clearInterval(resend);
|
|
108
117
|
timer.clear();
|
|
109
118
|
}
|
|
110
119
|
}
|
|
@@ -116,13 +125,15 @@ export async function runPythonInWorker(spec, opts) {
|
|
|
116
125
|
Atomics.store(worker.view, 0, 0);
|
|
117
126
|
let cancelled = false;
|
|
118
127
|
let sent = false;
|
|
128
|
+
const stop = new AbortController();
|
|
129
|
+
const ask = opts.askConnect;
|
|
119
130
|
const outcome = (async () => {
|
|
120
131
|
await worker.ready;
|
|
121
132
|
if (cancelled)
|
|
122
133
|
throw new AbortedError();
|
|
123
134
|
sent = true;
|
|
124
135
|
await worker.link.write({ type: 'run', id, spec });
|
|
125
|
-
return resultFor(worker, id);
|
|
136
|
+
return resultFor(worker, id, ask && ((host) => raceAbort(ask(host), stop.signal)));
|
|
126
137
|
})();
|
|
127
138
|
let timer;
|
|
128
139
|
const timeout = new Promise((resolve) => {
|
|
@@ -135,6 +146,7 @@ export async function runPythonInWorker(spec, opts) {
|
|
|
135
146
|
catch (error) {
|
|
136
147
|
if (error instanceof AbortedError) {
|
|
137
148
|
cancelled = true;
|
|
149
|
+
stop.abort(undefined);
|
|
138
150
|
if (sent)
|
|
139
151
|
await interrupt(worker, outcome);
|
|
140
152
|
else
|
|
@@ -150,6 +162,7 @@ export async function runPythonInWorker(spec, opts) {
|
|
|
150
162
|
}
|
|
151
163
|
if (first === 'timeout') {
|
|
152
164
|
cancelled = true;
|
|
165
|
+
stop.abort(undefined);
|
|
153
166
|
if (sent)
|
|
154
167
|
await interrupt(worker, outcome);
|
|
155
168
|
else
|
|
@@ -90,9 +90,22 @@ try {
|
|
|
90
90
|
const message = asHostMessage(await port.read());
|
|
91
91
|
if (!message || message.type === 'close')
|
|
92
92
|
break;
|
|
93
|
-
|
|
93
|
+
if (message.type === 'connect-answer')
|
|
94
|
+
continue;
|
|
95
|
+
let closing = false;
|
|
96
|
+
const askConnect = async (host) => {
|
|
97
|
+
if (closing)
|
|
98
|
+
return false;
|
|
99
|
+
port.write({ type: 'connect-ask', host });
|
|
100
|
+
const answer = asHostMessage(await port.read());
|
|
101
|
+
if (answer?.type === 'connect-answer')
|
|
102
|
+
return answer.allowed;
|
|
103
|
+
closing = true;
|
|
104
|
+
return false;
|
|
105
|
+
};
|
|
106
|
+
const { result, evict } = await performRun(py, message.spec, { askConnect });
|
|
94
107
|
port.write({ type: 'result', id: message.id, result: toWireResult(result), evict });
|
|
95
|
-
if (evict)
|
|
108
|
+
if (evict || closing)
|
|
96
109
|
break;
|
|
97
110
|
}
|
|
98
111
|
}
|
|
@@ -5,6 +5,9 @@ export type HostMessage = {
|
|
|
5
5
|
readonly spec: RunSpec;
|
|
6
6
|
} | {
|
|
7
7
|
readonly type: 'close';
|
|
8
|
+
} | {
|
|
9
|
+
readonly type: 'connect-answer';
|
|
10
|
+
readonly allowed: boolean;
|
|
8
11
|
};
|
|
9
12
|
export type WireResult = {
|
|
10
13
|
readonly stdout: string;
|
|
@@ -31,6 +34,9 @@ export type WorkerMessage = {
|
|
|
31
34
|
readonly id: number;
|
|
32
35
|
readonly result: WireResult;
|
|
33
36
|
readonly evict: boolean;
|
|
37
|
+
} | {
|
|
38
|
+
readonly type: 'connect-ask';
|
|
39
|
+
readonly host: string;
|
|
34
40
|
};
|
|
35
41
|
export declare function toWireResult(result: PythonRunResult): WireResult;
|
|
36
42
|
export declare function fromWireResult(wire: WireResult): PythonRunResult;
|
|
@@ -28,13 +28,17 @@ function messageType(value) {
|
|
|
28
28
|
}
|
|
29
29
|
export function asHostMessage(value) {
|
|
30
30
|
const type = messageType(value);
|
|
31
|
-
if (type === 'run' || type === 'close')
|
|
31
|
+
if (type === 'run' || type === 'close' || type === 'connect-answer')
|
|
32
32
|
return value;
|
|
33
33
|
return null;
|
|
34
34
|
}
|
|
35
35
|
export function asWorkerMessage(value) {
|
|
36
36
|
const type = messageType(value);
|
|
37
|
-
if (type === 'log' ||
|
|
37
|
+
if (type === 'log' ||
|
|
38
|
+
type === 'ready' ||
|
|
39
|
+
type === 'boot-error' ||
|
|
40
|
+
type === 'result' ||
|
|
41
|
+
type === 'connect-ask') {
|
|
38
42
|
return value;
|
|
39
43
|
}
|
|
40
44
|
return null;
|
|
@@ -1,5 +1,13 @@
|
|
|
1
1
|
import Channel from 'bare-channel';
|
|
2
2
|
import safetyCatch from 'safety-catch';
|
|
3
|
+
export function spawnInlineThread(source, data) {
|
|
4
|
+
const Thread = Bare.Thread;
|
|
5
|
+
const thread = new Thread('inline-thread.js', {
|
|
6
|
+
data,
|
|
7
|
+
source: `const data = Bare.Thread.self.data\n${source}`
|
|
8
|
+
});
|
|
9
|
+
return () => thread.join();
|
|
10
|
+
}
|
|
3
11
|
export function spawnPythonWorker(entry, data) {
|
|
4
12
|
const channel = new Channel();
|
|
5
13
|
const thread = new Bare.Thread(entry, { data: { handle: channel.handle, ...data } });
|
|
@@ -43,3 +43,11 @@ export function spawnPythonWorker(entry, data) {
|
|
|
43
43
|
unref: () => worker.unref()
|
|
44
44
|
};
|
|
45
45
|
}
|
|
46
|
+
export function spawnInlineThread(source, data) {
|
|
47
|
+
const worker = new Worker(`const data = require('worker_threads').workerData\n${source}`, {
|
|
48
|
+
eval: true,
|
|
49
|
+
workerData: data
|
|
50
|
+
});
|
|
51
|
+
worker.unref();
|
|
52
|
+
return () => { };
|
|
53
|
+
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { runCommand } from '../../runner.js';
|
|
2
|
+
import { shellDownloadNote } from '../../shell-guidance.js';
|
|
2
3
|
export function createShellAdapter(options) {
|
|
3
4
|
return {
|
|
4
5
|
resolve(args) {
|
|
@@ -8,11 +9,16 @@ export function createShellAdapter(options) {
|
|
|
8
9
|
const runtime = options.spawn
|
|
9
10
|
? { spawn: options.spawn, killProcess: options.killProcess }
|
|
10
11
|
: await options.loadRuntime();
|
|
11
|
-
|
|
12
|
+
const result = await runCommand(command, {
|
|
12
13
|
...execution,
|
|
13
14
|
spawn: runtime.spawn,
|
|
14
15
|
killProcess: runtime.killProcess
|
|
15
16
|
});
|
|
17
|
+
const note = result.exitCode === 0 ? shellDownloadNote(command) : null;
|
|
18
|
+
if (!note)
|
|
19
|
+
return result;
|
|
20
|
+
const stdout = `${result.stdout}${result.stdout && !result.stdout.endsWith('\n') ? '\n' : ''}${note}`;
|
|
21
|
+
return { ...result, stdout };
|
|
16
22
|
}
|
|
17
23
|
};
|
|
18
24
|
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export declare const PYTHON_BIN_ERROR: string;
|
|
2
|
+
export declare const PACKAGES_NEED_PYTHON_ERROR: string;
|
|
3
|
+
export declare const FILES_NEED_PYTHON_ERROR: string;
|
|
4
|
+
export declare function videoToolError(command: string, packages: readonly string[] | undefined, python: boolean, network?: boolean): string | null;
|
|
5
|
+
export declare const SHELL_DOWNLOAD_NOTE: string;
|
|
6
|
+
export declare function shellDownloadNote(command: string): string | null;
|
|
7
|
+
export declare function pipError(command: string, sandbox?: boolean): string | null;
|
|
8
|
+
export declare function startsLikePython(command: string): boolean;
|
|
9
|
+
export declare function isPythonSourceShaped(command: string): boolean;
|
|
10
|
+
export declare function pythonRedirectError(command: string, sandbox?: boolean): string | null;
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
import { pipInstallTargets, pipMessage, unwrapPythonSource, withoutProgramDir } from './adapters/python/python-adapter.js';
|
|
2
|
+
import { bareBin, firstCommandToken, sanitizeCommand } from './scope.js';
|
|
3
|
+
import { shellArgv } from './shell-argv.js';
|
|
4
|
+
import { downloaderShellError, isTranscoder, TRANSCODE_ERROR, VIDEO_TOOLS_ERROR, videoTool } from './video-tools.js';
|
|
5
|
+
const INTERPRETER = /^(python[\d.]*|py)$/;
|
|
6
|
+
const PIP = /^pip[\d.]*$/;
|
|
7
|
+
export const PYTHON_BIN_ERROR = 'python is not a shell command here. Call exec with language "python" and command set to the ' +
|
|
8
|
+
'Python source itself: drop "python -c" and the quotes around the code. PyPI names go in packages.';
|
|
9
|
+
export const PACKAGES_NEED_PYTHON_ERROR = 'packages only work with language "python". Call exec with language "python", the same packages ' +
|
|
10
|
+
'and the Python source in command.';
|
|
11
|
+
export const FILES_NEED_PYTHON_ERROR = 'inputs and outputs only work with language "python". Call exec with language "python" and the ' +
|
|
12
|
+
'Python source in command.';
|
|
13
|
+
function commandBin(command) {
|
|
14
|
+
return bareBin(firstCommandToken(command).replace(/^.*[/\\]/, '')) ?? '';
|
|
15
|
+
}
|
|
16
|
+
function dashModule(command) {
|
|
17
|
+
return /^\s*(?:python[\d.]*|py)\s+-m\s+([\w.-]+)/.exec(withoutProgramDir(command))?.[1] ?? null;
|
|
18
|
+
}
|
|
19
|
+
function importedModules(source) {
|
|
20
|
+
const modules = [];
|
|
21
|
+
for (const line of source.split('\n')) {
|
|
22
|
+
const from = /^\s*from\s+([\w.]+)\s+import\b/.exec(line)?.[1];
|
|
23
|
+
if (from) {
|
|
24
|
+
modules.push(from);
|
|
25
|
+
continue;
|
|
26
|
+
}
|
|
27
|
+
const imported = /^\s*import\s+([\w.]+(?:\s*,\s*[\w.]+)*)/.exec(line)?.[1];
|
|
28
|
+
if (imported)
|
|
29
|
+
modules.push(...imported.split(',').map((m) => m.trim()));
|
|
30
|
+
}
|
|
31
|
+
return modules.map((m) => m.split('.')[0]);
|
|
32
|
+
}
|
|
33
|
+
export function videoToolError(command, packages, python, network = false) {
|
|
34
|
+
const names = [
|
|
35
|
+
dashModule(command) ?? '',
|
|
36
|
+
...(pipInstallTargets(command) ?? []),
|
|
37
|
+
...(packages ?? []),
|
|
38
|
+
...(python ? importedModules(unwrapPythonSource(command) ?? command) : [commandBin(command)])
|
|
39
|
+
];
|
|
40
|
+
const tools = names.map(videoTool).filter((tool) => tool !== null);
|
|
41
|
+
if (tools.length === 0)
|
|
42
|
+
return null;
|
|
43
|
+
if (!network)
|
|
44
|
+
return VIDEO_TOOLS_ERROR;
|
|
45
|
+
if (tools.some(isTranscoder))
|
|
46
|
+
return TRANSCODE_ERROR;
|
|
47
|
+
if (python)
|
|
48
|
+
return null;
|
|
49
|
+
const program = videoTool(commandBin(command)) ?? videoTool(dashModule(command) ?? '');
|
|
50
|
+
return program ? downloaderShellError(program) : null;
|
|
51
|
+
}
|
|
52
|
+
export const SHELL_DOWNLOAD_NOTE = "The file was saved on this device's disk, not in the chat, so the user cannot see it. To " +
|
|
53
|
+
'give the user a downloaded file, fetch it in exec with language "python" and declare it in outputs.';
|
|
54
|
+
const CURL_VALUED = new Set('AbcCdDeEFHKmoPQrTuUwxXyYz'.split(''));
|
|
55
|
+
const WGET_VALUED = new Set('aABDeIilOoPQRtTUwX'.split(''));
|
|
56
|
+
function writesFile(words, curl) {
|
|
57
|
+
const valued = curl ? CURL_VALUED : WGET_VALUED;
|
|
58
|
+
const output = curl ? 'o' : 'O';
|
|
59
|
+
const long = curl ? '--output' : '--output-document';
|
|
60
|
+
let saves = !curl;
|
|
61
|
+
for (let i = 1; i < words.length; i++) {
|
|
62
|
+
const word = words[i];
|
|
63
|
+
if (curl && /^--remote-name(-all)?$/.test(word))
|
|
64
|
+
saves = true;
|
|
65
|
+
if (word === long || word.startsWith(`${long}=`)) {
|
|
66
|
+
const value = word === long ? words[++i] : word.slice(long.length + 1);
|
|
67
|
+
saves = value !== '-';
|
|
68
|
+
continue;
|
|
69
|
+
}
|
|
70
|
+
if (!/^-[A-Za-z]/.test(word))
|
|
71
|
+
continue;
|
|
72
|
+
for (let j = 1; j < word.length; j++) {
|
|
73
|
+
const flag = word[j];
|
|
74
|
+
if (curl && flag === 'O')
|
|
75
|
+
saves = true;
|
|
76
|
+
if (!valued.has(flag))
|
|
77
|
+
continue;
|
|
78
|
+
const value = j + 1 < word.length ? word.slice(j + 1) : words[++i];
|
|
79
|
+
if (flag === output)
|
|
80
|
+
saves = value !== '-';
|
|
81
|
+
break;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
return saves;
|
|
85
|
+
}
|
|
86
|
+
export function shellDownloadNote(command) {
|
|
87
|
+
const bin = commandBin(command);
|
|
88
|
+
if (bin !== 'curl' && bin !== 'wget')
|
|
89
|
+
return null;
|
|
90
|
+
return writesFile(shellArgv(command), bin === 'curl') ? SHELL_DOWNLOAD_NOTE : null;
|
|
91
|
+
}
|
|
92
|
+
export function pipError(command, sandbox = false) {
|
|
93
|
+
if (!PIP.test(commandBin(command)) && dashModule(command) !== 'pip')
|
|
94
|
+
return null;
|
|
95
|
+
return pipMessage(pipInstallTargets(command), sandbox);
|
|
96
|
+
}
|
|
97
|
+
function firstCodeLine(command) {
|
|
98
|
+
return (command
|
|
99
|
+
.split('\n')
|
|
100
|
+
.find((line) => line.trim() && !line.trim().startsWith('#'))
|
|
101
|
+
?.trim() ?? '');
|
|
102
|
+
}
|
|
103
|
+
const PYTHON_START = /^(from\s+[\w.]+\s+import\b|import\s+[\w.]+\s*(,|;|$|\sas\s)|def\s+\w+\s*\(|class\s+\w+|@[\w.]+|(for|while|if|with|try|async)\b[^\n]*:$|"""|\'\'\'|[A-Za-z_][\w.]*(\[[^\]]*\])?\s+=\s+)/;
|
|
104
|
+
export function startsLikePython(command) {
|
|
105
|
+
return PYTHON_START.test(firstCodeLine(command));
|
|
106
|
+
}
|
|
107
|
+
export function isPythonSourceShaped(command) {
|
|
108
|
+
const first = firstCodeLine(command);
|
|
109
|
+
if (/^from\s+[\w.]+\s+import\b/.test(first))
|
|
110
|
+
return true;
|
|
111
|
+
if (/^[A-Za-z_][\w.]*\s*\(/.test(first) && !/^[A-Za-z_]\w*\s*\(\s*\)\s*\{/.test(first)) {
|
|
112
|
+
return true;
|
|
113
|
+
}
|
|
114
|
+
if (!PYTHON_START.test(first))
|
|
115
|
+
return false;
|
|
116
|
+
return command.trim().includes('\n') || sanitizeCommand(command) !== null;
|
|
117
|
+
}
|
|
118
|
+
export function pythonRedirectError(command, sandbox = false) {
|
|
119
|
+
const pip = pipError(command, sandbox);
|
|
120
|
+
if (pip)
|
|
121
|
+
return pip;
|
|
122
|
+
return INTERPRETER.test(commandBin(command)) ? PYTHON_BIN_ERROR : null;
|
|
123
|
+
}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
export declare const VIDEO_TOOLS_ERROR: string;
|
|
2
|
+
export declare const TRANSCODE_ERROR: string;
|
|
3
|
+
export declare function downloaderShellError(tool: string): string;
|
|
4
|
+
export declare function videoTool(name: string): string | null;
|
|
5
|
+
export declare function isTranscoder(tool: string): boolean;
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
const TRANSCODERS = new Set([
|
|
2
|
+
'ffmpeg',
|
|
3
|
+
'ffprobe',
|
|
4
|
+
'ffmpeg-python',
|
|
5
|
+
'imageio-ffmpeg',
|
|
6
|
+
'moviepy',
|
|
7
|
+
'avconv',
|
|
8
|
+
'ytdl'
|
|
9
|
+
]);
|
|
10
|
+
const DOWNLOADERS = new Set([
|
|
11
|
+
'yt-dlp',
|
|
12
|
+
'youtube-dl',
|
|
13
|
+
'you-get',
|
|
14
|
+
'streamlink',
|
|
15
|
+
'pytube',
|
|
16
|
+
'pytubefix'
|
|
17
|
+
]);
|
|
18
|
+
export const VIDEO_TOOLS_ERROR = 'downloading or transcoding video is not possible on this device: yt-dlp, youtube-dl, ffmpeg ' +
|
|
19
|
+
'and similar tools cannot run here, on any path. Do not retry; tell the user it cannot be done.';
|
|
20
|
+
export const TRANSCODE_ERROR = 'ffmpeg is not available on this device: ffmpeg, ffprobe and the packages that run them ' +
|
|
21
|
+
'(moviepy, ffmpeg-python) cannot run here, on any path, so nothing can merge or convert ' +
|
|
22
|
+
'video. Fetch a single file that needs neither, or tell the user it cannot be done.';
|
|
23
|
+
export function downloaderShellError(tool) {
|
|
24
|
+
return (`${tool} is not a program here: it is a Python package. Call exec with language "python", ` +
|
|
25
|
+
`packages: ["${tool}"] and Python source that imports it, and declare the file it saves ` +
|
|
26
|
+
'in outputs so it reaches the user.');
|
|
27
|
+
}
|
|
28
|
+
export function videoTool(name) {
|
|
29
|
+
const bare = name
|
|
30
|
+
.trim()
|
|
31
|
+
.replace(/^['"]|['"]$/g, '')
|
|
32
|
+
.split(/[=<>!~[\s]/)[0]
|
|
33
|
+
.toLowerCase()
|
|
34
|
+
.replace(/\.exe$/, '')
|
|
35
|
+
.replace(/[-_.]+/g, '-');
|
|
36
|
+
return TRANSCODERS.has(bare) || DOWNLOADERS.has(bare) ? bare : null;
|
|
37
|
+
}
|
|
38
|
+
export function isTranscoder(tool) {
|
|
39
|
+
return TRANSCODERS.has(tool);
|
|
40
|
+
}
|
|
@@ -10,6 +10,7 @@ export interface ExecToolOptions {
|
|
|
10
10
|
readonly killProcess?: KillProcess;
|
|
11
11
|
readonly loadPyodide?: LoadPyodideFn;
|
|
12
12
|
readonly disablePyodide?: boolean;
|
|
13
|
+
readonly pythonNetwork?: boolean;
|
|
13
14
|
readonly credentialStore?: CredentialStoreFn;
|
|
14
15
|
readonly skillSettings?: SkillSettingsFn;
|
|
15
16
|
readonly realpath?: Realpath;
|
|
@@ -26,4 +27,4 @@ export declare function resolveScriptRun(rawArgs: Record<string, unknown>, skill
|
|
|
26
27
|
} | {
|
|
27
28
|
error: string;
|
|
28
29
|
} | null;
|
|
29
|
-
export declare function createExecTool({ spawn, killProcess, loadPyodide, disablePyodide, credentialStore, skillSettings, realpath, deniedRoots, skillsDir }?: ExecToolOptions): ToolDefinition;
|
|
30
|
+
export declare function createExecTool({ spawn, killProcess, loadPyodide, disablePyodide, pythonNetwork, credentialStore, skillSettings, realpath, deniedRoots, skillsDir }?: ExecToolOptions): ToolDefinition;
|