@qvac/harness 0.3.0 → 0.3.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/CHANGELOG.md +262 -0
  2. package/dist/harness.bundle +15 -11
  3. package/dist-lib/index.d.ts +1 -1
  4. package/dist-lib/lib/capabilities/completion.d.ts +5 -1
  5. package/dist-lib/lib/capabilities/completion.js +20 -8
  6. package/dist-lib/lib/helpers/tool-call-markup.js +15 -7
  7. package/dist-lib/lib/no-redirect-fetch-impl.d.ts +3 -2
  8. package/dist-lib/lib/no-redirect-fetch-impl.js +4 -1
  9. package/dist-lib/lib/tools/exec/adapters/adapter.d.ts +5 -1
  10. package/dist-lib/lib/tools/exec/adapters/python/distributions.d.ts +1 -0
  11. package/dist-lib/lib/tools/exec/adapters/python/distributions.js +4 -0
  12. package/dist-lib/lib/tools/exec/adapters/python/guarded-fetch.d.ts +72 -0
  13. package/dist-lib/lib/tools/exec/adapters/python/guarded-fetch.js +222 -0
  14. package/dist-lib/lib/tools/exec/adapters/python/interrupt-timer.d.ts +5 -0
  15. package/dist-lib/lib/tools/exec/adapters/python/interrupt-timer.js +29 -0
  16. package/dist-lib/lib/tools/exec/adapters/python/loader.js +2 -1
  17. package/dist-lib/lib/tools/exec/adapters/python/module-distributions.d.ts +1 -0
  18. package/dist-lib/lib/tools/exec/adapters/python/module-distributions.js +52 -0
  19. package/dist-lib/lib/tools/exec/adapters/python/named-hosts.d.ts +2 -0
  20. package/dist-lib/lib/tools/exec/adapters/python/named-hosts.js +15 -0
  21. package/dist-lib/lib/tools/exec/adapters/python/net-shim.d.ts +2 -0
  22. package/dist-lib/lib/tools/exec/adapters/python/net-shim.js +148 -0
  23. package/dist-lib/lib/tools/exec/adapters/python/output-policy.js +13 -1
  24. package/dist-lib/lib/tools/exec/adapters/python/pyodide.d.ts +4 -1
  25. package/dist-lib/lib/tools/exec/adapters/python/pyodide.js +42 -7
  26. package/dist-lib/lib/tools/exec/adapters/python/python-adapter.d.ts +9 -3
  27. package/dist-lib/lib/tools/exec/adapters/python/python-adapter.js +137 -21
  28. package/dist-lib/lib/tools/exec/adapters/python/run-session.d.ts +5 -0
  29. package/dist-lib/lib/tools/exec/adapters/python/run-session.js +33 -8
  30. package/dist-lib/lib/tools/exec/adapters/python/wheels.d.ts +6 -0
  31. package/dist-lib/lib/tools/exec/adapters/python/wheels.js +17 -3
  32. package/dist-lib/lib/tools/exec/adapters/python/worker/client.d.ts +2 -1
  33. package/dist-lib/lib/tools/exec/adapters/python/worker/client.js +16 -3
  34. package/dist-lib/lib/tools/exec/adapters/python/worker/entry.js +15 -2
  35. package/dist-lib/lib/tools/exec/adapters/python/worker/protocol.d.ts +6 -0
  36. package/dist-lib/lib/tools/exec/adapters/python/worker/protocol.js +6 -2
  37. package/dist-lib/lib/tools/exec/adapters/python/worker/spawn-bare.d.ts +1 -0
  38. package/dist-lib/lib/tools/exec/adapters/python/worker/spawn-bare.js +8 -0
  39. package/dist-lib/lib/tools/exec/adapters/python/worker/spawn-node.d.ts +1 -0
  40. package/dist-lib/lib/tools/exec/adapters/python/worker/spawn-node.js +8 -0
  41. package/dist-lib/lib/tools/exec/adapters/shell/shell-adapter.js +7 -1
  42. package/dist-lib/lib/tools/exec/shell-guidance.d.ts +10 -0
  43. package/dist-lib/lib/tools/exec/shell-guidance.js +123 -0
  44. package/dist-lib/lib/tools/exec/video-tools.d.ts +5 -0
  45. package/dist-lib/lib/tools/exec/video-tools.js +40 -0
  46. package/dist-lib/lib/tools/exec.d.ts +2 -1
  47. package/dist-lib/lib/tools/exec.js +93 -33
  48. package/dist-lib/lib/tools/http-request/truncate-response.d.ts +1 -0
  49. package/dist-lib/lib/tools/http-request/truncate-response.js +1 -1
  50. package/dist-lib/lib/tools/http-request.js +4 -1
  51. package/dist-lib/lib/tools/operations/builders.d.ts +1 -1
  52. package/dist-lib/lib/tools/operations/builders.js +9 -0
  53. package/dist-lib/lib/tools/operations/compile.js +41 -18
  54. package/dist-lib/lib/tools/operations/decoders.d.ts +6 -0
  55. package/dist-lib/lib/tools/operations/decoders.js +82 -0
  56. package/dist-lib/lib/tools/operations/manifest.d.ts +16 -2
  57. package/dist-lib/lib/tools/operations/manifest.js +6 -1
  58. package/dist-lib/lib/tools/skill.d.ts +3 -0
  59. package/dist-lib/lib/tools/skill.js +38 -5
  60. package/dist-lib/lib/tools/skills/manifest-parse.js +43 -1
  61. package/dist-lib/lib/tools/skills/manifest.d.ts +13 -0
  62. package/dist-lib/schema/harness.js +20 -1
  63. package/dist-lib/spec/hrpc/messages.js +93 -5
  64. package/dist-lib/spec/hyperschema/index.js +93 -5
  65. package/dist-lib/spec/hyperschema/schema.json +66 -1
  66. package/dist-lib/spec/hyperschema/types.d.ts +18 -1
  67. package/package.json +40 -2
  68. package/wheels/manifest.json +15 -0
@@ -2,6 +2,8 @@ import { AbortedError } from '../../../../helpers/race-abort.js';
2
2
  import { hlog } from '../../../../log.js';
3
3
  import { TRUNCATED_OUTPUT_SUFFIX } from '../../runner.js';
4
4
  import { collectOutputs, openScratch, stageInputFiles } from './memfs.js';
5
+ import { isGuardedFetch, setConnectPolicy } from './guarded-fetch.js';
6
+ import { NET_SHIM, pythonReachesNetwork } from './net-shim.js';
5
7
  import { planVendoredInstall, stageVendoredWheels } from './wheels.js';
6
8
  export function cap(text, limit) {
7
9
  if (text.length <= limit)
@@ -37,9 +39,30 @@ export function repairDoubleEscapedSource(code, message) {
37
39
  const repaired = unescapeSource(code);
38
40
  return repaired === code ? null : repaired;
39
41
  }
40
- function attempt(py, code, output, maxOutputChars) {
41
- try {
42
+ const promisingEval = new WeakMap();
43
+ async function enter(py, code, access) {
44
+ const guarded = pythonReachesNetwork() && isGuardedFetch(globalThis.fetch);
45
+ if (!access || !guarded || py.ffi === undefined || py.pyimport === undefined) {
42
46
  py.runPython(code);
47
+ return;
48
+ }
49
+ let evalCode = promisingEval.get(py);
50
+ if (!evalCode) {
51
+ py.runPython(NET_SHIM);
52
+ evalCode = py.pyimport('pyodide.code').eval_code;
53
+ promisingEval.set(py, evalCode);
54
+ }
55
+ setConnectPolicy(access.policy);
56
+ try {
57
+ await evalCode.callPromising(code, py.globals);
58
+ }
59
+ finally {
60
+ setConnectPolicy(null);
61
+ }
62
+ }
63
+ async function attempt(py, code, output, maxOutputChars, access) {
64
+ try {
65
+ await enter(py, code, access);
43
66
  return {
44
67
  result: {
45
68
  stdout: cap(output.stdout, maxOutputChars),
@@ -63,14 +86,14 @@ function attempt(py, code, output, maxOutputChars) {
63
86
  };
64
87
  }
65
88
  }
66
- function runOnce(py, code, output, maxOutputChars) {
67
- const first = attempt(py, code, output, maxOutputChars);
89
+ async function runOnce(py, code, output, maxOutputChars, access) {
90
+ const first = await attempt(py, code, output, maxOutputChars, access);
68
91
  if (first.error === null)
69
92
  return first;
70
93
  const repaired = repairDoubleEscapedSource(code, first.error);
71
94
  if (repaired === null)
72
95
  return first;
73
- const retry = attempt(py, repaired, output, maxOutputChars);
96
+ const retry = await attempt(py, repaired, output, maxOutputChars, access);
74
97
  if (retry.error !== null && retry.error.includes('SyntaxError'))
75
98
  return first;
76
99
  return retry;
@@ -168,7 +191,7 @@ async function installPackages(py, packages, timeoutMs, isCancelled) {
168
191
  clearTimeout(timer);
169
192
  }
170
193
  }
171
- const DELIVERABLE = /\.(pptx|docx|xlsx|pdf|png|jpe?g|gif|webp|csv|json|txt|md|zip)$/i;
194
+ const DELIVERABLE = /\.(pptx|docx|xlsx|pdf|png|jpe?g|gif|webp|csv|json|txt|md|zip|mp4|webm|m4a|mp3)$/i;
172
195
  function listFiles(py, dir) {
173
196
  try {
174
197
  return new Set(py.FS.readdir(dir).filter((name) => name !== '.' && name !== '..'));
@@ -196,6 +219,8 @@ function fitDecks(py, outputs) {
196
219
  }
197
220
  export async function performRun(py, spec, opts = {}) {
198
221
  const isCancelled = opts.isCancelled ?? (() => false);
222
+ const ask = opts.askConnect ?? (async () => false);
223
+ const access = spec.network === true ? { policy: spec.hosts ? { named: spec.hosts, ask } : null } : null;
199
224
  const output = { stdout: '', stderr: '' };
200
225
  py.setStdout({ batched: (chunk) => (output.stdout += chunk + '\n') });
201
226
  py.setStderr({ batched: (chunk) => (output.stderr += chunk + '\n') });
@@ -234,7 +259,7 @@ export async function performRun(py, spec, opts = {}) {
234
259
  if (!spec.inputs?.length && !spec.outputs?.length && !spec.workdir) {
235
260
  const cwd = py.FS.cwd();
236
261
  const before = listFiles(py, cwd);
237
- const { result, fatal } = runOnce(py, spec.code, output, spec.maxOutputChars);
262
+ const { result, fatal } = await runOnce(py, spec.code, output, spec.maxOutputChars, access);
238
263
  if (result.exitCode !== 0)
239
264
  return { result, evict: fatal };
240
265
  const undeclared = newDeliverables(before, listFiles(py, cwd));
@@ -245,7 +270,7 @@ export async function performRun(py, spec, opts = {}) {
245
270
  py.FS.chdir(scratch.cwd);
246
271
  stageInputFiles(py.FS, scratch.cwd, spec.inputs ?? []);
247
272
  const before = listFiles(py, scratch.cwd);
248
- const { result, fatal } = runOnce(py, spec.code, output, spec.maxOutputChars);
273
+ const { result, fatal } = await runOnce(py, spec.code, output, spec.maxOutputChars, access);
249
274
  let evict = fatal;
250
275
  if (result.exitCode === 0)
251
276
  evict = fitDecks(py, spec.outputs ?? []);
@@ -3,6 +3,7 @@ declare const ManifestSchema: z.ZodObject<{
3
3
  pyodide: z.ZodString;
4
4
  packages: z.ZodRecord<z.ZodString, z.ZodObject<{
5
5
  version: z.ZodOptional<z.ZodString>;
6
+ imports: z.ZodOptional<z.ZodArray<z.ZodString>>;
6
7
  pypi: z.ZodArray<z.ZodString>;
7
8
  pyodide: z.ZodArray<z.ZodString>;
8
9
  }, z.core.$strip>>;
@@ -29,6 +30,11 @@ type WheelFs = {
29
30
  };
30
31
  export declare function wheelsDir(): string | undefined;
31
32
  export declare function vendoredManifest(dir?: string | undefined): WheelManifest | null;
33
+ export declare function normalizeDistribution(name: string): string;
34
+ export declare function vendoredSpec(distribution: string, opts?: {
35
+ readonly dir?: string | undefined;
36
+ readonly manifest?: WheelManifest | null;
37
+ }): string | null;
32
38
  export declare function planVendoredInstall(specs: readonly string[], opts?: {
33
39
  readonly dir?: string | undefined;
34
40
  readonly manifest?: WheelManifest | null;
@@ -7,6 +7,7 @@ const ManifestSchema = z.object({
7
7
  pyodide: z.string(),
8
8
  packages: z.record(z.string(), z.object({
9
9
  version: z.string().optional(),
10
+ imports: z.array(z.string()).optional(),
10
11
  pypi: z.array(z.string()),
11
12
  pyodide: z.array(z.string())
12
13
  })),
@@ -42,12 +43,25 @@ export function vendoredManifest(dir = wheelsDir()) {
42
43
  return null;
43
44
  }
44
45
  }
45
- function normalize(name) {
46
- return name.toLowerCase().replace(/[_.]+/g, '-');
46
+ export function normalizeDistribution(name) {
47
+ return name
48
+ .trim()
49
+ .toLowerCase()
50
+ .replace(/[-_.]+/g, '-');
47
51
  }
48
52
  function parseSpec(spec) {
49
53
  const [name, version] = spec.split('==');
50
- return { name: normalize((name ?? '').trim()), version: version?.trim() };
54
+ return { name: normalizeDistribution(name ?? ''), version: version?.trim() };
55
+ }
56
+ export function vendoredSpec(distribution, opts = {}) {
57
+ const name = normalizeDistribution(distribution);
58
+ const dir = opts.dir ?? wheelsDir();
59
+ const manifest = opts.manifest === undefined ? vendoredManifest(dir) : opts.manifest;
60
+ const entry = manifest?.packages[name];
61
+ if (!entry)
62
+ return null;
63
+ const spec = entry.version ? `${name}==${entry.version}` : name;
64
+ return planVendoredInstall([spec], { dir, manifest }) ? spec : null;
51
65
  }
52
66
  export function planVendoredInstall(specs, opts = {}) {
53
67
  const dir = opts.dir ?? wheelsDir();
@@ -1,7 +1,8 @@
1
- import type { AbortSignal as BareAbortSignal } from 'bare-abort-controller';
1
+ import { type AbortSignal as BareAbortSignal } from 'bare-abort-controller';
2
2
  import type { PythonRunResult, RunSpec } from '../run-session.ts';
3
3
  export declare function runPythonInWorker(spec: RunSpec, opts: {
4
4
  readonly timeoutMs: number;
5
5
  readonly signal?: BareAbortSignal | undefined;
6
+ readonly askConnect?: ((host: string) => Promise<boolean>) | undefined;
6
7
  }): Promise<PythonRunResult>;
7
8
  export declare function closePythonWorker(): Promise<void>;
@@ -1,10 +1,12 @@
1
1
  import { spawnPythonWorker } from '#pyodide-worker-spawn';
2
+ import { AbortController } from 'bare-abort-controller';
2
3
  import process from 'process';
3
4
  import { fileURLToPath } from 'url';
4
5
  import { AbortedError, raceAbort } from '../../../../../helpers/race-abort.js';
5
6
  import { harnessLogEnabled, hlog } from '../../../../../log.js';
6
7
  import { asWorkerMessage, fromWireResult } from './protocol.js';
7
8
  const SIGINT = 2;
9
+ const RESEND_MS = 100;
8
10
  function interruptGraceMs() {
9
11
  const ms = Number(process.env.QVAC_PYODIDE_INTERRUPT_GRACE_MS);
10
12
  return ms > 0 ? ms : 30_000;
@@ -69,8 +71,13 @@ async function drainAndReap(worker) {
69
71
  timer.clear();
70
72
  }
71
73
  }
72
- async function resultFor(worker, id) {
73
- const message = await readMessage(worker.link);
74
+ async function resultFor(worker, id, askConnect) {
75
+ let message = await readMessage(worker.link);
76
+ while (message?.type === 'connect-ask') {
77
+ const allowed = askConnect ? await askConnect(message.host).catch(() => false) : false;
78
+ await worker.link.write({ type: 'connect-answer', allowed });
79
+ message = await readMessage(worker.link);
80
+ }
74
81
  if (message?.type === 'result') {
75
82
  if (message.id !== id)
76
83
  throw new Error('python worker answered a different run');
@@ -89,6 +96,7 @@ function grace(ms) {
89
96
  }
90
97
  async function interrupt(worker, outcome) {
91
98
  Atomics.store(worker.view, 0, SIGINT);
99
+ const resend = setInterval(() => Atomics.store(worker.view, 0, SIGINT), RESEND_MS);
92
100
  const graceMs = interruptGraceMs();
93
101
  const timer = grace(graceMs);
94
102
  try {
@@ -105,6 +113,7 @@ async function interrupt(worker, outcome) {
105
113
  }
106
114
  }
107
115
  finally {
116
+ clearInterval(resend);
108
117
  timer.clear();
109
118
  }
110
119
  }
@@ -116,13 +125,15 @@ export async function runPythonInWorker(spec, opts) {
116
125
  Atomics.store(worker.view, 0, 0);
117
126
  let cancelled = false;
118
127
  let sent = false;
128
+ const stop = new AbortController();
129
+ const ask = opts.askConnect;
119
130
  const outcome = (async () => {
120
131
  await worker.ready;
121
132
  if (cancelled)
122
133
  throw new AbortedError();
123
134
  sent = true;
124
135
  await worker.link.write({ type: 'run', id, spec });
125
- return resultFor(worker, id);
136
+ return resultFor(worker, id, ask && ((host) => raceAbort(ask(host), stop.signal)));
126
137
  })();
127
138
  let timer;
128
139
  const timeout = new Promise((resolve) => {
@@ -135,6 +146,7 @@ export async function runPythonInWorker(spec, opts) {
135
146
  catch (error) {
136
147
  if (error instanceof AbortedError) {
137
148
  cancelled = true;
149
+ stop.abort(undefined);
138
150
  if (sent)
139
151
  await interrupt(worker, outcome);
140
152
  else
@@ -150,6 +162,7 @@ export async function runPythonInWorker(spec, opts) {
150
162
  }
151
163
  if (first === 'timeout') {
152
164
  cancelled = true;
165
+ stop.abort(undefined);
153
166
  if (sent)
154
167
  await interrupt(worker, outcome);
155
168
  else
@@ -90,9 +90,22 @@ try {
90
90
  const message = asHostMessage(await port.read());
91
91
  if (!message || message.type === 'close')
92
92
  break;
93
- const { result, evict } = await performRun(py, message.spec);
93
+ if (message.type === 'connect-answer')
94
+ continue;
95
+ let closing = false;
96
+ const askConnect = async (host) => {
97
+ if (closing)
98
+ return false;
99
+ port.write({ type: 'connect-ask', host });
100
+ const answer = asHostMessage(await port.read());
101
+ if (answer?.type === 'connect-answer')
102
+ return answer.allowed;
103
+ closing = true;
104
+ return false;
105
+ };
106
+ const { result, evict } = await performRun(py, message.spec, { askConnect });
94
107
  port.write({ type: 'result', id: message.id, result: toWireResult(result), evict });
95
- if (evict)
108
+ if (evict || closing)
96
109
  break;
97
110
  }
98
111
  }
@@ -5,6 +5,9 @@ export type HostMessage = {
5
5
  readonly spec: RunSpec;
6
6
  } | {
7
7
  readonly type: 'close';
8
+ } | {
9
+ readonly type: 'connect-answer';
10
+ readonly allowed: boolean;
8
11
  };
9
12
  export type WireResult = {
10
13
  readonly stdout: string;
@@ -31,6 +34,9 @@ export type WorkerMessage = {
31
34
  readonly id: number;
32
35
  readonly result: WireResult;
33
36
  readonly evict: boolean;
37
+ } | {
38
+ readonly type: 'connect-ask';
39
+ readonly host: string;
34
40
  };
35
41
  export declare function toWireResult(result: PythonRunResult): WireResult;
36
42
  export declare function fromWireResult(wire: WireResult): PythonRunResult;
@@ -28,13 +28,17 @@ function messageType(value) {
28
28
  }
29
29
  export function asHostMessage(value) {
30
30
  const type = messageType(value);
31
- if (type === 'run' || type === 'close')
31
+ if (type === 'run' || type === 'close' || type === 'connect-answer')
32
32
  return value;
33
33
  return null;
34
34
  }
35
35
  export function asWorkerMessage(value) {
36
36
  const type = messageType(value);
37
- if (type === 'log' || type === 'ready' || type === 'boot-error' || type === 'result') {
37
+ if (type === 'log' ||
38
+ type === 'ready' ||
39
+ type === 'boot-error' ||
40
+ type === 'result' ||
41
+ type === 'connect-ask') {
38
42
  return value;
39
43
  }
40
44
  return null;
@@ -1,2 +1,3 @@
1
1
  import type { WorkerData, WorkerLink } from './spawn-node.ts';
2
+ export declare function spawnInlineThread(source: string, data: object): () => void;
2
3
  export declare function spawnPythonWorker(entry: string, data: WorkerData): WorkerLink;
@@ -1,5 +1,13 @@
1
1
  import Channel from 'bare-channel';
2
2
  import safetyCatch from 'safety-catch';
3
+ export function spawnInlineThread(source, data) {
4
+ const Thread = Bare.Thread;
5
+ const thread = new Thread('inline-thread.js', {
6
+ data,
7
+ source: `const data = Bare.Thread.self.data\n${source}`
8
+ });
9
+ return () => thread.join();
10
+ }
3
11
  export function spawnPythonWorker(entry, data) {
4
12
  const channel = new Channel();
5
13
  const thread = new Bare.Thread(entry, { data: { handle: channel.handle, ...data } });
@@ -11,3 +11,4 @@ export type WorkerData = {
11
11
  readonly log: boolean;
12
12
  };
13
13
  export declare function spawnPythonWorker(entry: string, data: WorkerData): WorkerLink;
14
+ export declare function spawnInlineThread(source: string, data: object): () => void;
@@ -43,3 +43,11 @@ export function spawnPythonWorker(entry, data) {
43
43
  unref: () => worker.unref()
44
44
  };
45
45
  }
46
+ export function spawnInlineThread(source, data) {
47
+ const worker = new Worker(`const data = require('worker_threads').workerData\n${source}`, {
48
+ eval: true,
49
+ workerData: data
50
+ });
51
+ worker.unref();
52
+ return () => { };
53
+ }
@@ -1,4 +1,5 @@
1
1
  import { runCommand } from '../../runner.js';
2
+ import { shellDownloadNote } from '../../shell-guidance.js';
2
3
  export function createShellAdapter(options) {
3
4
  return {
4
5
  resolve(args) {
@@ -8,11 +9,16 @@ export function createShellAdapter(options) {
8
9
  const runtime = options.spawn
9
10
  ? { spawn: options.spawn, killProcess: options.killProcess }
10
11
  : await options.loadRuntime();
11
- return runCommand(command, {
12
+ const result = await runCommand(command, {
12
13
  ...execution,
13
14
  spawn: runtime.spawn,
14
15
  killProcess: runtime.killProcess
15
16
  });
17
+ const note = result.exitCode === 0 ? shellDownloadNote(command) : null;
18
+ if (!note)
19
+ return result;
20
+ const stdout = `${result.stdout}${result.stdout && !result.stdout.endsWith('\n') ? '\n' : ''}${note}`;
21
+ return { ...result, stdout };
16
22
  }
17
23
  };
18
24
  }
@@ -0,0 +1,10 @@
1
+ export declare const PYTHON_BIN_ERROR: string;
2
+ export declare const PACKAGES_NEED_PYTHON_ERROR: string;
3
+ export declare const FILES_NEED_PYTHON_ERROR: string;
4
+ export declare function videoToolError(command: string, packages: readonly string[] | undefined, python: boolean, network?: boolean): string | null;
5
+ export declare const SHELL_DOWNLOAD_NOTE: string;
6
+ export declare function shellDownloadNote(command: string): string | null;
7
+ export declare function pipError(command: string, sandbox?: boolean): string | null;
8
+ export declare function startsLikePython(command: string): boolean;
9
+ export declare function isPythonSourceShaped(command: string): boolean;
10
+ export declare function pythonRedirectError(command: string, sandbox?: boolean): string | null;
@@ -0,0 +1,123 @@
1
+ import { pipInstallTargets, pipMessage, unwrapPythonSource, withoutProgramDir } from './adapters/python/python-adapter.js';
2
+ import { bareBin, firstCommandToken, sanitizeCommand } from './scope.js';
3
+ import { shellArgv } from './shell-argv.js';
4
+ import { downloaderShellError, isTranscoder, TRANSCODE_ERROR, VIDEO_TOOLS_ERROR, videoTool } from './video-tools.js';
5
+ const INTERPRETER = /^(python[\d.]*|py)$/;
6
+ const PIP = /^pip[\d.]*$/;
7
+ export const PYTHON_BIN_ERROR = 'python is not a shell command here. Call exec with language "python" and command set to the ' +
8
+ 'Python source itself: drop "python -c" and the quotes around the code. PyPI names go in packages.';
9
+ export const PACKAGES_NEED_PYTHON_ERROR = 'packages only work with language "python". Call exec with language "python", the same packages ' +
10
+ 'and the Python source in command.';
11
+ export const FILES_NEED_PYTHON_ERROR = 'inputs and outputs only work with language "python". Call exec with language "python" and the ' +
12
+ 'Python source in command.';
13
+ function commandBin(command) {
14
+ return bareBin(firstCommandToken(command).replace(/^.*[/\\]/, '')) ?? '';
15
+ }
16
+ function dashModule(command) {
17
+ return /^\s*(?:python[\d.]*|py)\s+-m\s+([\w.-]+)/.exec(withoutProgramDir(command))?.[1] ?? null;
18
+ }
19
+ function importedModules(source) {
20
+ const modules = [];
21
+ for (const line of source.split('\n')) {
22
+ const from = /^\s*from\s+([\w.]+)\s+import\b/.exec(line)?.[1];
23
+ if (from) {
24
+ modules.push(from);
25
+ continue;
26
+ }
27
+ const imported = /^\s*import\s+([\w.]+(?:\s*,\s*[\w.]+)*)/.exec(line)?.[1];
28
+ if (imported)
29
+ modules.push(...imported.split(',').map((m) => m.trim()));
30
+ }
31
+ return modules.map((m) => m.split('.')[0]);
32
+ }
33
+ export function videoToolError(command, packages, python, network = false) {
34
+ const names = [
35
+ dashModule(command) ?? '',
36
+ ...(pipInstallTargets(command) ?? []),
37
+ ...(packages ?? []),
38
+ ...(python ? importedModules(unwrapPythonSource(command) ?? command) : [commandBin(command)])
39
+ ];
40
+ const tools = names.map(videoTool).filter((tool) => tool !== null);
41
+ if (tools.length === 0)
42
+ return null;
43
+ if (!network)
44
+ return VIDEO_TOOLS_ERROR;
45
+ if (tools.some(isTranscoder))
46
+ return TRANSCODE_ERROR;
47
+ if (python)
48
+ return null;
49
+ const program = videoTool(commandBin(command)) ?? videoTool(dashModule(command) ?? '');
50
+ return program ? downloaderShellError(program) : null;
51
+ }
52
+ export const SHELL_DOWNLOAD_NOTE = "The file was saved on this device's disk, not in the chat, so the user cannot see it. To " +
53
+ 'give the user a downloaded file, fetch it in exec with language "python" and declare it in outputs.';
54
+ const CURL_VALUED = new Set('AbcCdDeEFHKmoPQrTuUwxXyYz'.split(''));
55
+ const WGET_VALUED = new Set('aABDeIilOoPQRtTUwX'.split(''));
56
+ function writesFile(words, curl) {
57
+ const valued = curl ? CURL_VALUED : WGET_VALUED;
58
+ const output = curl ? 'o' : 'O';
59
+ const long = curl ? '--output' : '--output-document';
60
+ let saves = !curl;
61
+ for (let i = 1; i < words.length; i++) {
62
+ const word = words[i];
63
+ if (curl && /^--remote-name(-all)?$/.test(word))
64
+ saves = true;
65
+ if (word === long || word.startsWith(`${long}=`)) {
66
+ const value = word === long ? words[++i] : word.slice(long.length + 1);
67
+ saves = value !== '-';
68
+ continue;
69
+ }
70
+ if (!/^-[A-Za-z]/.test(word))
71
+ continue;
72
+ for (let j = 1; j < word.length; j++) {
73
+ const flag = word[j];
74
+ if (curl && flag === 'O')
75
+ saves = true;
76
+ if (!valued.has(flag))
77
+ continue;
78
+ const value = j + 1 < word.length ? word.slice(j + 1) : words[++i];
79
+ if (flag === output)
80
+ saves = value !== '-';
81
+ break;
82
+ }
83
+ }
84
+ return saves;
85
+ }
86
+ export function shellDownloadNote(command) {
87
+ const bin = commandBin(command);
88
+ if (bin !== 'curl' && bin !== 'wget')
89
+ return null;
90
+ return writesFile(shellArgv(command), bin === 'curl') ? SHELL_DOWNLOAD_NOTE : null;
91
+ }
92
+ export function pipError(command, sandbox = false) {
93
+ if (!PIP.test(commandBin(command)) && dashModule(command) !== 'pip')
94
+ return null;
95
+ return pipMessage(pipInstallTargets(command), sandbox);
96
+ }
97
+ function firstCodeLine(command) {
98
+ return (command
99
+ .split('\n')
100
+ .find((line) => line.trim() && !line.trim().startsWith('#'))
101
+ ?.trim() ?? '');
102
+ }
103
+ const PYTHON_START = /^(from\s+[\w.]+\s+import\b|import\s+[\w.]+\s*(,|;|$|\sas\s)|def\s+\w+\s*\(|class\s+\w+|@[\w.]+|(for|while|if|with|try|async)\b[^\n]*:$|"""|\'\'\'|[A-Za-z_][\w.]*(\[[^\]]*\])?\s+=\s+)/;
104
+ export function startsLikePython(command) {
105
+ return PYTHON_START.test(firstCodeLine(command));
106
+ }
107
+ export function isPythonSourceShaped(command) {
108
+ const first = firstCodeLine(command);
109
+ if (/^from\s+[\w.]+\s+import\b/.test(first))
110
+ return true;
111
+ if (/^[A-Za-z_][\w.]*\s*\(/.test(first) && !/^[A-Za-z_]\w*\s*\(\s*\)\s*\{/.test(first)) {
112
+ return true;
113
+ }
114
+ if (!PYTHON_START.test(first))
115
+ return false;
116
+ return command.trim().includes('\n') || sanitizeCommand(command) !== null;
117
+ }
118
+ export function pythonRedirectError(command, sandbox = false) {
119
+ const pip = pipError(command, sandbox);
120
+ if (pip)
121
+ return pip;
122
+ return INTERPRETER.test(commandBin(command)) ? PYTHON_BIN_ERROR : null;
123
+ }
@@ -0,0 +1,5 @@
1
+ export declare const VIDEO_TOOLS_ERROR: string;
2
+ export declare const TRANSCODE_ERROR: string;
3
+ export declare function downloaderShellError(tool: string): string;
4
+ export declare function videoTool(name: string): string | null;
5
+ export declare function isTranscoder(tool: string): boolean;
@@ -0,0 +1,40 @@
1
+ const TRANSCODERS = new Set([
2
+ 'ffmpeg',
3
+ 'ffprobe',
4
+ 'ffmpeg-python',
5
+ 'imageio-ffmpeg',
6
+ 'moviepy',
7
+ 'avconv',
8
+ 'ytdl'
9
+ ]);
10
+ const DOWNLOADERS = new Set([
11
+ 'yt-dlp',
12
+ 'youtube-dl',
13
+ 'you-get',
14
+ 'streamlink',
15
+ 'pytube',
16
+ 'pytubefix'
17
+ ]);
18
+ export const VIDEO_TOOLS_ERROR = 'downloading or transcoding video is not possible on this device: yt-dlp, youtube-dl, ffmpeg ' +
19
+ 'and similar tools cannot run here, on any path. Do not retry; tell the user it cannot be done.';
20
+ export const TRANSCODE_ERROR = 'ffmpeg is not available on this device: ffmpeg, ffprobe and the packages that run them ' +
21
+ '(moviepy, ffmpeg-python) cannot run here, on any path, so nothing can merge or convert ' +
22
+ 'video. Fetch a single file that needs neither, or tell the user it cannot be done.';
23
+ export function downloaderShellError(tool) {
24
+ return (`${tool} is not a program here: it is a Python package. Call exec with language "python", ` +
25
+ `packages: ["${tool}"] and Python source that imports it, and declare the file it saves ` +
26
+ 'in outputs so it reaches the user.');
27
+ }
28
+ export function videoTool(name) {
29
+ const bare = name
30
+ .trim()
31
+ .replace(/^['"]|['"]$/g, '')
32
+ .split(/[=<>!~[\s]/)[0]
33
+ .toLowerCase()
34
+ .replace(/\.exe$/, '')
35
+ .replace(/[-_.]+/g, '-');
36
+ return TRANSCODERS.has(bare) || DOWNLOADERS.has(bare) ? bare : null;
37
+ }
38
+ export function isTranscoder(tool) {
39
+ return TRANSCODERS.has(tool);
40
+ }
@@ -10,6 +10,7 @@ export interface ExecToolOptions {
10
10
  readonly killProcess?: KillProcess;
11
11
  readonly loadPyodide?: LoadPyodideFn;
12
12
  readonly disablePyodide?: boolean;
13
+ readonly pythonNetwork?: boolean;
13
14
  readonly credentialStore?: CredentialStoreFn;
14
15
  readonly skillSettings?: SkillSettingsFn;
15
16
  readonly realpath?: Realpath;
@@ -26,4 +27,4 @@ export declare function resolveScriptRun(rawArgs: Record<string, unknown>, skill
26
27
  } | {
27
28
  error: string;
28
29
  } | null;
29
- export declare function createExecTool({ spawn, killProcess, loadPyodide, disablePyodide, credentialStore, skillSettings, realpath, deniedRoots, skillsDir }?: ExecToolOptions): ToolDefinition;
30
+ export declare function createExecTool({ spawn, killProcess, loadPyodide, disablePyodide, pythonNetwork, credentialStore, skillSettings, realpath, deniedRoots, skillsDir }?: ExecToolOptions): ToolDefinition;