@qvac/harness 0.6.0 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +132 -1
- package/README.md +47 -29
- package/dist-lib/index.d.ts +4 -0
- package/dist-lib/index.js +31 -19
- package/dist-lib/lib/capabilities/preview-model-source.d.ts +0 -1
- package/dist-lib/lib/connect.js +5 -2
- package/dist-lib/lib/contract-types.d.ts +1 -0
- package/dist-lib/lib/docx/index.d.ts +1 -1
- package/dist-lib/lib/docx/index.js +1 -1
- package/dist-lib/lib/hardware/capability-probe.js +9 -4
- package/dist-lib/lib/hooks/built-in/sandbox.js +0 -5
- package/dist-lib/lib/models/model-catalog-constants.d.ts +7 -0
- package/dist-lib/lib/models/model-catalog-constants.js +17 -0
- package/dist-lib/lib/models/model-catalog.d.ts +14 -6
- package/dist-lib/lib/models/model-catalog.js +14 -6
- package/dist-lib/lib/models/model-handle.d.ts +5 -1
- package/dist-lib/lib/models/model-handle.js +8 -3
- package/dist-lib/lib/models/model-list.d.ts +7 -3
- package/dist-lib/lib/models/model-list.js +9 -4
- package/dist-lib/lib/models/model-pool.d.ts +4 -1
- package/dist-lib/lib/models/model-pool.js +33 -10
- package/dist-lib/lib/models/model-task.d.ts +3 -0
- package/dist-lib/lib/models/model-task.js +14 -0
- package/dist-lib/lib/models/music-catalog.d.ts +2 -0
- package/dist-lib/lib/models/music-catalog.js +9 -0
- package/dist-lib/lib/office/index.d.ts +1 -1
- package/dist-lib/lib/office/index.js +1 -1
- package/dist-lib/lib/office/python.js +1 -2
- package/dist-lib/lib/serve.js +1 -1
- package/dist-lib/lib/tools/exec/adapters/python/deadline-state.d.ts +10 -0
- package/dist-lib/lib/tools/exec/adapters/python/deadline-state.js +5 -0
- package/dist-lib/lib/tools/exec/adapters/python/interrupt-timer.js +10 -17
- package/dist-lib/lib/tools/exec/adapters/python/loader.js +15 -11
- package/dist-lib/lib/tools/exec/adapters/python/pyodide.js +1 -15
- package/dist-lib/lib/tools/exec/adapters/python/python-adapter.d.ts +1 -0
- package/dist-lib/lib/tools/exec/adapters/python/python-adapter.js +13 -2
- package/dist-lib/lib/tools/exec/adapters/python/wheels.d.ts +3 -3
- package/dist-lib/lib/tools/exec/adapters/python/wheels.js +3 -14
- package/dist-lib/lib/tools/exec/adapters/python/worker/active.d.ts +19 -0
- package/dist-lib/lib/tools/exec/adapters/python/worker/active.js +72 -0
- package/dist-lib/lib/tools/exec/adapters/python/worker/client.d.ts +0 -1
- package/dist-lib/lib/tools/exec/adapters/python/worker/client.js +8 -76
- package/dist-lib/lib/tools/exec/adapters/python/worker/entry.js +2 -126
- package/dist-lib/lib/tools/exec/adapters/python/worker/interrupt-deadline.d.ts +1 -0
- package/dist-lib/lib/tools/exec/adapters/python/worker/interrupt-deadline.js +21 -0
- package/dist-lib/lib/tools/exec/adapters/python/worker/serve.d.ts +2 -0
- package/dist-lib/lib/tools/exec/adapters/python/worker/serve.js +90 -0
- package/dist-lib/lib/tools/exec/adapters/python/worker/{spawn-node.d.ts → spawn.d.ts} +0 -1
- package/dist-lib/lib/tools/exec/adapters/python/worker/{spawn-node.js → spawn.js} +0 -8
- package/dist-lib/lib/tools/exec/bare-runtime.d.ts +2 -0
- package/dist-lib/lib/tools/exec/bare-runtime.js +10 -0
- package/dist-lib/lib/tools/exec.d.ts +0 -2
- package/dist-lib/lib/tools/exec.js +6 -11
- package/dist-lib/schema/harness.js +4 -2
- package/dist-lib/spec/hrpc/messages.js +11 -8
- package/dist-lib/spec/hyperschema/index.js +11 -8
- package/dist-lib/spec/hyperschema/schema.json +11 -1
- package/dist-lib/spec/hyperschema/types.d.ts +3 -1
- package/package.json +39 -20
- package/dist/harness.bundle +0 -42313
- package/dist/harness.cjs +0 -18
- package/dist/harness.d.ts +0 -8
- package/dist/prebuilds-manifest.json +0 -372
- package/dist-lib/lib/tools/exec/adapters/python/worker/spawn-bare.d.ts +0 -3
- package/dist-lib/lib/tools/exec/adapters/python/worker/spawn-bare.js +0 -38
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { type WorkerMessage } from './protocol.ts';
|
|
2
|
+
import type { WorkerLink } from './spawn.ts';
|
|
3
|
+
export interface ActiveWorker {
|
|
4
|
+
readonly link: WorkerLink;
|
|
5
|
+
readonly view: Uint8Array;
|
|
6
|
+
readonly ready: Promise<void>;
|
|
7
|
+
nextId: number;
|
|
8
|
+
}
|
|
9
|
+
export declare function activeWorker(boot: () => ActiveWorker): ActiveWorker;
|
|
10
|
+
export declare function forget(worker: ActiveWorker): void;
|
|
11
|
+
export declare function destroy(worker: ActiveWorker): void;
|
|
12
|
+
export declare function readMessage(link: WorkerLink): Promise<WorkerMessage | null>;
|
|
13
|
+
export declare function drainAndReap(worker: ActiveWorker): Promise<void>;
|
|
14
|
+
export interface Grace {
|
|
15
|
+
expired: Promise<'grace'>;
|
|
16
|
+
clear: () => void;
|
|
17
|
+
}
|
|
18
|
+
export declare function grace(ms: number): Grace;
|
|
19
|
+
export declare function closePythonWorker(): Promise<void>;
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import { hlog } from '../../../../../log.js';
|
|
2
|
+
import { asWorkerMessage } from './protocol.js';
|
|
3
|
+
const SHUTDOWN_GRACE_MS = 3_000;
|
|
4
|
+
let active = null;
|
|
5
|
+
export function activeWorker(boot) {
|
|
6
|
+
return (active ??= boot());
|
|
7
|
+
}
|
|
8
|
+
export function forget(worker) {
|
|
9
|
+
if (active === worker) {
|
|
10
|
+
active = null;
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
export function destroy(worker) {
|
|
14
|
+
forget(worker);
|
|
15
|
+
worker.link.kill();
|
|
16
|
+
}
|
|
17
|
+
export async function readMessage(link) {
|
|
18
|
+
for (;;) {
|
|
19
|
+
const message = asWorkerMessage(await link.read());
|
|
20
|
+
if (message?.type !== 'log') {
|
|
21
|
+
return message;
|
|
22
|
+
}
|
|
23
|
+
hlog(`python worker: ${message.message}`);
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
export async function drainAndReap(worker) {
|
|
27
|
+
forget(worker);
|
|
28
|
+
const closed = readMessage(worker.link).then(() => 'message', () => 'closed');
|
|
29
|
+
const timer = grace(SHUTDOWN_GRACE_MS);
|
|
30
|
+
try {
|
|
31
|
+
const settled = await Promise.race([closed, timer.expired]);
|
|
32
|
+
if (settled === 'closed') {
|
|
33
|
+
worker.link.reap();
|
|
34
|
+
}
|
|
35
|
+
else {
|
|
36
|
+
worker.link.kill();
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
finally {
|
|
40
|
+
timer.clear();
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
export function grace(ms) {
|
|
44
|
+
let timer;
|
|
45
|
+
const expired = new Promise((resolve) => {
|
|
46
|
+
timer = setTimeout(() => resolve('grace'), ms);
|
|
47
|
+
});
|
|
48
|
+
return { expired, clear: () => clearTimeout(timer) };
|
|
49
|
+
}
|
|
50
|
+
export async function closePythonWorker() {
|
|
51
|
+
const worker = active;
|
|
52
|
+
if (!worker) {
|
|
53
|
+
return;
|
|
54
|
+
}
|
|
55
|
+
active = null;
|
|
56
|
+
const timer = grace(SHUTDOWN_GRACE_MS);
|
|
57
|
+
try {
|
|
58
|
+
const booted = await Promise.race([worker.ready.then(() => 'ready'), timer.expired]);
|
|
59
|
+
if (booted === 'grace') {
|
|
60
|
+
worker.link.kill();
|
|
61
|
+
return;
|
|
62
|
+
}
|
|
63
|
+
await worker.link.write({ type: 'close' });
|
|
64
|
+
await drainAndReap(worker);
|
|
65
|
+
}
|
|
66
|
+
catch {
|
|
67
|
+
worker.link.kill();
|
|
68
|
+
}
|
|
69
|
+
finally {
|
|
70
|
+
timer.clear();
|
|
71
|
+
}
|
|
72
|
+
}
|
|
@@ -6,4 +6,3 @@ export interface RunSpecOptions {
|
|
|
6
6
|
readonly askConnect?: ((host: string) => Promise<boolean>) | undefined;
|
|
7
7
|
}
|
|
8
8
|
export declare function runPythonInWorker(spec: RunSpec, opts: RunSpecOptions): Promise<PythonRunResult>;
|
|
9
|
-
export declare function closePythonWorker(): Promise<void>;
|
|
@@ -1,30 +1,19 @@
|
|
|
1
|
-
import { spawnPythonWorker } from '#pyodide-worker-spawn';
|
|
2
1
|
import { AbortController } from 'bare-abort-controller';
|
|
3
2
|
import process from 'process';
|
|
4
3
|
import { fileURLToPath } from 'url';
|
|
5
4
|
import { AbortedError, raceAbort } from '../../../../../helpers/race-abort.js';
|
|
6
5
|
import { harnessLogEnabled, hlog } from '../../../../../log.js';
|
|
7
|
-
import {
|
|
8
|
-
|
|
9
|
-
|
|
6
|
+
import { RESEND_MS, SIGINT } from '../deadline-state.js';
|
|
7
|
+
import { fromWireResult } from './protocol.js';
|
|
8
|
+
import { activeWorker, destroy, drainAndReap, forget, grace, readMessage } from './active.js';
|
|
9
|
+
import { spawnPythonWorker } from './spawn.js';
|
|
10
10
|
function interruptGraceMs() {
|
|
11
11
|
const ms = Number(process.env.QVAC_PYODIDE_INTERRUPT_GRACE_MS);
|
|
12
12
|
return ms > 0 ? ms : 30_000;
|
|
13
13
|
}
|
|
14
|
-
const SHUTDOWN_GRACE_MS = 3_000;
|
|
15
|
-
let active = null;
|
|
16
14
|
function workerEntry() {
|
|
17
|
-
return (process.env.QVAC_PYODIDE_WORKER_ENTRY
|
|
18
|
-
fileURLToPath(
|
|
19
|
-
}
|
|
20
|
-
async function readMessage(link) {
|
|
21
|
-
for (;;) {
|
|
22
|
-
const message = asWorkerMessage(await link.read());
|
|
23
|
-
if (message?.type !== 'log') {
|
|
24
|
-
return message;
|
|
25
|
-
}
|
|
26
|
-
hlog(`python worker: ${message.message}`);
|
|
27
|
-
}
|
|
15
|
+
return (process.env.QVAC_PYODIDE_WORKER_ENTRY ||
|
|
16
|
+
fileURLToPath(import.meta.resolve('#pyodide-worker-entry')));
|
|
28
17
|
}
|
|
29
18
|
async function awaitReady(link) {
|
|
30
19
|
const message = await readMessage(link);
|
|
@@ -47,38 +36,11 @@ function boot() {
|
|
|
47
36
|
nextId: 1
|
|
48
37
|
};
|
|
49
38
|
worker.ready.catch(() => {
|
|
50
|
-
|
|
51
|
-
active = null;
|
|
52
|
-
}
|
|
39
|
+
forget(worker);
|
|
53
40
|
link.kill();
|
|
54
41
|
});
|
|
55
42
|
return worker;
|
|
56
43
|
}
|
|
57
|
-
function destroy(worker) {
|
|
58
|
-
if (active === worker) {
|
|
59
|
-
active = null;
|
|
60
|
-
}
|
|
61
|
-
worker.link.kill();
|
|
62
|
-
}
|
|
63
|
-
async function drainAndReap(worker) {
|
|
64
|
-
if (active === worker) {
|
|
65
|
-
active = null;
|
|
66
|
-
}
|
|
67
|
-
const closed = readMessage(worker.link).then(() => 'message', () => 'closed');
|
|
68
|
-
const timer = grace(SHUTDOWN_GRACE_MS);
|
|
69
|
-
try {
|
|
70
|
-
const settled = await Promise.race([closed, timer.expired]);
|
|
71
|
-
if (settled === 'closed') {
|
|
72
|
-
worker.link.reap();
|
|
73
|
-
}
|
|
74
|
-
else {
|
|
75
|
-
worker.link.kill();
|
|
76
|
-
}
|
|
77
|
-
}
|
|
78
|
-
finally {
|
|
79
|
-
timer.clear();
|
|
80
|
-
}
|
|
81
|
-
}
|
|
82
44
|
async function resultFor(worker, id, askConnect) {
|
|
83
45
|
let message = await readMessage(worker.link);
|
|
84
46
|
while (message?.type === 'connect-ask') {
|
|
@@ -97,13 +59,6 @@ async function resultFor(worker, id, askConnect) {
|
|
|
97
59
|
}
|
|
98
60
|
throw new Error('python worker sent an unexpected message');
|
|
99
61
|
}
|
|
100
|
-
function grace(ms) {
|
|
101
|
-
let timer;
|
|
102
|
-
const expired = new Promise((resolve) => {
|
|
103
|
-
timer = setTimeout(() => resolve('grace'), ms);
|
|
104
|
-
});
|
|
105
|
-
return { expired, clear: () => clearTimeout(timer) };
|
|
106
|
-
}
|
|
107
62
|
async function interrupt(worker, outcome) {
|
|
108
63
|
Atomics.store(worker.view, 0, SIGINT);
|
|
109
64
|
const resend = setInterval(() => Atomics.store(worker.view, 0, SIGINT), RESEND_MS);
|
|
@@ -128,7 +83,7 @@ async function interrupt(worker, outcome) {
|
|
|
128
83
|
}
|
|
129
84
|
}
|
|
130
85
|
export async function runPythonInWorker(spec, opts) {
|
|
131
|
-
const worker = (
|
|
86
|
+
const worker = activeWorker(boot);
|
|
132
87
|
worker.link.ref();
|
|
133
88
|
try {
|
|
134
89
|
const id = worker.nextId++;
|
|
@@ -190,27 +145,4 @@ export async function runPythonInWorker(spec, opts) {
|
|
|
190
145
|
worker.link.unref();
|
|
191
146
|
}
|
|
192
147
|
}
|
|
193
|
-
export async function closePythonWorker() {
|
|
194
|
-
const worker = active;
|
|
195
|
-
if (!worker) {
|
|
196
|
-
return;
|
|
197
|
-
}
|
|
198
|
-
active = null;
|
|
199
|
-
const timer = grace(SHUTDOWN_GRACE_MS);
|
|
200
|
-
try {
|
|
201
|
-
const booted = await Promise.race([worker.ready.then(() => 'ready'), timer.expired]);
|
|
202
|
-
if (booted === 'grace') {
|
|
203
|
-
worker.link.kill();
|
|
204
|
-
return;
|
|
205
|
-
}
|
|
206
|
-
await worker.link.write({ type: 'close' });
|
|
207
|
-
await drainAndReap(worker);
|
|
208
|
-
}
|
|
209
|
-
catch {
|
|
210
|
-
worker.link.kill();
|
|
211
|
-
}
|
|
212
|
-
finally {
|
|
213
|
-
timer.clear();
|
|
214
|
-
}
|
|
215
|
-
}
|
|
216
148
|
function noop() { }
|
|
@@ -1,127 +1,3 @@
|
|
|
1
|
-
var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExtension) || function (path, preserveJsx) {
|
|
2
|
-
if (typeof path === "string" && /^\.\.?\//.test(path)) {
|
|
3
|
-
return path.replace(/\.(tsx)$|((?:\.d)?)((?:\.[^./]+?)?)\.([cm]?)ts$/i, function (m, tsx, d, ext, cm) {
|
|
4
|
-
return tsx ? preserveJsx ? ".jsx" : ".js" : d && (!ext || !cm) ? m : (d + ext + "." + cm.toLowerCase() + "js");
|
|
5
|
-
});
|
|
6
|
-
}
|
|
7
|
-
return path;
|
|
8
|
-
};
|
|
9
|
-
import process from 'process';
|
|
10
|
-
import { errorMessage, isRecord } from '../../../../../guards.js';
|
|
11
|
-
import { setHarnessLog, setHarnessLogSink } from '../../../../../log.js';
|
|
12
1
|
import { defaultLoad } from '../loader.js';
|
|
13
|
-
import {
|
|
14
|
-
|
|
15
|
-
function narrowThreadData(data) {
|
|
16
|
-
if (isRecord(data) && 'sab' in data && 'handle' in data) {
|
|
17
|
-
const sab = data.sab;
|
|
18
|
-
const log = 'log' in data && data.log === true;
|
|
19
|
-
if (sab instanceof SharedArrayBuffer) {
|
|
20
|
-
return { handle: data.handle, sab, log };
|
|
21
|
-
}
|
|
22
|
-
}
|
|
23
|
-
throw new Error('python worker started without { handle, sab } thread data');
|
|
24
|
-
}
|
|
25
|
-
async function openBarePort() {
|
|
26
|
-
const { default: Channel } = await import('bare-channel');
|
|
27
|
-
const self = Bare.Thread.self;
|
|
28
|
-
if (!self) {
|
|
29
|
-
throw new Error('python worker entry loaded on the main thread');
|
|
30
|
-
}
|
|
31
|
-
const { handle, sab, log } = narrowThreadData(self.data);
|
|
32
|
-
const port = Channel.from(handle).connect();
|
|
33
|
-
return {
|
|
34
|
-
port: {
|
|
35
|
-
read: () => port.read(),
|
|
36
|
-
write: (message) => void port.write(message),
|
|
37
|
-
close: () => void port.close()
|
|
38
|
-
},
|
|
39
|
-
handle,
|
|
40
|
-
sab,
|
|
41
|
-
log
|
|
42
|
-
};
|
|
43
|
-
}
|
|
44
|
-
async function openNodePort() {
|
|
45
|
-
const { parentPort, workerData } = await import('worker_threads');
|
|
46
|
-
if (!parentPort) {
|
|
47
|
-
throw new Error('python worker entry loaded on the main thread');
|
|
48
|
-
}
|
|
49
|
-
const data = narrowThreadData({ handle: parentPort, ...workerData });
|
|
50
|
-
const queued = [];
|
|
51
|
-
const waiting = [];
|
|
52
|
-
parentPort.on('message', (value) => {
|
|
53
|
-
const next = waiting.shift();
|
|
54
|
-
if (next) {
|
|
55
|
-
next(value);
|
|
56
|
-
}
|
|
57
|
-
else {
|
|
58
|
-
queued.push(value);
|
|
59
|
-
}
|
|
60
|
-
});
|
|
61
|
-
return {
|
|
62
|
-
port: {
|
|
63
|
-
read() {
|
|
64
|
-
if (queued.length > 0) {
|
|
65
|
-
return Promise.resolve(queued.shift());
|
|
66
|
-
}
|
|
67
|
-
return new Promise((resolve) => waiting.push(resolve));
|
|
68
|
-
},
|
|
69
|
-
write: (message) => parentPort.postMessage(message),
|
|
70
|
-
close: () => parentPort.close()
|
|
71
|
-
},
|
|
72
|
-
...data
|
|
73
|
-
};
|
|
74
|
-
}
|
|
75
|
-
async function loadInterpreter() {
|
|
76
|
-
const testLoader = process.env.QVAC_PYODIDE_TEST_LOADER;
|
|
77
|
-
if (!testLoader) {
|
|
78
|
-
return defaultLoad;
|
|
79
|
-
}
|
|
80
|
-
const loaded = await import(__rewriteRelativeImportExtension(testLoader));
|
|
81
|
-
if (isRecord(loaded) && 'default' in loaded && typeof loaded.default === 'function') {
|
|
82
|
-
return loaded.default;
|
|
83
|
-
}
|
|
84
|
-
throw new Error(`QVAC_PYODIDE_TEST_LOADER module has no default export: ${testLoader}`);
|
|
85
|
-
}
|
|
86
|
-
const { port, sab, log } = typeof globalThis.Bare !== 'undefined' ? await openBarePort() : await openNodePort();
|
|
87
|
-
setHarnessLog(log);
|
|
88
|
-
setHarnessLogSink((message) => port.write({ type: 'log', message }));
|
|
89
|
-
try {
|
|
90
|
-
const load = await loadInterpreter();
|
|
91
|
-
const py = await load();
|
|
92
|
-
py.setInterruptBuffer(new Uint8Array(sab));
|
|
93
|
-
port.write({ type: 'ready' });
|
|
94
|
-
for (;;) {
|
|
95
|
-
const message = asHostMessage(await port.read());
|
|
96
|
-
if (!message || message.type === 'close') {
|
|
97
|
-
break;
|
|
98
|
-
}
|
|
99
|
-
if (message.type === 'connect-answer') {
|
|
100
|
-
continue;
|
|
101
|
-
}
|
|
102
|
-
let closing = false;
|
|
103
|
-
const askConnect = async (host) => {
|
|
104
|
-
if (closing)
|
|
105
|
-
return false;
|
|
106
|
-
port.write({ type: 'connect-ask', host });
|
|
107
|
-
const answer = asHostMessage(await port.read());
|
|
108
|
-
if (answer?.type === 'connect-answer') {
|
|
109
|
-
return answer.allowed;
|
|
110
|
-
}
|
|
111
|
-
closing = true;
|
|
112
|
-
return false;
|
|
113
|
-
};
|
|
114
|
-
const { result, evict } = await performRun(py, message.spec, { askConnect });
|
|
115
|
-
port.write({ type: 'result', id: message.id, result: toWireResult(result), evict });
|
|
116
|
-
if (evict || closing) {
|
|
117
|
-
break;
|
|
118
|
-
}
|
|
119
|
-
}
|
|
120
|
-
}
|
|
121
|
-
catch (error) {
|
|
122
|
-
const message = errorMessage(error);
|
|
123
|
-
port.write({ type: 'boot-error', message });
|
|
124
|
-
}
|
|
125
|
-
finally {
|
|
126
|
-
port.close();
|
|
127
|
-
}
|
|
2
|
+
import { serve } from './serve.js';
|
|
3
|
+
await serve(defaultLoad);
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { workerData } from 'worker_threads';
|
|
2
|
+
import { isRecord } from '../../../../../guards.js';
|
|
3
|
+
import { ARMED, FIRED, RESEND_MS, SIGINT } from '../deadline-state.js';
|
|
4
|
+
function narrowDeadlineData(data) {
|
|
5
|
+
if (isRecord(data)) {
|
|
6
|
+
const { state, interrupt, ms } = data;
|
|
7
|
+
if (state instanceof SharedArrayBuffer && interrupt instanceof SharedArrayBuffer) {
|
|
8
|
+
return { state, interrupt, ms: Number(ms) };
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
throw new Error('interrupt deadline started without { state, interrupt } worker data');
|
|
12
|
+
}
|
|
13
|
+
const data = narrowDeadlineData(workerData);
|
|
14
|
+
const state = new Int32Array(data.state);
|
|
15
|
+
const interrupt = new Uint8Array(data.interrupt);
|
|
16
|
+
Atomics.wait(state, 0, ARMED, data.ms);
|
|
17
|
+
if (Atomics.compareExchange(state, 0, ARMED, FIRED) === ARMED) {
|
|
18
|
+
do {
|
|
19
|
+
Atomics.store(interrupt, 0, SIGINT);
|
|
20
|
+
} while (Atomics.wait(state, 0, FIRED, RESEND_MS) === 'timed-out');
|
|
21
|
+
}
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import { parentPort, workerData } from 'worker_threads';
|
|
2
|
+
import { errorMessage, isRecord } from '../../../../../guards.js';
|
|
3
|
+
import { setHarnessLog, setHarnessLogSink } from '../../../../../log.js';
|
|
4
|
+
import { performRun } from '../run-session.js';
|
|
5
|
+
import { asHostMessage, toWireResult } from './protocol.js';
|
|
6
|
+
function narrowThreadData(data) {
|
|
7
|
+
if (isRecord(data) && 'sab' in data) {
|
|
8
|
+
const sab = data.sab;
|
|
9
|
+
const log = 'log' in data && data.log === true;
|
|
10
|
+
if (sab instanceof SharedArrayBuffer) {
|
|
11
|
+
return { sab, log };
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
throw new Error('python worker started without { sab } worker data');
|
|
15
|
+
}
|
|
16
|
+
function openPort() {
|
|
17
|
+
const parent = parentPort;
|
|
18
|
+
if (!parent) {
|
|
19
|
+
throw new Error('python worker entry loaded on the main thread');
|
|
20
|
+
}
|
|
21
|
+
const queued = [];
|
|
22
|
+
const waiting = [];
|
|
23
|
+
parent.on('message', (value) => {
|
|
24
|
+
const next = waiting.shift();
|
|
25
|
+
if (next) {
|
|
26
|
+
next(value);
|
|
27
|
+
}
|
|
28
|
+
else {
|
|
29
|
+
queued.push(value);
|
|
30
|
+
}
|
|
31
|
+
});
|
|
32
|
+
return {
|
|
33
|
+
port: {
|
|
34
|
+
read() {
|
|
35
|
+
if (queued.length > 0) {
|
|
36
|
+
return Promise.resolve(queued.shift());
|
|
37
|
+
}
|
|
38
|
+
return new Promise((resolve) => waiting.push(resolve));
|
|
39
|
+
},
|
|
40
|
+
write: (message) => parent.postMessage(message),
|
|
41
|
+
close: () => parent.close()
|
|
42
|
+
},
|
|
43
|
+
...narrowThreadData(workerData)
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
async function serveRuns(port, py) {
|
|
47
|
+
for (;;) {
|
|
48
|
+
const message = asHostMessage(await port.read());
|
|
49
|
+
if (!message || message.type === 'close') {
|
|
50
|
+
break;
|
|
51
|
+
}
|
|
52
|
+
if (message.type === 'connect-answer') {
|
|
53
|
+
continue;
|
|
54
|
+
}
|
|
55
|
+
let closing = false;
|
|
56
|
+
const askConnect = async (host) => {
|
|
57
|
+
if (closing)
|
|
58
|
+
return false;
|
|
59
|
+
port.write({ type: 'connect-ask', host });
|
|
60
|
+
const answer = asHostMessage(await port.read());
|
|
61
|
+
if (answer?.type === 'connect-answer') {
|
|
62
|
+
return answer.allowed;
|
|
63
|
+
}
|
|
64
|
+
closing = true;
|
|
65
|
+
return false;
|
|
66
|
+
};
|
|
67
|
+
const { result, evict } = await performRun(py, message.spec, { askConnect });
|
|
68
|
+
port.write({ type: 'result', id: message.id, result: toWireResult(result), evict });
|
|
69
|
+
if (evict || closing) {
|
|
70
|
+
break;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
export async function serve(load) {
|
|
75
|
+
const { port, sab, log } = openPort();
|
|
76
|
+
setHarnessLog(log);
|
|
77
|
+
setHarnessLogSink((message) => port.write({ type: 'log', message }));
|
|
78
|
+
try {
|
|
79
|
+
const py = await load();
|
|
80
|
+
py.setInterruptBuffer(new Uint8Array(sab));
|
|
81
|
+
port.write({ type: 'ready' });
|
|
82
|
+
await serveRuns(port, py);
|
|
83
|
+
}
|
|
84
|
+
catch (error) {
|
|
85
|
+
port.write({ type: 'boot-error', message: errorMessage(error) });
|
|
86
|
+
}
|
|
87
|
+
finally {
|
|
88
|
+
port.close();
|
|
89
|
+
}
|
|
90
|
+
}
|
|
@@ -48,11 +48,3 @@ export function spawnPythonWorker(entry, data) {
|
|
|
48
48
|
unref: () => worker.unref()
|
|
49
49
|
};
|
|
50
50
|
}
|
|
51
|
-
export function spawnInlineThread(source, data) {
|
|
52
|
-
const worker = new Worker(`const data = require('worker_threads').workerData\n${source}`, {
|
|
53
|
-
eval: true,
|
|
54
|
-
workerData: data
|
|
55
|
-
});
|
|
56
|
-
worker.unref();
|
|
57
|
-
return () => { };
|
|
58
|
-
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { spawnEnv } from '../../helpers/spawn-path.js';
|
|
2
|
+
export async function bareRuntime() {
|
|
3
|
+
const [{ spawn }, { default: process }] = await Promise.all([
|
|
4
|
+
import('bare-subprocess'),
|
|
5
|
+
import('bare-process')
|
|
6
|
+
]);
|
|
7
|
+
const rawSpawn = spawn;
|
|
8
|
+
const spawnFn = (command, opts = {}) => rawSpawn(command, { ...opts, env: spawnEnv(process.env, process.platform, opts.env) });
|
|
9
|
+
return { spawn: spawnFn, killProcess: process.kill.bind(process) };
|
|
10
|
+
}
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import type { ToolDefinition } from '../capabilities/completion.ts';
|
|
2
2
|
import type { ExecError } from './exec/adapters/python/output-policy.ts';
|
|
3
3
|
import type { LoadPyodideFn } from './exec/adapters/python/pyodide.ts';
|
|
4
|
-
import { type ShellRuntime } from './exec/adapters/shell/shell-adapter.ts';
|
|
5
4
|
import { type KillProcess, type SpawnFn } from './exec/runner.ts';
|
|
6
5
|
import { type Realpath } from './fs/scope.ts';
|
|
7
6
|
import type { CredentialStoreFn } from './skills/auth-resolver.ts';
|
|
@@ -19,7 +18,6 @@ export interface ExecToolOptions {
|
|
|
19
18
|
readonly deniedRoots?: readonly string[];
|
|
20
19
|
readonly skillsDir?: string;
|
|
21
20
|
}
|
|
22
|
-
export declare function bareRuntime(): Promise<Required<ShellRuntime>>;
|
|
23
21
|
export interface ScriptRun {
|
|
24
22
|
args: Record<string, unknown>;
|
|
25
23
|
trusted: boolean;
|
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
import fs from 'fs';
|
|
2
2
|
import path from 'path';
|
|
3
3
|
import { AbortedError } from '../helpers/race-abort.js';
|
|
4
|
-
import {
|
|
5
|
-
import { checkPdfBuildSource, checkBareLengthArgs, checkPptxBuildSource, checkPythonSource, checkUnstagedRead, createPythonAdapter, isPythonLanguage, parseInputs, parseOutputs, parsePackages, parseWorkdir, pinned, runsThroughPyodide, unwrapPythonSource } from './exec/adapters/python/python-adapter.js';
|
|
4
|
+
import { checkPdfBuildSource, checkBareLengthArgs, checkPptxBuildSource, checkPythonSource, checkUnstagedRead, createPythonAdapter, isPythonLanguage, parseInputs, parseOutputs, parsePackages, parseWorkdir, pinned, runsThroughPyodide, sandboxInstallError, unwrapPythonSource } from './exec/adapters/python/python-adapter.js';
|
|
6
5
|
import { pythonCreateEffect } from './exec/adapters/python/create-effect.js';
|
|
7
6
|
import { namedHosts } from './exec/adapters/python/named-hosts.js';
|
|
8
7
|
import { pythonReachesNetwork } from './exec/adapters/python/net-shim.js';
|
|
@@ -12,6 +11,7 @@ import { MAX_OUTPUT_CHARS_LIMIT } from './exec/runner.js';
|
|
|
12
11
|
import { bareBin, firstCommandToken, sanitizeCommand } from './exec/scope.js';
|
|
13
12
|
import { FILES_NEED_PYTHON_ERROR, isPythonSourceShaped, PACKAGES_NEED_PYTHON_ERROR, pythonRedirectError, startsLikePython, videoToolError } from './exec/shell-guidance.js';
|
|
14
13
|
import { shellArgv } from './exec/shell-argv.js';
|
|
14
|
+
import { bareRuntime } from './exec/bare-runtime.js';
|
|
15
15
|
import { canonicalDenyRoots, defaultRealpath, hostPath, resolveKey } from './fs/scope.js';
|
|
16
16
|
import { unquoteArg } from './model-args.js';
|
|
17
17
|
import { settingsEnv } from './skills/skill-settings.js';
|
|
@@ -48,15 +48,6 @@ function shellEffect(command, toolOps) {
|
|
|
48
48
|
}
|
|
49
49
|
return matched.every((op) => op.annotation(argv) === 'read') ? 'read' : 'write';
|
|
50
50
|
}
|
|
51
|
-
export async function bareRuntime() {
|
|
52
|
-
const [{ spawn }, { default: process }] = await Promise.all([
|
|
53
|
-
import('bare-subprocess'),
|
|
54
|
-
import('bare-process')
|
|
55
|
-
]);
|
|
56
|
-
const rawSpawn = spawn;
|
|
57
|
-
const spawnFn = (command, opts = {}) => rawSpawn(command, { ...opts, env: spawnEnv(process.env, process.platform, opts.env) });
|
|
58
|
-
return { spawn: spawnFn, killProcess: process.kill.bind(process) };
|
|
59
|
-
}
|
|
60
51
|
function resolveExecution({ adapters, fallback }, args, ctx) {
|
|
61
52
|
for (const adapter of adapters) {
|
|
62
53
|
const plan = adapter.resolve(args, ctx);
|
|
@@ -468,6 +459,10 @@ export function createExecTool({ spawn, killProcess, loadPyodide, disablePyodide
|
|
|
468
459
|
if (packages && 'error' in packages) {
|
|
469
460
|
return packages.error;
|
|
470
461
|
}
|
|
462
|
+
const refusal = ctx?.sandboxMode === true ? sandboxInstallError(command, packages) : null;
|
|
463
|
+
if (refusal) {
|
|
464
|
+
return refusal;
|
|
465
|
+
}
|
|
471
466
|
if (!scriptRun || !scriptRun.trusted) {
|
|
472
467
|
const sourceError = checkPythonSource(command);
|
|
473
468
|
if (sourceError) {
|
|
@@ -339,7 +339,8 @@ export function registerHarnessTypes(ns) {
|
|
|
339
339
|
{ name: 'supportsImages', type: 'bool', required: true },
|
|
340
340
|
{ name: 'expectedSize', type: 'uint', required: true },
|
|
341
341
|
{ name: 'actualSize', type: 'uint' },
|
|
342
|
-
{ name: 'label', type: 'string' }
|
|
342
|
+
{ name: 'label', type: 'string' },
|
|
343
|
+
{ name: 'isLoading', type: 'bool' }
|
|
343
344
|
]
|
|
344
345
|
});
|
|
345
346
|
ns.register({
|
|
@@ -450,7 +451,8 @@ export function registerHarnessTypes(ns) {
|
|
|
450
451
|
fields: [
|
|
451
452
|
{ name: 'capabilityProfile', type: '@harness/capability-profile', required: true },
|
|
452
453
|
{ name: 'recommendedModelName', type: 'string', required: true },
|
|
453
|
-
{ name: 'memoryBudget', type: '@harness/memory-budget' }
|
|
454
|
+
{ name: 'memoryBudget', type: '@harness/memory-budget' },
|
|
455
|
+
{ name: 'mobile', type: 'bool' }
|
|
454
456
|
]
|
|
455
457
|
});
|
|
456
458
|
ns.register({
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
// This file is autogenerated by the hyperschema compiler
|
|
2
|
-
// Schema Version:
|
|
2
|
+
// Schema Version: 18
|
|
3
3
|
/* eslint-disable camelcase */
|
|
4
4
|
/* eslint-disable quotes */
|
|
5
5
|
/* eslint-disable space-before-function-paren */
|
|
6
6
|
|
|
7
7
|
import { c } from 'hyperschema/runtime'
|
|
8
8
|
|
|
9
|
-
const VERSION =
|
|
9
|
+
const VERSION = 18
|
|
10
10
|
|
|
11
11
|
// eslint-disable-next-line no-unused-vars
|
|
12
12
|
let version = VERSION
|
|
@@ -1610,7 +1610,7 @@ const encoding31 = {
|
|
|
1610
1610
|
c.string.preencode(state, m.type)
|
|
1611
1611
|
c.string.preencode(state, m.registryPath)
|
|
1612
1612
|
c.string.preencode(state, m.registrySource)
|
|
1613
|
-
state.end++ // max flag is
|
|
1613
|
+
state.end++ // max flag is 32 so always one byte
|
|
1614
1614
|
c.uint.preencode(state, m.expectedSize)
|
|
1615
1615
|
|
|
1616
1616
|
if (m.actualSize) c.uint.preencode(state, m.actualSize)
|
|
@@ -1622,7 +1622,8 @@ const encoding31 = {
|
|
|
1622
1622
|
(m.isLoaded ? 2 : 0) |
|
|
1623
1623
|
(m.supportsImages ? 4 : 0) |
|
|
1624
1624
|
(m.actualSize ? 8 : 0) |
|
|
1625
|
-
((version >= 15 && m.label) ? 16 : 0)
|
|
1625
|
+
((version >= 15 && m.label) ? 16 : 0) |
|
|
1626
|
+
((version >= 18 && m.isLoading) ? 32 : 0)
|
|
1626
1627
|
|
|
1627
1628
|
c.string.encode(state, m.name)
|
|
1628
1629
|
c.string.encode(state, m.type)
|
|
@@ -1651,7 +1652,8 @@ const encoding31 = {
|
|
|
1651
1652
|
supportsImages: (flags & 4) !== 0,
|
|
1652
1653
|
expectedSize: c.uint.decode(state),
|
|
1653
1654
|
actualSize: (flags & 8) !== 0 ? c.uint.decode(state) : 0,
|
|
1654
|
-
label: (version >= 15 && (flags & 16) !== 0) ? c.string.decode(state) : null
|
|
1655
|
+
label: (version >= 15 && (flags & 16) !== 0) ? c.string.decode(state) : null,
|
|
1656
|
+
isLoading: (version >= 18 && (flags & 32) !== 0)
|
|
1655
1657
|
}
|
|
1656
1658
|
}
|
|
1657
1659
|
}
|
|
@@ -1919,12 +1921,12 @@ const encoding42 = {
|
|
|
1919
1921
|
preencode(state, m) {
|
|
1920
1922
|
encoding38.preencode(state, m.capabilityProfile)
|
|
1921
1923
|
c.string.preencode(state, m.recommendedModelName)
|
|
1922
|
-
state.end++ // max flag is
|
|
1924
|
+
state.end++ // max flag is 2 so always one byte
|
|
1923
1925
|
|
|
1924
1926
|
if (version >= 2 && m.memoryBudget) encoding42_2.preencode(state, m.memoryBudget)
|
|
1925
1927
|
},
|
|
1926
1928
|
encode(state, m) {
|
|
1927
|
-
const flags = (version >= 2 && m.memoryBudget) ? 1 : 0
|
|
1929
|
+
const flags = ((version >= 2 && m.memoryBudget) ? 1 : 0) | ((version >= 17 && m.mobile) ? 2 : 0)
|
|
1928
1930
|
|
|
1929
1931
|
encoding38.encode(state, m.capabilityProfile)
|
|
1930
1932
|
c.string.encode(state, m.recommendedModelName)
|
|
@@ -1940,7 +1942,8 @@ const encoding42 = {
|
|
|
1940
1942
|
return {
|
|
1941
1943
|
capabilityProfile: r0,
|
|
1942
1944
|
recommendedModelName: r1,
|
|
1943
|
-
memoryBudget: (version >= 2 && (flags & 1) !== 0) ? encoding42_2.decode(state) : null
|
|
1945
|
+
memoryBudget: (version >= 2 && (flags & 1) !== 0) ? encoding42_2.decode(state) : null,
|
|
1946
|
+
mobile: (version >= 17 && (flags & 2) !== 0)
|
|
1944
1947
|
}
|
|
1945
1948
|
}
|
|
1946
1949
|
}
|