@qvac/harness 0.6.0 → 0.7.0

package/dist-lib/lib/tools/exec/adapters/python/deadline-state.js

@@ -0,0 +1,5 @@
+export const ARMED = 0;
+export const DISARMED = 1;
+export const FIRED = 2;
+export const SIGINT = 2;
+export const RESEND_MS = 100;

package/dist-lib/lib/tools/exec/adapters/python/interrupt-timer.js

@@ -1,28 +1,21 @@
-import { spawnInlineThread } from '#pyodide-worker-spawn';
-const ARMED = 0;
-const DISARMED = 1;
-const FIRED = 2;
-const SIGINT = 2;
-const RESEND_MS = 100;
-const SOURCE = `
-const state = new Int32Array(data.state)
-const interrupt = new Uint8Array(data.interrupt)
-Atomics.wait(state, 0, ${ARMED}, data.ms)
-if (Atomics.compareExchange(state, 0, ${ARMED}, ${FIRED}) === ${ARMED}) {
-  do Atomics.store(interrupt, 0, ${SIGINT})
-  while (Atomics.wait(state, 0, ${FIRED}, ${RESEND_MS}) === 'timed-out')
-}
-`;
+import { fileURLToPath } from 'url';
+import { Worker } from 'worker_threads';
+import { errorMessage } from '../../../../guards.js';
+import { hlog } from '../../../../log.js';
+import { DISARMED, FIRED } from './deadline-state.js';
 export function startInterruptTimer(ms) {
     const state = new Int32Array(new SharedArrayBuffer(4));
     const interrupt = new Uint8Array(new SharedArrayBuffer(1));
-    const reap = spawnInlineThread(SOURCE, { state: state.buffer, interrupt: interrupt.buffer, ms });
+    const workerData = { state: state.buffer, interrupt: interrupt.buffer, ms };
+    const entry = fileURLToPath(import.meta.resolve('#pyodide-interrupt-deadline'));
+    new Worker(entry, { workerData })
+        .on('error', (error) => hlog('python: interrupt deadline failed', { err: errorMessage(error) }))
+        .unref();
     return {
         interrupt,
         disarm() {
             const fired = Atomics.exchange(state, 0, DISARMED) === FIRED;
             Atomics.notify(state, 0);
-            reap();
             return fired;
         }
     };

package/dist-lib/lib/tools/exec/adapters/python/loader.js

@@ -1,14 +1,8 @@
-var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExtension) || function (path, preserveJsx) {
-    if (typeof path === "string" && /^\.\.?\//.test(path)) {
-        return path.replace(/\.(tsx)$|((?:\.d)?)((?:\.[^./]+?)?)\.([cm]?)ts$/i, function (m, tsx, d, ext, cm) {
-            return tsx ? preserveJsx ? ".jsx" : ".js" : d && (!ext || !cm) ? m : (d + ext + "." + cm.toLowerCase() + "js");
-        });
-    }
-    return path;
-};
 import b4a from 'b4a';
 import { constants as fsConstants } from 'fs';
+import path from 'path';
 import process from 'process';
+import { fileURLToPath } from 'url';
 import { createGuardedFetch } from './guarded-fetch.js';
 import { seedPackageCache } from './wheels.js';
 class B4aTextDecoder {
@@ -95,15 +89,25 @@ export function pyodideLoadOptions() {
     }
     return { ...silentStdio, packageCacheDir: cacheDir };
 }
+function pyodideIndexURL() {
+    import.meta.asset('pyodide/python_stdlib.zip');
+    import.meta.asset('pyodide/pyodide-lock.json');
+    return path.dirname(fileURLToPath(import.meta.asset('pyodide/pyodide.asm.wasm'))) + '/';
+}
 export const defaultLoad = async () => {
     if (typeof globalThis.Bare !== 'undefined') {
         await shapeBareGlobals();
     }
-    const moduleName = 'pyodide';
-    const { loadPyodide } = await import(__rewriteRelativeImportExtension(moduleName));
+    const { loadPyodide } = await import('pyodide');
+    const { default: createPyodideModule } = await import('pyodide/pyodide.asm.mjs');
     const options = pyodideLoadOptions();
     if (options.packageCacheDir) {
         seedPackageCache(options.packageCacheDir);
     }
-    return (await loadPyodide(options));
+    const indexURL = pyodideIndexURL();
+    return (await loadPyodide({
+        ...options,
+        indexURL,
+        createPyodideModule
+    }));
 };

package/dist-lib/lib/tools/exec/adapters/python/pyodide.js

@@ -1,12 +1,11 @@
 import { AbortController } from 'bare-abort-controller';
-import process from 'process';
 import ScopeLock from 'scope-lock';
 import { errorMessage } from '../../../../guards.js';
 import { AbortedError, raceAbort } from '../../../../helpers/race-abort.js';
 import { hlog } from '../../../../log.js';
 import { MAX_OUTPUT_CHARS } from '../../runner.js';
 import { startInterruptTimer } from './interrupt-timer.js';
-import { defaultLoad, pyodideLoadOptions } from './loader.js';
+import { pyodideLoadOptions } from './loader.js';
 import { isPyodideFatal, performRun } from './run-session.js';
 import { runPythonInWorker } from './worker/client.js';
 export { pyodideLoadOptions };
@@ -98,11 +97,6 @@ async function runInProcess(load, spec, options) {
         stop.abort(undefined);
     }
 }
-function workerAvailable() {
-    if (process.env.QVAC_PYODIDE_WORKER_ENTRY)
-        return true;
-    return !import.meta.url.includes('.bundle/');
-}
 export async function runPython(code, options = {}) {
     if (options.signal?.aborted)
         throw new AbortedError();
@@ -126,14 +120,6 @@ export async function runPython(code, options = {}) {
             askConnect: options.askConnect
         });
     }
-    if (!workerAvailable()) {
-        hlog('python: worker entry unavailable in this bundle; running in-process');
-        return runInProcess(defaultLoad, spec, {
-            timeoutMs,
-            signal: options.signal,
-            askConnect: options.askConnect
-        });
-    }
     await runLock.lock();
     try {
         if (options.signal?.aborted) {

package/dist-lib/lib/tools/exec/adapters/python/python-adapter.d.ts

@@ -12,6 +12,7 @@ export declare function pinned(spec: string): string;
 export declare function pipMessage(targets: readonly string[] | null, { sandbox }?: {
     sandbox?: boolean | undefined;
 }): string;
+export declare function sandboxInstallError(command: string, declared: readonly string[] | undefined): string | null;
 export interface PipFromSourceHintOptions {
     readonly packages?: readonly string[] | undefined;
     readonly sandbox?: boolean;

package/dist-lib/lib/tools/exec/adapters/python/python-adapter.js

@@ -242,6 +242,16 @@ export function pipMessage(targets, { sandbox = false } = {}) {
     const declare = specs ? `packages: ${JSON.stringify(specs)}` : 'the PyPI names in packages';
     return `pip is not available. Call exec with language "python", ${declare} and the Python source in command. Installing and running are the same one call.`;
 }
+export function sandboxInstallError(command, declared) {
+    const install = pipInstallTargets(command)?.map(pinned) ?? null;
+    const packages = [...new Set([...(declared ?? []), ...(install ?? [])])];
+    if (packages.length === 0 || planVendoredInstall(packages)) {
+        return null;
+    }
+    return install
+        ? pipMessage(install, { sandbox: true })
+        : 'sandbox mode: installing packages reaches the network';
+}
 const PIP_CALL = /\b(?:subprocess\.\w+|os\.(?:system|popen)|Popen|check_(?:output|call))\s*\(/;
 const STRING_LITERAL = /"([^"\n]*)"|'([^'\n]*)'/g;
 function pipLineTargets(line) {
@@ -1279,8 +1289,9 @@ export function createPythonAdapter(disabled, loadPyodide, network = false) {
                     const packages = install
                         ? [...new Set([...(prepared.packages ?? []), ...install])]
                         : prepared.packages;
-                    if (sandbox && install && !planVendoredInstall(packages ?? [])) {
-                        return fail(pipMessage(install, { sandbox: true }));
+                    const refusal = sandbox ? sandboxInstallError(command, prepared.packages) : null;
+                    if (refusal) {
+                        return fail(refusal);
                     }
                     const result = await runPython(install ? 'pass' : source, {
                         timeoutMs: options.timeoutMs,

package/dist-lib/lib/tools/exec/adapters/python/wheels.d.ts

@@ -28,8 +28,8 @@ interface WheelFs {
         exists: boolean;
     };
 }
-export declare function wheelsDir(): string | undefined;
-export declare function vendoredManifest(dir?: string | undefined): WheelManifest | null;
+export declare function wheelsDir(): string;
+export declare function vendoredManifest(dir?: string): WheelManifest | null;
 export declare function normalizeDistribution(name: string): string;
 export interface VendoredOptions {
     readonly dir?: string | undefined;
@@ -38,5 +38,5 @@ export interface VendoredOptions {
 export declare function vendoredSpec(distribution: string, opts?: VendoredOptions): string | null;
 export declare function planVendoredInstall(specs: readonly string[], opts?: VendoredOptions): VendoredInstall | null;
 export declare function stageVendoredWheels(target: WheelFs, install: VendoredInstall): string[];
-export declare function seedPackageCache(cacheDir: string, dir?: string | undefined): void;
+export declare function seedPackageCache(cacheDir: string, dir?: string): void;
 export {};

package/dist-lib/lib/tools/exec/adapters/python/wheels.js

@@ -16,20 +16,9 @@ const ManifestSchema = z.object({
 });
 const STAGE_DIR = '/qvac-wheels';
 export function wheelsDir() {
-    const fromHost = process.env.QVAC_PYTHON_WHEELS_DIR;
-    if (fromHost) {
-        return fromHost;
-    }
-    const root = new URL('../../../../../', import.meta.url);
-    const source = new URL(root.pathname.endsWith('/dist-lib/') ? '../wheels' : 'wheels', root);
-    if (source.href.includes('.bundle/')) {
-        return undefined;
-    }
-    return fileURLToPath(source.href);
+    return process.env.QVAC_PYTHON_WHEELS_DIR || fileURLToPath(import.meta.asset('#pyodide-wheels'));
 }
 export function vendoredManifest(dir = wheelsDir()) {
-    if (!dir)
-        return null;
     const file = `${dir}/manifest.json`;
     if (!fs.existsSync(file)) {
         hlog('python: no vendored wheels', { dir });
@@ -70,7 +59,7 @@ export function vendoredSpec(distribution, opts = {}) {
 export function planVendoredInstall(specs, opts = {}) {
     const dir = opts.dir ?? wheelsDir();
     const manifest = opts.manifest === undefined ? vendoredManifest(dir) : opts.manifest;
-    if (!manifest || !dir || specs.length === 0) {
+    if (!manifest || specs.length === 0) {
         return null;
     }
     const wheels = new Set();
@@ -108,7 +97,7 @@ export function stageVendoredWheels(target, install) {
 }
 export function seedPackageCache(cacheDir, dir = wheelsDir()) {
     const manifest = vendoredManifest(dir);
-    if (!manifest || !dir) {
+    if (!manifest) {
         return;
     }
     fs.mkdirSync(cacheDir, { recursive: true });

package/dist-lib/lib/tools/exec/adapters/python/worker/active.d.ts

@@ -0,0 +1,19 @@
+import { type WorkerMessage } from './protocol.ts';
+import type { WorkerLink } from './spawn.ts';
+export interface ActiveWorker {
+    readonly link: WorkerLink;
+    readonly view: Uint8Array;
+    readonly ready: Promise<void>;
+    nextId: number;
+}
+export declare function activeWorker(boot: () => ActiveWorker): ActiveWorker;
+export declare function forget(worker: ActiveWorker): void;
+export declare function destroy(worker: ActiveWorker): void;
+export declare function readMessage(link: WorkerLink): Promise<WorkerMessage | null>;
+export declare function drainAndReap(worker: ActiveWorker): Promise<void>;
+export interface Grace {
+    expired: Promise<'grace'>;
+    clear: () => void;
+}
+export declare function grace(ms: number): Grace;
+export declare function closePythonWorker(): Promise<void>;

package/dist-lib/lib/tools/exec/adapters/python/worker/active.js

@@ -0,0 +1,72 @@
+import { hlog } from '../../../../../log.js';
+import { asWorkerMessage } from './protocol.js';
+const SHUTDOWN_GRACE_MS = 3_000;
+let active = null;
+export function activeWorker(boot) {
+    return (active ??= boot());
+}
+export function forget(worker) {
+    if (active === worker) {
+        active = null;
+    }
+}
+export function destroy(worker) {
+    forget(worker);
+    worker.link.kill();
+}
+export async function readMessage(link) {
+    for (;;) {
+        const message = asWorkerMessage(await link.read());
+        if (message?.type !== 'log') {
+            return message;
+        }
+        hlog(`python worker: ${message.message}`);
+    }
+}
+export async function drainAndReap(worker) {
+    forget(worker);
+    const closed = readMessage(worker.link).then(() => 'message', () => 'closed');
+    const timer = grace(SHUTDOWN_GRACE_MS);
+    try {
+        const settled = await Promise.race([closed, timer.expired]);
+        if (settled === 'closed') {
+            worker.link.reap();
+        }
+        else {
+            worker.link.kill();
+        }
+    }
+    finally {
+        timer.clear();
+    }
+}
+export function grace(ms) {
+    let timer;
+    const expired = new Promise((resolve) => {
+        timer = setTimeout(() => resolve('grace'), ms);
+    });
+    return { expired, clear: () => clearTimeout(timer) };
+}
+export async function closePythonWorker() {
+    const worker = active;
+    if (!worker) {
+        return;
+    }
+    active = null;
+    const timer = grace(SHUTDOWN_GRACE_MS);
+    try {
+        const booted = await Promise.race([worker.ready.then(() => 'ready'), timer.expired]);
+        if (booted === 'grace') {
+            worker.link.kill();
+            return;
+        }
+        await worker.link.write({ type: 'close' });
+        await drainAndReap(worker);
+    }
+    catch {
+        worker.link.kill();
+    }
+    finally {
+        timer.clear();
+    }
+}

package/dist-lib/lib/tools/exec/adapters/python/worker/client.d.ts

@@ -6,4 +6,3 @@ export interface RunSpecOptions {
     readonly askConnect?: ((host: string) => Promise<boolean>) | undefined;
 }
 export declare function runPythonInWorker(spec: RunSpec, opts: RunSpecOptions): Promise<PythonRunResult>;
-export declare function closePythonWorker(): Promise<void>;

package/dist-lib/lib/tools/exec/adapters/python/worker/client.js

@@ -1,30 +1,19 @@
-import { spawnPythonWorker } from '#pyodide-worker-spawn';
 import { AbortController } from 'bare-abort-controller';
 import process from 'process';
 import { fileURLToPath } from 'url';
 import { AbortedError, raceAbort } from '../../../../../helpers/race-abort.js';
 import { harnessLogEnabled, hlog } from '../../../../../log.js';
-import { asWorkerMessage, fromWireResult } from './protocol.js';
-const SIGINT = 2;
-const RESEND_MS = 100;
+import { RESEND_MS, SIGINT } from '../deadline-state.js';
+import { fromWireResult } from './protocol.js';
+import { activeWorker, destroy, drainAndReap, forget, grace, readMessage } from './active.js';
+import { spawnPythonWorker } from './spawn.js';
 function interruptGraceMs() {
     const ms = Number(process.env.QVAC_PYODIDE_INTERRUPT_GRACE_MS);
     return ms > 0 ? ms : 30_000;
 }
-const SHUTDOWN_GRACE_MS = 3_000;
-let active = null;
 function workerEntry() {
-    return (process.env.QVAC_PYODIDE_WORKER_ENTRY ??
-        fileURLToPath(new URL(import.meta.url.endsWith('.ts') ? './entry.ts' : './entry.js', import.meta.url).href));
-}
-async function readMessage(link) {
-    for (;;) {
-        const message = asWorkerMessage(await link.read());
-        if (message?.type !== 'log') {
-            return message;
-        }
-        hlog(`python worker: ${message.message}`);
-    }
+    return (process.env.QVAC_PYODIDE_WORKER_ENTRY ||
+        fileURLToPath(import.meta.resolve('#pyodide-worker-entry')));
 }
 async function awaitReady(link) {
     const message = await readMessage(link);
@@ -47,38 +36,11 @@ function boot() {
         nextId: 1
     };
     worker.ready.catch(() => {
-        if (active === worker) {
-            active = null;
-        }
+        forget(worker);
         link.kill();
     });
     return worker;
 }
-function destroy(worker) {
-    if (active === worker) {
-        active = null;
-    }
-    worker.link.kill();
-}
-async function drainAndReap(worker) {
-    if (active === worker) {
-        active = null;
-    }
-    const closed = readMessage(worker.link).then(() => 'message', () => 'closed');
-    const timer = grace(SHUTDOWN_GRACE_MS);
-    try {
-        const settled = await Promise.race([closed, timer.expired]);
-        if (settled === 'closed') {
-            worker.link.reap();
-        }
-        else {
-            worker.link.kill();
-        }
-    }
-    finally {
-        timer.clear();
-    }
-}
 async function resultFor(worker, id, askConnect) {
     let message = await readMessage(worker.link);
     while (message?.type === 'connect-ask') {
@@ -97,13 +59,6 @@ async function resultFor(worker, id, askConnect) {
     }
     throw new Error('python worker sent an unexpected message');
 }
-function grace(ms) {
-    let timer;
-    const expired = new Promise((resolve) => {
-        timer = setTimeout(() => resolve('grace'), ms);
-    });
-    return { expired, clear: () => clearTimeout(timer) };
-}
 async function interrupt(worker, outcome) {
     Atomics.store(worker.view, 0, SIGINT);
     const resend = setInterval(() => Atomics.store(worker.view, 0, SIGINT), RESEND_MS);
@@ -128,7 +83,7 @@ async function interrupt(worker, outcome) {
     }
 }
 export async function runPythonInWorker(spec, opts) {
-    const worker = (active ??= boot());
+    const worker = activeWorker(boot);
     worker.link.ref();
     try {
         const id = worker.nextId++;
@@ -190,27 +145,4 @@ export async function runPythonInWorker(spec, opts) {
         worker.link.unref();
     }
 }
-export async function closePythonWorker() {
-    const worker = active;
-    if (!worker) {
-        return;
-    }
-    active = null;
-    const timer = grace(SHUTDOWN_GRACE_MS);
-    try {
-        const booted = await Promise.race([worker.ready.then(() => 'ready'), timer.expired]);
-        if (booted === 'grace') {
-            worker.link.kill();
-            return;
-        }
-        await worker.link.write({ type: 'close' });
-        await drainAndReap(worker);
-    }
-    catch {
-        worker.link.kill();
-    }
-    finally {
-        timer.clear();
-    }
-}
 function noop() { }

package/dist-lib/lib/tools/exec/adapters/python/worker/entry.js

@@ -1,127 +1,3 @@
-var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExtension) || function (path, preserveJsx) {
-    if (typeof path === "string" && /^\.\.?\//.test(path)) {
-        return path.replace(/\.(tsx)$|((?:\.d)?)((?:\.[^./]+?)?)\.([cm]?)ts$/i, function (m, tsx, d, ext, cm) {
-            return tsx ? preserveJsx ? ".jsx" : ".js" : d && (!ext || !cm) ? m : (d + ext + "." + cm.toLowerCase() + "js");
-        });
-    }
-    return path;
-};
-import process from 'process';
-import { errorMessage, isRecord } from '../../../../../guards.js';
-import { setHarnessLog, setHarnessLogSink } from '../../../../../log.js';
 import { defaultLoad } from '../loader.js';
-import { performRun } from '../run-session.js';
-import { asHostMessage, toWireResult } from './protocol.js';
-function narrowThreadData(data) {
-    if (isRecord(data) && 'sab' in data && 'handle' in data) {
-        const sab = data.sab;
-        const log = 'log' in data && data.log === true;
-        if (sab instanceof SharedArrayBuffer) {
-            return { handle: data.handle, sab, log };
-        }
-    }
-    throw new Error('python worker started without { handle, sab } thread data');
-}
-async function openBarePort() {
-    const { default: Channel } = await import('bare-channel');
-    const self = Bare.Thread.self;
-    if (!self) {
-        throw new Error('python worker entry loaded on the main thread');
-    }
-    const { handle, sab, log } = narrowThreadData(self.data);
-    const port = Channel.from(handle).connect();
-    return {
-        port: {
-            read: () => port.read(),
-            write: (message) => void port.write(message),
-            close: () => void port.close()
-        },
-        handle,
-        sab,
-        log
-    };
-}
-async function openNodePort() {
-    const { parentPort, workerData } = await import('worker_threads');
-    if (!parentPort) {
-        throw new Error('python worker entry loaded on the main thread');
-    }
-    const data = narrowThreadData({ handle: parentPort, ...workerData });
-    const queued = [];
-    const waiting = [];
-    parentPort.on('message', (value) => {
-        const next = waiting.shift();
-        if (next) {
-            next(value);
-        }
-        else {
-            queued.push(value);
-        }
-    });
-    return {
-        port: {
-            read() {
-                if (queued.length > 0) {
-                    return Promise.resolve(queued.shift());
-                }
-                return new Promise((resolve) => waiting.push(resolve));
-            },
-            write: (message) => parentPort.postMessage(message),
-            close: () => parentPort.close()
-        },
-        ...data
-    };
-}
-async function loadInterpreter() {
-    const testLoader = process.env.QVAC_PYODIDE_TEST_LOADER;
-    if (!testLoader) {
-        return defaultLoad;
-    }
-    const loaded = await import(__rewriteRelativeImportExtension(testLoader));
-    if (isRecord(loaded) && 'default' in loaded && typeof loaded.default === 'function') {
-        return loaded.default;
-    }
-    throw new Error(`QVAC_PYODIDE_TEST_LOADER module has no default export: ${testLoader}`);
-}
-const { port, sab, log } = typeof globalThis.Bare !== 'undefined' ? await openBarePort() : await openNodePort();
-setHarnessLog(log);
-setHarnessLogSink((message) => port.write({ type: 'log', message }));
-try {
-    const load = await loadInterpreter();
-    const py = await load();
-    py.setInterruptBuffer(new Uint8Array(sab));
-    port.write({ type: 'ready' });
-    for (;;) {
-        const message = asHostMessage(await port.read());
-        if (!message || message.type === 'close') {
-            break;
-        }
-        if (message.type === 'connect-answer') {
-            continue;
-        }
-        let closing = false;
-        const askConnect = async (host) => {
-            if (closing)
-                return false;
-            port.write({ type: 'connect-ask', host });
-            const answer = asHostMessage(await port.read());
-            if (answer?.type === 'connect-answer') {
-                return answer.allowed;
-            }
-            closing = true;
-            return false;
-        };
-        const { result, evict } = await performRun(py, message.spec, { askConnect });
-        port.write({ type: 'result', id: message.id, result: toWireResult(result), evict });
-        if (evict || closing) {
-            break;
-        }
-    }
-}
-catch (error) {
-    const message = errorMessage(error);
-    port.write({ type: 'boot-error', message });
-}
-finally {
-    port.close();
-}
+import { serve } from './serve.js';
+await serve(defaultLoad);

package/dist-lib/lib/tools/exec/adapters/python/worker/interrupt-deadline.d.ts

@@ -0,0 +1 @@
+export {};

package/dist-lib/lib/tools/exec/adapters/python/worker/interrupt-deadline.js

@@ -0,0 +1,21 @@
+import { workerData } from 'worker_threads';
+import { isRecord } from '../../../../../guards.js';
+import { ARMED, FIRED, RESEND_MS, SIGINT } from '../deadline-state.js';
+function narrowDeadlineData(data) {
+    if (isRecord(data)) {
+        const { state, interrupt, ms } = data;
+        if (state instanceof SharedArrayBuffer && interrupt instanceof SharedArrayBuffer) {
+            return { state, interrupt, ms: Number(ms) };
+        }
+    }
+    throw new Error('interrupt deadline started without { state, interrupt } worker data');
+}
+const data = narrowDeadlineData(workerData);
+const state = new Int32Array(data.state);
+const interrupt = new Uint8Array(data.interrupt);
+Atomics.wait(state, 0, ARMED, data.ms);
+if (Atomics.compareExchange(state, 0, ARMED, FIRED) === ARMED) {
+    do {
+        Atomics.store(interrupt, 0, SIGINT);
+    } while (Atomics.wait(state, 0, FIRED, RESEND_MS) === 'timed-out');
+}

package/dist-lib/lib/tools/exec/adapters/python/worker/serve.d.ts

@@ -0,0 +1,2 @@
+import { type LoadPyodideFn } from '../run-session.ts';
+export declare function serve(load: LoadPyodideFn): Promise<void>;