@qvac/harness 0.3.0 → 0.3.1

package/dist-lib/lib/tools/exec/adapters/python/net-shim.js

@@ -0,0 +1,148 @@
+export function pythonReachesNetwork() {
+    return (typeof globalThis.Bare !== 'undefined' &&
+        typeof WebAssembly.Suspending === 'function');
+}
+export const NET_SHIM = `
+def _qvac_install_net():
+    import errno, http.client, io
+    import js
+    from pyodide.ffi import can_run_sync, run_sync, to_js
+
+    if getattr(http.client, '_qvac_net', False):
+        return
+
+    class _Response(http.client.HTTPResponse):
+        # the body is read in full by the fetch, so there is no socket behind it
+        def __init__(self, status, headers, body, url):
+            self.status = self.code = status
+            self.reason = http.client.responses.get(status, '')
+            self.headers = self.msg = http.client.HTTPMessage()
+            for name, value in headers:
+                self.headers[name] = value
+            self.fp = io.BytesIO(body)
+            self.version = 11
+            self.will_close = True
+            self.chunked = False
+            self.length = len(body)
+            self.url = url
+            self.debuglevel = 0
+
+        def read(self, amt=None):
+            return self.fp.read() if amt is None or amt < 0 else self.fp.read(amt)
+
+        def read1(self, n=-1):
+            return self.read(n)
+
+        def readinto(self, b):
+            data = self.fp.read(len(b))
+            b[: len(data)] = data
+            return len(data)
+
+        def peek(self, n=0):
+            pos = self.fp.tell()
+            data = self.fp.read(max(n, 1))
+            self.fp.seek(pos)
+            return data
+
+        def isclosed(self):
+            return self.fp.closed
+
+        def close(self):
+            self.fp.close()
+
+        def fileno(self):
+            raise OSError('this response has no socket')
+
+    class _Connection(http.client.HTTPConnection):
+        scheme = 'http'
+
+        def __init__(self, host, port=None, timeout=None, source_address=None, blocksize=8192, **kw):
+            self.host, self.port, self.timeout = host, port, timeout
+            self.sock = None
+            self.debuglevel = 0
+            self._tunnel_host = None
+            self._request = None
+            self._response = None
+
+        def set_debuglevel(self, level):
+            pass
+
+        def set_tunnel(self, host, port=None, headers=None):
+            self._tunnel_host = host
+
+        def connect(self):
+            pass
+
+        def close(self):
+            pass
+
+        def putrequest(self, method, url, skip_host=False, skip_accept_encoding=False):
+            self._request = [method, url, {}, b'']
+            self._response = None
+
+        def putheader(self, header, *values):
+            self._request[2][header] = ', '.join(str(v) for v in values)
+
+        # A socket connects while the request is sent, so that is where a failure lands and
+        # where urllib turns it into URLError. A body sent after the headers waits for it.
+        def endheaders(self, message_body=None, *, encode_chunked=False):
+            if message_body is not None:
+                self.send(message_body)
+            if message_body is not None or self._request[0] in ('GET', 'HEAD', 'DELETE', 'OPTIONS'):
+                self._response = self._fetch()
+
+        def send(self, data):
+            if hasattr(data, 'read'):
+                data = data.read()
+            if isinstance(data, str):
+                data = data.encode()
+            self._request[3] += bytes(data)
+
+        def request(self, method, url, body=None, headers={}, *, encode_chunked=False):
+            self.putrequest(method, url)
+            for name, value in dict(headers).items():
+                self.putheader(name, value)
+            self.endheaders(body)
+
+        def getresponse(self):
+            response = self._response or self._fetch()
+            self._response = None
+            return response
+
+        def _fetch(self):
+            method, url, headers, body = self._request
+            if not url.startswith(('http://', 'https://')):
+                port = '' if self.port in (None, self.default_port) else ':%d' % self.port
+                url = '%s://%s%s%s' % (self.scheme, self.host, port, url)
+            if not can_run_sync():
+                raise OSError(errno.EHOSTUNREACH, 'Host is unreachable')
+            # the fetch hands back decoded bytes, so no encoding is asked for and none reported
+            skip = ('accept-encoding', 'connection', 'host', 'content-length')
+            headers = {k: v for k, v in headers.items() if k.lower() not in skip}
+            headers['Accept-Encoding'] = 'identity'
+            init = {'method': method, 'headers': headers, 'redirect': 'manual'}
+            if body:
+                init['body'] = to_js(body)
+            try:
+                response = run_sync(js.fetch(url, to_js(init, dict_converter=js.Object.fromEntries)))
+                data = bytes(run_sync(response.arrayBuffer()).to_py())
+            except Exception as error:
+                # the guard's own refusal, or the transport's: either says more than an errno
+                raise OSError(getattr(error, 'message', None) or str(error)) from None
+            entries = [(str(k), str(v)) for k, v in response.headers.entries()]
+            drop = ('content-encoding', 'transfer-encoding', 'content-length')
+            kept = [(k, v) for k, v in entries if k.lower() not in drop]
+            kept.append(('Content-Length', str(len(data))))
+            return _Response(int(response.status), kept, data, url)
+
+    class _HTTPSConnection(_Connection):
+        scheme = 'https'
+        default_port = 443
+
+    http.client.HTTPConnection = _Connection
+    http.client.HTTPSConnection = _HTTPSConnection
+    http.client._qvac_net = True
+
+_qvac_install_net()
+del _qvac_install_net
+`;

package/dist-lib/lib/tools/exec/adapters/python/output-policy.js

@@ -19,7 +19,11 @@ const MIME_TYPES = {
     json: 'application/json',
     txt: 'text/plain',
     md: 'text/markdown',
-    zip: 'application/zip'
+    zip: 'application/zip',
+    mp4: 'video/mp4',
+    webm: 'video/webm',
+    m4a: 'audio/mp4',
+    mp3: 'audio/mpeg'
 };
 const DENIED_EXTENSIONS = new Set([
     'pptm',
@@ -48,6 +52,14 @@ function hasExpectedSignature(extension, bytes) {
     if (extension === 'pptx' || extension === 'docx' || extension === 'xlsx' || extension === 'zip') {
         return matches(bytes, [0x50, 0x4b, 0x03, 0x04]);
     }
+    if (extension === 'mp4' || extension === 'm4a') {
+        return matches(bytes.slice(4), [0x66, 0x74, 0x79, 0x70]);
+    }
+    if (extension === 'webm')
+        return matches(bytes, [0x1a, 0x45, 0xdf, 0xa3]);
+    if (extension === 'mp3') {
+        return (matches(bytes, [0x49, 0x44, 0x33]) || (bytes[0] === 0xff && ((bytes[1] ?? 0) & 0xe0) === 0xe0));
+    }
     if (extension === 'pdf')
         return matches(bytes, [0x25, 0x50, 0x44, 0x46, 0x2d]);
     if (extension === 'png')

package/dist-lib/lib/tools/exec/adapters/python/pyodide.d.ts

@@ -15,5 +15,6 @@ export type RunPythonOptions = {
     readonly outputs?: readonly string[];
     readonly workdir?: string;
     readonly loadPyodide?: LoadPyodideFn;
+    readonly network?: boolean;
 };
 export declare function runPython(code: string, options?: RunPythonOptions): Promise<PythonRunResult>;

package/dist-lib/lib/tools/exec/adapters/python/pyodide.js

@@ -88,6 +88,7 @@ export async function runPython(code, options = {}) {
         outputs: options.outputs,
         workdir: options.workdir,
         maxOutputChars: options.maxOutputChars ?? MAX_OUTPUT_CHARS,
+        network: options.network === true,
         installTimeoutMs: timeoutMs
     };
     if (options.loadPyodide) {

package/dist-lib/lib/tools/exec/adapters/python/python-adapter.d.ts

@@ -4,8 +4,11 @@ import { type LoadPyodideFn } from './pyodide.ts';
 export declare function isPythonLanguage(language: unknown): boolean;
 export declare function runsThroughPyodide(language: unknown, disablePyodide: boolean): boolean;
 export declare function unwrapPythonSource(command: string): string | null;
+export declare function withoutProgramDir(command: string): string;
 export declare function pipInstallTargets(command: string): string[] | null;
-export declare function looksLikePythonSource(command: string): boolean;
+export declare function pinned(spec: string): string;
+export declare function pipMessage(targets: readonly string[] | null, sandbox?: boolean): string;
+export declare function pipFromSourceHint(stderr: string, source: string, packages: readonly string[] | undefined, sandbox?: boolean): string | null;
 export declare function stringLiteralProgramHint(source: string): string | null;
 export declare function checkCollapsedCompound(source: string): string | null;
 export declare function mangledPptxApiHint(source: string): string | null;
@@ -44,7 +47,7 @@ export declare function parseOutputs(value: unknown): string[] | {
 export declare function parseWorkdir(value: unknown): string | {
     error: string;
 } | undefined;
-export declare function missingModuleHint(stderr: string, packages: readonly string[] | undefined): string | null;
+export declare function missingModuleHint(stderr: string, packages: readonly string[] | undefined, sandbox?: boolean, network?: boolean): string | null;
 export declare function nonePlaceholderHint(stderr: string): string | null;
 export declare function slideShapesHint(stderr: string): string | null;
 export declare function shapeFontHint(stderr: string): string | null;
@@ -53,6 +56,9 @@ export declare function paragraphFillHint(stderr: string): string | null;
 export declare function readOnlyPropertyHint(stderr: string): string | null;
 export declare function paragraphIndexHint(stderr: string, outputs: readonly string[]): string | null;
 export declare function shellNameHint(stderr: string): string | null;
+export declare function noProcessHint(stderr: string, sandbox?: boolean): string | null;
+export declare function noNetworkHint(stderr: string, tools?: readonly string[], network?: boolean): string | null;
+export declare function nativeWheelHint(stderr: string): string | null;
 export declare function packageFetchFailedHint(stderr: string): string | null;
 export declare const IMAGE_INPUT: RegExp;
 export declare function pptxHasEmbeddedMedia(bytes: Uint8Array): boolean;
@@ -72,4 +78,4 @@ export declare function checkTurnImagesStaged(source: string, inputs: readonly E
 export declare function deckDeliveredStop(attachments: readonly {
     readonly fileName: string;
 }[] | undefined, exitCode: number, source?: string): string | null;
-export declare function createPythonAdapter(disabled: boolean, loadPyodide?: LoadPyodideFn): ExecutionAdapter;
+export declare function createPythonAdapter(disabled: boolean, loadPyodide?: LoadPyodideFn, network?: boolean): ExecutionAdapter;

package/dist-lib/lib/tools/exec/adapters/python/python-adapter.js

@@ -2,6 +2,9 @@ import b4a from 'b4a';
 import { attachOutputs, stageInputs } from './attachments.js';
 import { checkFileName, checkOutputName, MAX_INPUTS, MAX_OUTPUTS } from './output-policy.js';
 import { runPython } from './pyodide.js';
+import { isTranscoder, TRANSCODE_ERROR, VIDEO_TOOLS_ERROR, videoTool } from '../../video-tools.js';
+import { moduleDistribution } from './distributions.js';
+import { normalizeDistribution, planVendoredInstall, vendoredSpec } from './wheels.js';
 export function isPythonLanguage(language) {
     return typeof language === 'string' && language.trim().toLowerCase() === 'python';
 }
@@ -114,7 +117,7 @@ function unwrapBareInterpreter(s) {
     return body || null;
 }
 export function unwrapPythonSource(command) {
-    const trimmed = command.trim();
+    const trimmed = withoutProgramDir(command).trim();
     const heredoc = unwrapHeredoc(trimmed);
     if (heredoc !== null)
         return heredoc;
@@ -123,8 +126,11 @@ export function unwrapPythonSource(command) {
         return dashC;
     return unwrapBareInterpreter(trimmed);
 }
+export function withoutProgramDir(command) {
+    return command.trimStart().replace(/^[^\s'"]*[/\\](?=[^\s/\\]+(\s|$))/, '');
+}
 function pipInstallRest(command) {
-    const s = command.trim();
+    const s = withoutProgramDir(command).trim();
     let i = 0;
     const pip = leadingToken(s, PIPS);
     if (pip) {
@@ -178,17 +184,64 @@ export function pipInstallTargets(command) {
             continue;
         if (!REQUIREMENT.test(token))
             return null;
+        if (/^pip3?$/i.test(token.split(/[=<>!~[]/)[0]))
+            continue;
         targets.push(token);
     }
     return targets.length > 0 ? targets : null;
 }
-export function looksLikePythonSource(command) {
-    const first = command.trimStart().split('\n')[0];
-    if (/^\s*(from\s+\w+\s+import\b|import\s+\w+)/.test(first))
-        return true;
-    if (/^\s*(from|import)\s+(fpdf|pypdf|pptx)\b/m.test(command))
-        return true;
-    return /\b(FPDF|PdfReader|PdfWriter|Presentation)\s*\(/.test(command);
+const SANDBOX_NO_INSTALL = 'and sandbox mode installs only the packages bundled with this app, which do not include it. Use only what is already installed, or tell the user it is unavailable while sandbox mode is on.';
+const specName = (spec) => normalizeDistribution(spec.split(/[=<>!~[]/)[0]);
+export function pinned(spec) {
+    return /[=<>!~]/.test(spec) ? spec : (vendoredSpec(spec) ?? spec);
+}
+export function pipMessage(targets, sandbox = false) {
+    const specs = targets?.map(pinned) ?? null;
+    if (sandbox && !(specs?.length && planVendoredInstall(specs))) {
+        return `pip is not available, ${SANDBOX_NO_INSTALL}`;
+    }
+    const declare = specs ? `packages: ${JSON.stringify(specs)}` : 'the PyPI names in packages';
+    return `pip is not available. Call exec with language "python", ${declare} and the Python source in command. Installing and running are the same one call.`;
+}
+const PIP_CALL = /\b(?:subprocess\.\w+|os\.(?:system|popen)|Popen|check_(?:output|call))\s*\(/;
+const STRING_LITERAL = /"([^"\n]*)"|'([^'\n]*)'/g;
+function pipLineTargets(line) {
+    const words = [...line.matchAll(STRING_LITERAL)].map((match) => (match[1] ?? match[2]));
+    const whole = words.find((word) => /\bpip3?\b[^]*\binstall\b/.test(word));
+    if (whole !== undefined)
+        return pipInstallTargets(whole.trim());
+    const at = words.findIndex((word) => /^pip3?$/.test(word));
+    if (at === -1)
+        return undefined;
+    return pipInstallTargets(words.slice(at).join(' '));
+}
+export function pipFromSourceHint(stderr, source, packages, sandbox = false) {
+    if (!/emscripten does not support processes/.test(stderr))
+        return null;
+    for (const line of source.split('\n')) {
+        if (!PIP_CALL.test(line))
+            continue;
+        const targets = pipLineTargets(line);
+        if (targets === undefined)
+            continue;
+        const declared = packages ?? [];
+        const has = new Set(declared.map(specName));
+        const missing = (targets ?? []).filter((t) => !has.has(specName(t))).map(pinned);
+        if (sandbox && (targets === null || !planVendoredInstall([...declared, ...missing]))) {
+            return `pip cannot run here: this Python cannot start processes, so that line failed, ${SANDBOX_NO_INSTALL} Delete the pip line.`;
+        }
+        if (targets === null) {
+            return 'pip cannot run here: this Python cannot start processes, so that line failed. Delete it and declare the PyPI names in packages on this same call.';
+        }
+        if (missing.length) {
+            return `pip cannot run here: this Python cannot start processes, so that line failed. Delete it and send the same source with packages: ${JSON.stringify([...declared, ...missing])} on this one call. Installing and running are the same call.`;
+        }
+        if (declared.length) {
+            return `pip cannot run here: this Python cannot start processes, so that line failed. packages: ${JSON.stringify(declared)} already installs before your code runs, so delete the pip line and send the rest.`;
+        }
+        return 'pip cannot run here: this Python cannot start processes, so that line failed. Delete it and declare the PyPI names in packages on this same call.';
+    }
+    return null;
 }
 export function stringLiteralProgramHint(source) {
     const first = source.trimStart()[0];
@@ -771,16 +824,35 @@ function prepare(args, cwd) {
         return workdir;
     return { packages, inputs, outputs, workdir };
 }
-export function missingModuleHint(stderr, packages) {
+export function missingModuleHint(stderr, packages, sandbox = false, network = false) {
     const module = /ModuleNotFoundError: No module named '([^']+)'/.exec(stderr)?.[1];
     if (!module)
         return null;
     const root = module.split('.')[0];
-    const declared = (packages ?? []).some((p) => p.split(/[=<>!~[]/)[0].trim().toLowerCase() === root.toLowerCase());
-    const because = declared
-        ? `"${root}" was declared in packages but does not provide that module`
-        : `"${root}" was not declared`;
-    return `${because}: add the PyPI distribution that provides it to the packages argument (the distribution name can differ from the module, e.g. module "pptx" comes from "python-pptx"). There is no pip and no shell on this path.`;
+    const tool = videoTool(root);
+    if (tool && (!network || isTranscoder(tool)))
+        return network ? TRANSCODE_ERROR : VIDEO_TOOLS_ERROR;
+    if (module !== root) {
+        const then = sandbox
+            ? 'If the path is right it ships as its own distribution under a shared namespace, which sandbox mode cannot install: tell the user.'
+            : 'If the path is right it ships as its own distribution under a shared namespace, as google.* does: declare that distribution in packages.';
+        return `"${module}" failed to import although "${root}" loaded. Check the import path first. ${then}`;
+    }
+    const distribution = moduleDistribution(root);
+    const provider = distribution ?? root;
+    const vendored = vendoredSpec(provider);
+    if (sandbox && !vendored)
+        return `module "${root}" is not installed, ${SANDBOX_NO_INSTALL}`;
+    const spec = vendored ?? provider;
+    const renamed = distribution !== null && distribution.toLowerCase() !== root.toLowerCase();
+    const declared = (packages ?? []).map(specName);
+    if (declared.includes(root.toLowerCase()) && renamed) {
+        return `"${root}" was declared in packages but does not provide that module; the distribution that does is "${spec}": put that in packages instead. There is no pip.`;
+    }
+    if (declared.includes(root.toLowerCase()) || declared.includes(provider.toLowerCase())) {
+        return `"${provider}" is already declared in packages and still does not provide "${root}", so declaring it again cannot help. Import the module name the library really exports, or tell the user it is unavailable on this device.`;
+    }
+    return `module "${root}" is not installed: add "${spec}" to packages (packages: ["${spec}"]) and run the same source again. There is no pip.`;
 }
 export function nonePlaceholderHint(stderr) {
     if (!/AttributeError: 'NoneType' object has no attribute '(text|text_frame)'/.test(stderr)) {
@@ -833,6 +905,33 @@ export function shellNameHint(stderr) {
         return null;
     return `"${name}" is not defined because command already runs as Python source — remove the ${name} invocation and pass only the Python code itself`;
 }
+export function noProcessHint(stderr, sandbox = false) {
+    if (!/emscripten does not support processes/.test(stderr))
+        return null;
+    const packages = sandbox
+        ? ' In sandbox mode only the packages bundled with this app install;'
+        : ' Declare PyPI packages in packages;';
+    return `Python here cannot start processes: there is no shell and no pip from the source.${packages} run a shell command as its own exec call without language.`;
+}
+const FETCH_TOOLS = ['http_request', 'web_fetch'];
+export function noNetworkHint(stderr, tools, network = false) {
+    if (network)
+        return null;
+    if (!/Host is unreachable|Network is unreachable|socket\.gaierror|URLError|Temporary failure in name resolution|NewConnectionError|XMLHttpRequest is not defined/.test(stderr)) {
+        return null;
+    }
+    const failed = 'Python here cannot open network connections (requests, urllib and sockets fail), and retrying will not change that.';
+    const fetchers = FETCH_TOOLS.filter((name) => !tools || tools.includes(name));
+    return fetchers.length
+        ? `${failed} Fetch the URL with the ${fetchers.join(' or ')} tool and pass the data into the source.`
+        : `${failed} No tool that fetches a URL is available to you, so tell the user.`;
+}
+export function nativeWheelHint(stderr) {
+    const name = /Can't find a pure Python 3 wheel for '([^']+)'/.exec(stderr)?.[1];
+    if (!name)
+        return null;
+    return `"${name}" is built from compiled code, and this device installs only pure-Python packages plus the set already built into its Python. Nothing about this call can install it: use a pure-Python library instead, or tell the user it is not available here.`;
+}
 export function packageFetchFailedHint(stderr) {
     const name = /Can't fetch metadata for '([^']+)'/.exec(stderr)?.[1];
     if (!name)
@@ -987,7 +1086,7 @@ async function collect(result, declared, attach, runBudget, source) {
         ...(rejected.length > 0 && { rejectedOutputs: rejected })
     };
 }
-export function createPythonAdapter(disabled, loadPyodide) {
+export function createPythonAdapter(disabled, loadPyodide, network = false) {
     return {
         resolve(args, ctx) {
             if (!runsThroughPyodide(args.language, disabled))
@@ -997,6 +1096,9 @@ export function createPythonAdapter(disabled, loadPyodide) {
             const fetchAttachment = ctx?.fetchAttachment;
             const turnImages = ctx?.turnImages;
             const runBudget = ctx?.runBudget;
+            const sandbox = ctx?.sandboxMode === true;
+            const toolNames = ctx?.toolNames;
+            const reach = network && !sandbox;
             return {
                 async run(options) {
                     if (!command)
@@ -1035,10 +1137,13 @@ export function createPythonAdapter(disabled, loadPyodide) {
                     });
                     if (staged && 'error' in staged)
                         return fail(staged.error);
-                    const install = pipInstallTargets(command);
+                    const install = pipInstallTargets(command)?.map(pinned) ?? null;
                     const packages = install
                         ? [...new Set([...(prepared.packages ?? []), ...install])]
                         : prepared.packages;
+                    if (sandbox && install && !planVendoredInstall(packages ?? [])) {
+                        return fail(pipMessage(install, true));
+                    }
                     const result = await runPython(install ? 'pass' : source, {
                         timeoutMs: options.timeoutMs,
                         maxOutputChars: options.maxOutputChars,
@@ -1047,7 +1152,8 @@ export function createPythonAdapter(disabled, loadPyodide) {
                         inputs: staged?.files,
                         outputs: prepared.outputs,
                         workdir: prepared.workdir,
-                        loadPyodide
+                        loadPyodide,
+                        network: reach
                     });
                     const soft = softFailedDeckHint(result.stdout, result.exitCode, source);
                     if (soft)
@@ -1056,7 +1162,11 @@ export function createPythonAdapter(disabled, loadPyodide) {
                     if (missingPic)
                         return fail(missingPic);
                     const hint = packageFetchFailedHint(result.stderr) ??
-                        missingModuleHint(result.stderr, packages) ??
+                        nativeWheelHint(result.stderr) ??
+                        missingModuleHint(result.stderr, packages, sandbox, reach) ??
+                        pipFromSourceHint(result.stderr, source, packages, sandbox) ??
+                        noProcessHint(result.stderr, sandbox) ??
+                        noNetworkHint(result.stderr, toolNames, reach) ??
                         missingFileHint(result.stderr) ??
                         rgbColorImportHint(result.stderr) ??
                         nonePlaceholderHint(result.stderr) ??
@@ -1077,7 +1187,7 @@ export function createPythonAdapter(disabled, loadPyodide) {
                     const installed = install && result.exitCode === 0
                         ? {
                             ...hinted,
-                            stdout: `installed: ${packages?.join(', ')}\nnothing was run — send the Python source that builds the deck in command\n${hinted.stdout}`
+                            stdout: `installed: ${packages?.join(', ')}\nnothing was run — call exec again with language "python", packages: ${JSON.stringify(packages ?? [])} and the Python source in command\n${hinted.stdout}`
                         }
                         : hinted;
                     const undelivered = result.undeclared?.length

package/dist-lib/lib/tools/exec/adapters/python/run-session.d.ts

@@ -4,6 +4,8 @@ export type PyodideLike = {
         filename?: string;
     }): unknown;
     runPythonAsync(code: string): Promise<unknown>;
+    ffi?: unknown;
+    pyimport?(name: string): unknown;
     loadPackage(name: string): Promise<unknown>;
     setStdout(options: {
         batched: (output: string) => void;
@@ -62,6 +64,7 @@ export type RunSpec = {
     readonly workdir?: string | undefined;
     readonly maxOutputChars: number;
     readonly installTimeoutMs: number;
+    readonly network?: boolean | undefined;
 };
 export type RunOutcome = {
     readonly result: PythonRunResult;

package/dist-lib/lib/tools/exec/adapters/python/run-session.js

@@ -2,6 +2,8 @@ import { AbortedError } from '../../../../helpers/race-abort.js';
 import { hlog } from '../../../../log.js';
 import { TRUNCATED_OUTPUT_SUFFIX } from '../../runner.js';
 import { collectOutputs, openScratch, stageInputFiles } from './memfs.js';
+import { isGuardedFetch } from './guarded-fetch.js';
+import { NET_SHIM, pythonReachesNetwork } from './net-shim.js';
 import { planVendoredInstall, stageVendoredWheels } from './wheels.js';
 export function cap(text, limit) {
     if (text.length <= limit)
@@ -37,9 +39,24 @@ export function repairDoubleEscapedSource(code, message) {
     const repaired = unescapeSource(code);
     return repaired === code ? null : repaired;
 }
-function attempt(py, code, output, maxOutputChars) {
-    try {
+const promisingEval = new WeakMap();
+async function enter(py, code, network) {
+    const guarded = pythonReachesNetwork() && isGuardedFetch(globalThis.fetch);
+    if (!network || !guarded || py.ffi === undefined || py.pyimport === undefined) {
         py.runPython(code);
+        return;
+    }
+    let evalCode = promisingEval.get(py);
+    if (!evalCode) {
+        py.runPython(NET_SHIM);
+        evalCode = py.pyimport('pyodide.code').eval_code;
+        promisingEval.set(py, evalCode);
+    }
+    await evalCode.callPromising(code, py.globals);
+}
+async function attempt(py, code, output, maxOutputChars, network) {
+    try {
+        await enter(py, code, network);
         return {
             result: {
                 stdout: cap(output.stdout, maxOutputChars),
@@ -63,14 +80,14 @@ function attempt(py, code, output, maxOutputChars) {
         };
     }
 }
-function runOnce(py, code, output, maxOutputChars) {
-    const first = attempt(py, code, output, maxOutputChars);
+async function runOnce(py, code, output, maxOutputChars, network) {
+    const first = await attempt(py, code, output, maxOutputChars, network);
     if (first.error === null)
         return first;
     const repaired = repairDoubleEscapedSource(code, first.error);
     if (repaired === null)
         return first;
-    const retry = attempt(py, repaired, output, maxOutputChars);
+    const retry = await attempt(py, repaired, output, maxOutputChars, network);
     if (retry.error !== null && retry.error.includes('SyntaxError'))
         return first;
     return retry;
@@ -168,7 +185,7 @@ async function installPackages(py, packages, timeoutMs, isCancelled) {
             clearTimeout(timer);
     }
 }
-const DELIVERABLE = /\.(pptx|docx|xlsx|pdf|png|jpe?g|gif|webp|csv|json|txt|md|zip)$/i;
+const DELIVERABLE = /\.(pptx|docx|xlsx|pdf|png|jpe?g|gif|webp|csv|json|txt|md|zip|mp4|webm|m4a|mp3)$/i;
 function listFiles(py, dir) {
     try {
         return new Set(py.FS.readdir(dir).filter((name) => name !== '.' && name !== '..'));
@@ -234,7 +251,7 @@ export async function performRun(py, spec, opts = {}) {
     if (!spec.inputs?.length && !spec.outputs?.length && !spec.workdir) {
         const cwd = py.FS.cwd();
         const before = listFiles(py, cwd);
-        const { result, fatal } = runOnce(py, spec.code, output, spec.maxOutputChars);
+        const { result, fatal } = await runOnce(py, spec.code, output, spec.maxOutputChars, spec.network === true);
         if (result.exitCode !== 0)
             return { result, evict: fatal };
         const undeclared = newDeliverables(before, listFiles(py, cwd));
@@ -245,7 +262,7 @@ export async function performRun(py, spec, opts = {}) {
         py.FS.chdir(scratch.cwd);
         stageInputFiles(py.FS, scratch.cwd, spec.inputs ?? []);
         const before = listFiles(py, scratch.cwd);
-        const { result, fatal } = runOnce(py, spec.code, output, spec.maxOutputChars);
+        const { result, fatal } = await runOnce(py, spec.code, output, spec.maxOutputChars, spec.network === true);
         let evict = fatal;
         if (result.exitCode === 0)
             evict = fitDecks(py, spec.outputs ?? []);

package/dist-lib/lib/tools/exec/adapters/python/wheels.d.ts

@@ -3,6 +3,7 @@ declare const ManifestSchema: z.ZodObject<{
     pyodide: z.ZodString;
     packages: z.ZodRecord<z.ZodString, z.ZodObject<{
         version: z.ZodOptional<z.ZodString>;
+        imports: z.ZodOptional<z.ZodArray<z.ZodString>>;
         pypi: z.ZodArray<z.ZodString>;
         pyodide: z.ZodArray<z.ZodString>;
     }, z.core.$strip>>;
@@ -29,6 +30,11 @@ type WheelFs = {
 };
 export declare function wheelsDir(): string | undefined;
 export declare function vendoredManifest(dir?: string | undefined): WheelManifest | null;
+export declare function normalizeDistribution(name: string): string;
+export declare function vendoredSpec(distribution: string, opts?: {
+    readonly dir?: string | undefined;
+    readonly manifest?: WheelManifest | null;
+}): string | null;
 export declare function planVendoredInstall(specs: readonly string[], opts?: {
     readonly dir?: string | undefined;
     readonly manifest?: WheelManifest | null;

package/dist-lib/lib/tools/exec/adapters/python/wheels.js

@@ -7,6 +7,7 @@ const ManifestSchema = z.object({
     pyodide: z.string(),
     packages: z.record(z.string(), z.object({
         version: z.string().optional(),
+        imports: z.array(z.string()).optional(),
         pypi: z.array(z.string()),
         pyodide: z.array(z.string())
     })),
@@ -42,12 +43,25 @@ export function vendoredManifest(dir = wheelsDir()) {
         return null;
     }
 }
-function normalize(name) {
-    return name.toLowerCase().replace(/[_.]+/g, '-');
+export function normalizeDistribution(name) {
+    return name
+        .trim()
+        .toLowerCase()
+        .replace(/[-_.]+/g, '-');
 }
 function parseSpec(spec) {
     const [name, version] = spec.split('==');
-    return { name: normalize((name ?? '').trim()), version: version?.trim() };
+    return { name: normalizeDistribution(name ?? ''), version: version?.trim() };
+}
+export function vendoredSpec(distribution, opts = {}) {
+    const name = normalizeDistribution(distribution);
+    const dir = opts.dir ?? wheelsDir();
+    const manifest = opts.manifest === undefined ? vendoredManifest(dir) : opts.manifest;
+    const entry = manifest?.packages[name];
+    if (!entry)
+        return null;
+    const spec = entry.version ? `${name}==${entry.version}` : name;
+    return planVendoredInstall([spec], { dir, manifest }) ? spec : null;
 }
 export function planVendoredInstall(specs, opts = {}) {
     const dir = opts.dir ?? wheelsDir();

package/dist-lib/lib/tools/exec/adapters/shell/shell-adapter.js

@@ -1,4 +1,5 @@
 import { runCommand } from '../../runner.js';
+import { shellDownloadNote } from '../../shell-guidance.js';
 export function createShellAdapter(options) {
     return {
         resolve(args) {
@@ -8,11 +9,16 @@ export function createShellAdapter(options) {
                     const runtime = options.spawn
                         ? { spawn: options.spawn, killProcess: options.killProcess }
                         : await options.loadRuntime();
-                    return runCommand(command, {
+                    const result = await runCommand(command, {
                         ...execution,
                         spawn: runtime.spawn,
                         killProcess: runtime.killProcess
                     });
+                    const note = result.exitCode === 0 ? shellDownloadNote(command) : null;
+                    if (!note)
+                        return result;
+                    const stdout = `${result.stdout}${result.stdout && !result.stdout.endsWith('\n') ? '\n' : ''}${note}`;
+                    return { ...result, stdout };
                 }
             };
         }