@nimbus-sh/worker 0.2.1 → 0.2.3

package/dist/npm/tarball-stream.js

@@ -1,235 +0,0 @@
-/**
- * npm-tarball-stream.ts — pure streaming tar primitives.
- *
- * Extracted from npm-tarball.ts so these helpers can be esbuild-bundled
- * WITHOUT dragging in NpmCache / NpmResolver / SqliteVFS. Consumed by
- * scripts/bundle-facet-workers.mjs which emits a string constant the
- * NimbusLoaderPool uses to inject the tar parser into dynamic workers.
- *
- * Zero dependencies. Works identically on the supervisor and inside a
- * facet isolate. Never buffers the full decompressed tarball — peak
- * transient heap is one file's bytes plus a 512-byte carry.
- */
-/**
- * Maximum size of a single file inside a tarball. Larger entries are skipped.
- *
- * History: 5 MB was too low — it silently dropped `esbuild-wasm/esbuild.wasm`
- * (11.35 MB on v0.24.2), which made Nimbus-in-Nimbus `npm run dev` fail with
- * `No such module "esbuild-wasm/esbuild.wasm"` since the missing file caused
- * esbuild's VFS plugin to mark the import `external`, and workerd's LOADER
- * has no entry for that specifier. 20 MB covers esbuild-wasm with headroom
- * while keeping per-facet peak heap bounded for the streaming extractor.
- */
-export const MAX_FILE_BYTES = 20_000_000;
-/**
- * Read one tar header (USTAR) out of `block`. Returns parsed fields or
- * `null` for an end-of-archive block (all zeros).
- */
-/**
- * Collapse "."/".." segments in a tar entry's package-relative path.
- * Returns the canonical relative path, or '' when the entry escapes its
- * package root (a leading ".." that pops above the root) — the caller
- * treats '' as a no-name entry and skips it. Mirrors the segment logic in
- * w7-frame's canonicalPath so joined write paths are always accepted.
- */
-export function canonicalTarName(name) {
-    const out = [];
-    for (const seg of name.split('/')) {
-        if (seg === '..') {
-            if (out.length === 0)
-                return '';
-            out.pop();
-        }
-        else if (seg !== '' && seg !== '.') {
-            out.push(seg);
-        }
-    }
-    return out.join('/');
-}
-export function parseTarHeader(block) {
-    if (block[0] === 0)
-        return null;
-    let name = '';
-    for (let i = 0; i < 100 && block[i] !== 0; i++) {
-        name += String.fromCharCode(block[i]);
-    }
-    let prefix = '';
-    for (let i = 345; i < 500 && block[i] !== 0; i++) {
-        prefix += String.fromCharCode(block[i]);
-    }
-    if (prefix)
-        name = prefix + '/' + name;
-    // Strip the npm `package/` convention.
-    name = name.replace(/^package\//, '');
-    // Canonicalize the entry-relative path. npm tarballs legitimately carry
-    // entries like "./dist/index.js" (agent-base, http-proxy-agent,
-    // protobufjs, ...); left as-is the "./" survives into the VFS write path
-    // and the w7-frame writer rejects the noncanonical path, failing the
-    // whole shared install wave and dropping shard-mates' completion markers.
-    // Collapsing here (the one place entry names are assembled) keeps every
-    // downstream join canonical. An entry that escapes its package root via
-    // ".." is dropped to '' → skipped as a no-name entry.
-    name = canonicalTarName(name);
-    let sizeStr = '';
-    for (let i = 124; i < 136 && block[i] !== 0; i++) {
-        sizeStr += String.fromCharCode(block[i]);
-    }
-    const size = parseInt(sizeStr.trim(), 8) || 0;
-    const typeFlag = block[156];
-    return { name, size, typeFlag };
-}
-/**
- * Wrap a `ReadableStream<Uint8Array>` as an async iterable. Workerd and
- * Node both support `Symbol.asyncIterator` on ReadableStream, but we
- * spell the reader loop out so we don't depend on ambient lib typings.
- */
-export async function* readableStreamToAsyncIterable(rs) {
-    const reader = rs.getReader();
-    try {
-        while (true) {
-            const { value, done } = await reader.read();
-            if (done)
-                return;
-            if (value && value.length > 0)
-                yield value;
-        }
-    }
-    finally {
-        try {
-            reader.releaseLock();
-        }
-        catch { /* ignore */ }
-    }
-}
-/**
- * Streaming tar extractor.
- *
- * Consumes an async iterable of Uint8Array chunks (the decompressed tar
- * byte stream) and yields one `{ name, data }` entry per regular file,
- * as each file completes.
- *
- * Memory invariant: holds at most one pending file's bytes (≤ MAX_FILE_BYTES)
- * plus a small carry buffer for the tar header being assembled.
- *
- * Skips: symlinks, directories, hardlinks, long-name extensions (PaxHeader),
- * and any file whose declared size exceeds MAX_FILE_BYTES.
- *
- * If `onSkip` is provided, it is invoked for each skipped entry with the
- * name, declared size, and reason code. Callers that need to surface
- * dropped-file warnings to users should pass one; legacy callers that
- * omit the arg still behave exactly as before (silent skip).
- */
-export async function* streamTarEntries(source, onSkip) {
-    let carry = new Uint8Array(0);
-    let state = { kind: 'header' };
-    function concat(a, b) {
-        if (a.length === 0)
-            return b;
-        if (b.length === 0)
-            return a;
-        const out = new Uint8Array(a.length + b.length);
-        out.set(a, 0);
-        out.set(b, a.length);
-        return out;
-    }
-    for await (const chunkRaw of source) {
-        let buf = concat(carry, chunkRaw);
-        let cursor = 0;
-        while (true) {
-            if (state.kind === 'header') {
-                if (buf.length - cursor < 512)
-                    break;
-                const header = buf.subarray(cursor, cursor + 512);
-                const parsed = parseTarHeader(header);
-                cursor += 512;
-                if (!parsed)
-                    return; // end-of-archive
-                const { name, size, typeFlag } = parsed;
-                const pad = size === 0 ? 0 : (512 - (size % 512)) % 512;
-                const isRegularFile = (typeFlag === 48 /* '0' */ || typeFlag === 0);
-                if (size === 0) {
-                    if (isRegularFile && name) {
-                        yield { name, data: new Uint8Array(0) };
-                    }
-                    state = { kind: 'header' };
-                    continue;
-                }
-                if (!isRegularFile || !name || size > MAX_FILE_BYTES) {
-                    if (onSkip) {
-                        // Classify before entering the skip state so the reason is
-                        // exact. Precedence matches the condition order above:
-                        // non-regular first (directories/symlinks/PaxHeaders are
-                        // skipped regardless of size), then no-name, then too-large.
-                        const reason = !isRegularFile
-                            ? 'non-regular'
-                            : !name
-                                ? 'no-name'
-                                : 'too-large';
-                        try {
-                            onSkip(name, size, reason);
-                        }
-                        catch { /* best-effort */ }
-                    }
-                    state = { kind: 'skip', remaining: size + pad };
-                    continue;
-                }
-                state = {
-                    kind: 'file',
-                    name,
-                    remaining: size,
-                    fileBuf: new Uint8Array(size),
-                    fileOffset: 0,
-                    pad,
-                    skip: false,
-                };
-                continue;
-            }
-            if (state.kind === 'file') {
-                const avail = buf.length - cursor;
-                if (avail === 0)
-                    break;
-                if (state.remaining > 0) {
-                    const take = Math.min(state.remaining, avail);
-                    state.fileBuf.set(buf.subarray(cursor, cursor + take), state.fileOffset);
-                    state.fileOffset += take;
-                    state.remaining -= take;
-                    cursor += take;
-                    if (state.remaining > 0)
-                        break;
-                }
-                if (state.pad > 0) {
-                    const avail2 = buf.length - cursor;
-                    if (avail2 === 0)
-                        break;
-                    const take = Math.min(state.pad, avail2);
-                    state.pad -= take;
-                    cursor += take;
-                    if (state.pad > 0)
-                        break;
-                }
-                yield { name: state.name, data: state.fileBuf };
-                state = { kind: 'header' };
-                continue;
-            }
-            // state.kind === 'skip'
-            const avail = buf.length - cursor;
-            if (avail === 0)
-                break;
-            const take = Math.min(state.remaining, avail);
-            cursor += take;
-            state.remaining -= take;
-            if (state.remaining > 0)
-                break;
-            state = { kind: 'header' };
-        }
-        if (cursor >= buf.length) {
-            carry = new Uint8Array(0);
-        }
-        else if (cursor === 0) {
-            carry = buf;
-        }
-        else {
-            carry = buf.slice(cursor);
-        }
-    }
-}

package/dist/npm/tarball.d.ts

@@ -1,34 +0,0 @@
-/**
- * npm-tarball.ts — streaming tarball extraction (gunzip + tar).
- *
- * `extractTarball` / `extractTarballFromResponse` walk a gzipped tar stream
- * entry-by-entry, holding at most one file's bytes at a time. Used by the
- * install-batch facet (whose own 128 MiB envelope, not the supervisor's,
- * holds the bytes) and the ruby-gems runtime.
- *
- * The legacy supervisor-resident `fetchWaves` / `buildBatchPayload` and the
- * `buildCacheRestorePayload` fast path were removed — they ran in supervisor
- * heap and held tarball bytes long enough to OOM the DO on large installs.
- * The single batch-facet path (install-batch-facet.ts) supersedes them and
- * consults the shared L2/L3 tarball cache directly.
- */
-/**
- * Streaming extractor driven by a `Response` body.
- *
- * Pipes `resp.body` through `DecompressionStream('gzip')` (npm tarballs are
- * always gzipped) and walks the tar stream entry-by-entry. Never buffers the
- * full decompressed tarball — peak transient heap is one file's bytes plus a
- * small carry buffer.
- *
- * The returned Map is per-file Uint8Arrays. If the response has no body
- * (unusual but possible with some proxies), we fall back to `arrayBuffer()`
- * + `extractTarball` so we still make progress.
- */
-export declare function extractTarballFromResponse(resp: Response): Promise<Map<string, Uint8Array>>;
-/**
- * Legacy buffered extractor. Kept for code paths that receive a fully-buffered
- * tarball (e.g. the tarball cache restore path, which already stores bytes in
- * SQLite as Uint8Arrays). New install paths use `streamTarEntries` instead.
- */
-export declare function extractTarball(tarball: ArrayBuffer): Promise<Map<string, Uint8Array>>;
-//# sourceMappingURL=tarball.d.ts.map

package/dist/npm/tarball.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"tarball.d.ts","sourceRoot":"","sources":["../../src/npm/tarball.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAeH;;;;;;;;;;;GAWG;AACH,wBAAsB,0BAA0B,CAC9C,IAAI,EAAE,QAAQ,GACb,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAmClC;AAED;;;;GAIG;AACH,wBAAsB,cAAc,CAClC,OAAO,EAAE,WAAW,GACnB,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAsBlC"}

package/dist/npm/tarball.js

@@ -1,97 +0,0 @@
-/**
- * npm-tarball.ts — streaming tarball extraction (gunzip + tar).
- *
- * `extractTarball` / `extractTarballFromResponse` walk a gzipped tar stream
- * entry-by-entry, holding at most one file's bytes at a time. Used by the
- * install-batch facet (whose own 128 MiB envelope, not the supervisor's,
- * holds the bytes) and the ruby-gems runtime.
- *
- * The legacy supervisor-resident `fetchWaves` / `buildBatchPayload` and the
- * `buildCacheRestorePayload` fast path were removed — they ran in supervisor
- * heap and held tarball bytes long enough to OOM the DO on large installs.
- * The single batch-facet path (install-batch-facet.ts) supersedes them and
- * consults the shared L2/L3 tarball cache directly.
- */
-import { streamTarEntries, readableStreamToAsyncIterable, } from './tarball-stream.js';
-// ── Tarball extraction ──────────────────────────────────────────────────
-//
-// The streaming primitives (parseTarHeader, streamTarEntries,
-// readableStreamToAsyncIterable, MAX_FILE_BYTES) live in
-// ./npm-tarball-stream.ts — a dependency-free leaf so
-// scripts/bundle-facet-workers.mjs can esbuild it into a string the facet
-// pool injects into dynamic workers. Import them directly from that module.
-/**
- * Streaming extractor driven by a `Response` body.
- *
- * Pipes `resp.body` through `DecompressionStream('gzip')` (npm tarballs are
- * always gzipped) and walks the tar stream entry-by-entry. Never buffers the
- * full decompressed tarball — peak transient heap is one file's bytes plus a
- * small carry buffer.
- *
- * The returned Map is per-file Uint8Arrays. If the response has no body
- * (unusual but possible with some proxies), we fall back to `arrayBuffer()`
- * + `extractTarball` so we still make progress.
- */
-export async function extractTarballFromResponse(resp) {
-    const files = new Map();
-    const body = resp.body;
-    if (!body) {
-        // Fallback: no streamable body (shouldn't happen, but handle it).
-        const buf = await resp.arrayBuffer();
-        return extractTarball(buf);
-    }
-    // Time-bound the whole extraction to detect hung gunzip streams.
-    const controller = new AbortController();
-    const timer = setTimeout(() => controller.abort(), 60_000);
-    try {
-        const decompressed = body.pipeThrough(new DecompressionStream('gzip'));
-        const source = readableStreamToAsyncIterable(decompressed);
-        for await (const entry of streamTarEntries(source)) {
-            if (controller.signal.aborted)
-                throw new Error('tarball extract timeout');
-            files.set(entry.name, entry.data);
-        }
-    }
-    catch (e) {
-        // On any error, just return whatever we managed to extract. Callers
-        // treat `files.size === 0` as a failed fetch and mark the package failed.
-        // If partial data is present (some files succeeded before the stream
-        // broke), we propagate it; the installer will re-fetch on mismatch.
-        if (files.size === 0) {
-            clearTimeout(timer);
-            throw e;
-        }
-    }
-    finally {
-        clearTimeout(timer);
-    }
-    return files;
-}
-/**
- * Legacy buffered extractor. Kept for code paths that receive a fully-buffered
- * tarball (e.g. the tarball cache restore path, which already stores bytes in
- * SQLite as Uint8Arrays). New install paths use `streamTarEntries` instead.
- */
-export async function extractTarball(tarball) {
-    const files = new Map();
-    const raw = new Uint8Array(tarball);
-    // Adapter: wrap the single buffer as an async iterable. If gzipped, pipe
-    // through DecompressionStream so the streaming parser still sees tar bytes.
-    let source;
-    if (raw[0] === 0x1f && raw[1] === 0x8b) {
-        const rs = new Blob([tarball]).stream().pipeThrough(new DecompressionStream('gzip'));
-        source = readableStreamToAsyncIterable(rs);
-    }
-    else {
-        source = (async function* () { yield raw; })();
-    }
-    try {
-        for await (const entry of streamTarEntries(source)) {
-            files.set(entry.name, entry.data);
-        }
-    }
-    catch {
-        return files;
-    }
-    return files;
-}

package/dist/runtime/clang-runner.d.ts

@@ -1,37 +0,0 @@
-/**
- * clang-runner.ts — compile, link, and execute C programs for Nimbus WASI.
- *
- * Architecture (compile-link-run, two facet calls):
- *
- *   compile  : clang.wasm + sysroot subset for C includes + user .c
- *              → produces .o bytes (returned to supervisor).
- *   link     : lld.wasm + sysroot subset for link (crt1.o + libc.a)
- *              + .o from compile → produces .wasm executable.
- *   write    : final .wasm flushed to user VFS at the requested path.
- *
- * The filesystem both halves see is the one WASI layer every other
- * non-node runtime uses (wasi-instance.ts), seeded and sealed.
- *
- * Splitting compile and link into separate LOADER calls keeps each
- * call under the empirical loader payload ceiling. Each ships:
- *
- *   - compile: 31 MiB clang.wasm + ~1.3 MiB sysroot subset (C includes).
- *   - link   : 19 MiB lld.wasm + ~0.75 MiB libs + tiny .o.
- *
- * Sysroot subset extraction happens supervisor-side via a small ustar
- * parser. The full sysroot.tar is parsed once when the clang runtime
- * warms for a session; compile/link calls reuse the filtered subsets.
- *
- * Dispatch stays direct: no sleeps, no caller-side retries, and no
- * catch-and-continue around loader failures.
- */
-import type { RuntimeManifest } from '@nimbus-sh/core/runtime/runtime-manifest.js';
-import type { SqliteVFS } from '@nimbus-sh/core/vfs/sqlite-vfs.js';
-import type { FacetManager } from '../facets/manager.js';
-/** Build the runner factory. Closes over facetMgr + vfs. */
-export declare function makeClangRunnerFactory(deps: {
-    facetMgr: FacetManager;
-    vfs: SqliteVFS;
-}): (manifest: RuntimeManifest, installRoot: string, binName: string, binKind: string | undefined) => (ctx: any) => Promise<number>;
-export declare const CLANG_RUNNER_PREAMBLE: string;
-//# sourceMappingURL=clang-runner.d.ts.map

package/dist/runtime/clang-runner.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"clang-runner.d.ts","sourceRoot":"","sources":["../../src/runtime/clang-runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,6CAA6C,CAAC;AACnF,OAAO,KAAK,EAAmB,SAAS,EAAE,MAAM,mCAAmC,CAAC;AACpF,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AASzD,4DAA4D;AAC5D,wBAAgB,sBAAsB,CAAC,IAAI,EAAE;IAC3C,QAAQ,EAAE,YAAY,CAAC;IACvB,GAAG,EAAE,SAAS,CAAC;CAChB,GAAG,CAAC,QAAQ,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,SAAS,KAC7F,CAAC,GAAG,EAAE,GAAG,KAAK,OAAO,CAAC,MAAM,CAAC,CAkThC;AAyjBD,eAAO,MAAM,qBAAqB,QAAiE,CAAC"}