@nimbus-sh/core 0.3.0 → 0.5.0

package/src/_shared/tarball-stream.ts

@@ -0,0 +1,263 @@
+/**
+ * tarball-stream.ts — pure streaming tar primitives.
+ *
+ * A leaf with no imports at all, deliberately: `bundle-facet-workers.mjs`
+ * esbuilds this file into a string constant the loader pool injects into
+ * dynamic workers, and a facet isolate resolves no specifier. Anything this
+ * file imported would have to travel with it.
+ *
+ * Zero dependencies. Works identically on the supervisor and inside a
+ * facet isolate. Never buffers the full decompressed tarball — peak
+ * transient heap is one file's bytes plus a 512-byte carry.
+ */
+
+/**
+ * Maximum size of a single file inside a tarball. Larger entries are skipped.
+ *
+ * History: 5 MB was too low — it silently dropped `esbuild-wasm/esbuild.wasm`
+ * (11.35 MB on v0.24.2), which made Nimbus-in-Nimbus `npm run dev` fail with
+ * `No such module "esbuild-wasm/esbuild.wasm"` since the missing file caused
+ * esbuild's VFS plugin to mark the import `external`, and workerd's LOADER
+ * has no entry for that specifier. 20 MB covers esbuild-wasm with headroom
+ * while keeping per-facet peak heap bounded for the streaming extractor.
+ */
+export const MAX_FILE_BYTES = 20_000_000;
+
+/**
+ * Read one tar header (USTAR) out of `block`. Returns parsed fields or
+ * `null` for an end-of-archive block (all zeros).
+ */
+/**
+ * Collapse "."/".." segments in a tar entry's package-relative path.
+ * Returns the canonical relative path, or '' when the entry escapes its
+ * package root (a leading ".." that pops above the root) — the caller
+ * treats '' as a no-name entry and skips it. Mirrors the segment logic in
+ * w7-frame's canonicalPath so joined write paths are always accepted.
+ */
+export function canonicalTarName(name: string): string {
+  const out: string[] = [];
+  for (const seg of name.split('/')) {
+    if (seg === '..') {
+      if (out.length === 0) return '';
+      out.pop();
+    } else if (seg !== '' && seg !== '.') {
+      out.push(seg);
+    }
+  }
+  return out.join('/');
+}
+
+export function parseTarHeader(block: Uint8Array): { name: string; size: number; typeFlag: number } | null {
+  if (block[0] === 0) return null;
+
+  let name = '';
+  for (let i = 0; i < 100 && block[i] !== 0; i++) {
+    name += String.fromCharCode(block[i]);
+  }
+  let prefix = '';
+  for (let i = 345; i < 500 && block[i] !== 0; i++) {
+    prefix += String.fromCharCode(block[i]);
+  }
+  if (prefix) name = prefix + '/' + name;
+  // Strip the npm `package/` convention.
+  name = name.replace(/^package\//, '');
+  // Canonicalize the entry-relative path. npm tarballs legitimately carry
+  // entries like "./dist/index.js" (agent-base, http-proxy-agent,
+  // protobufjs, ...); left as-is the "./" survives into the VFS write path
+  // and the w7-frame writer rejects the noncanonical path, failing the
+  // whole shared install wave and dropping shard-mates' completion markers.
+  // Collapsing here (the one place entry names are assembled) keeps every
+  // downstream join canonical. An entry that escapes its package root via
+  // ".." is dropped to '' → skipped as a no-name entry.
+  name = canonicalTarName(name);
+
+  let sizeStr = '';
+  for (let i = 124; i < 136 && block[i] !== 0; i++) {
+    sizeStr += String.fromCharCode(block[i]);
+  }
+  const size = parseInt(sizeStr.trim(), 8) || 0;
+  const typeFlag = block[156];
+  return { name, size, typeFlag };
+}
+
+/**
+ * Wrap a `ReadableStream<Uint8Array>` as an async iterable. Workerd and
+ * Node both support `Symbol.asyncIterator` on ReadableStream, but we
+ * spell the reader loop out so we don't depend on ambient lib typings.
+ */
+export async function* readableStreamToAsyncIterable(
+  rs: ReadableStream<Uint8Array>,
+): AsyncGenerator<Uint8Array, void, undefined> {
+  const reader = rs.getReader();
+  try {
+    while (true) {
+      const { value, done } = await reader.read();
+      if (done) return;
+      if (value && value.length > 0) yield value;
+    }
+  } finally {
+    try { reader.releaseLock(); } catch { /* ignore */ }
+  }
+}
+
+/**
+ * Reason a tar entry was skipped and never yielded. Surfaced to callers
+ * via the optional `onSkip` callback on `streamTarEntries`.
+ *
+ * - 'too-large': size > MAX_FILE_BYTES. The most common reason Nimbus
+ *   cares about — it's what caused esbuild-wasm/esbuild.wasm to vanish
+ *   silently in Nimbus-in-Nimbus before the cap was raised.
+ * - 'non-regular': typeFlag indicates a symlink / hardlink / directory /
+ *   PaxHeader / GNU LongName etc. These aren't files we stage.
+ * - 'no-name': header parsed but name was empty (malformed or a PaxHeader
+ *   that our parser didn't recognize as non-regular).
+ */
+export type TarSkipReason = 'too-large' | 'non-regular' | 'no-name';
+
+/**
+ * Optional skip-observer passed to `streamTarEntries`. Called ONCE per
+ * skipped entry, with the declared name (may be empty for 'no-name'
+ * skips) and the declared size in bytes.
+ *
+ * Consumers typically push these into a per-package warnings array so
+ * users see what wasn't installed. The callback is synchronous and
+ * must not throw — thrown errors are swallowed to keep the extractor
+ * best-effort.
+ */
+export type TarSkipCallback = (
+  name: string,
+  size: number,
+  reason: TarSkipReason,
+) => void;
+
+/**
+ * Streaming tar extractor.
+ *
+ * Consumes an async iterable of Uint8Array chunks (the decompressed tar
+ * byte stream) and yields one `{ name, data }` entry per regular file,
+ * as each file completes.
+ *
+ * Memory invariant: holds at most one pending file's bytes (≤ MAX_FILE_BYTES)
+ * plus a small carry buffer for the tar header being assembled.
+ *
+ * Skips: symlinks, directories, hardlinks, long-name extensions (PaxHeader),
+ * and any file whose declared size exceeds MAX_FILE_BYTES.
+ *
+ * If `onSkip` is provided, it is invoked for each skipped entry with the
+ * name, declared size, and reason code. Callers that need to surface
+ * dropped-file warnings to users should pass one; legacy callers that
+ * omit the arg still behave exactly as before (silent skip).
+ */
+export async function* streamTarEntries(
+  source: AsyncIterable<Uint8Array>,
+  onSkip?: TarSkipCallback,
+): AsyncGenerator<{ name: string; data: Uint8Array }, void, undefined> {
+  let carry: Uint8Array<ArrayBufferLike> = new Uint8Array(0);
+
+  type State =
+    | { kind: 'header' }
+    | { kind: 'file'; name: string; remaining: number; fileBuf: Uint8Array; fileOffset: number; pad: number; skip: boolean }
+    | { kind: 'skip'; remaining: number };
+  let state: State = { kind: 'header' };
+
+  function concat(a: Uint8Array, b: Uint8Array): Uint8Array {
+    if (a.length === 0) return b;
+    if (b.length === 0) return a;
+    const out = new Uint8Array(a.length + b.length);
+    out.set(a, 0);
+    out.set(b, a.length);
+    return out;
+  }
+
+  for await (const chunkRaw of source) {
+    let buf = concat(carry, chunkRaw);
+    let cursor = 0;
+
+    while (true) {
+      if (state.kind === 'header') {
+        if (buf.length - cursor < 512) break;
+        const header = buf.subarray(cursor, cursor + 512);
+        const parsed = parseTarHeader(header);
+        cursor += 512;
+        if (!parsed) return; // end-of-archive
+        const { name, size, typeFlag } = parsed;
+        const pad = size === 0 ? 0 : (512 - (size % 512)) % 512;
+        const isRegularFile = (typeFlag === 48 /* '0' */ || typeFlag === 0);
+        if (size === 0) {
+          if (isRegularFile && name) {
+            yield { name, data: new Uint8Array(0) };
+          }
+          state = { kind: 'header' };
+          continue;
+        }
+        if (!isRegularFile || !name || size > MAX_FILE_BYTES) {
+          if (onSkip) {
+            // Classify before entering the skip state so the reason is
+            // exact. Precedence matches the condition order above:
+            // non-regular first (directories/symlinks/PaxHeaders are
+            // skipped regardless of size), then no-name, then too-large.
+            const reason: TarSkipReason = !isRegularFile
+              ? 'non-regular'
+              : !name
+              ? 'no-name'
+              : 'too-large';
+            try { onSkip(name, size, reason); } catch { /* best-effort */ }
+          }
+          state = { kind: 'skip', remaining: size + pad };
+          continue;
+        }
+        state = {
+          kind: 'file',
+          name,
+          remaining: size,
+          fileBuf: new Uint8Array(size),
+          fileOffset: 0,
+          pad,
+          skip: false,
+        };
+        continue;
+      }
+
+      if (state.kind === 'file') {
+        const avail = buf.length - cursor;
+        if (avail === 0) break;
+        if (state.remaining > 0) {
+          const take = Math.min(state.remaining, avail);
+          state.fileBuf.set(buf.subarray(cursor, cursor + take), state.fileOffset);
+          state.fileOffset += take;
+          state.remaining -= take;
+          cursor += take;
+          if (state.remaining > 0) break;
+        }
+        if (state.pad > 0) {
+          const avail2 = buf.length - cursor;
+          if (avail2 === 0) break;
+          const take = Math.min(state.pad, avail2);
+          state.pad -= take;
+          cursor += take;
+          if (state.pad > 0) break;
+        }
+        yield { name: state.name, data: state.fileBuf };
+        state = { kind: 'header' };
+        continue;
+      }
+
+      // state.kind === 'skip'
+      const avail = buf.length - cursor;
+      if (avail === 0) break;
+      const take = Math.min(state.remaining, avail);
+      cursor += take;
+      state.remaining -= take;
+      if (state.remaining > 0) break;
+      state = { kind: 'header' };
+    }
+
+    if (cursor >= buf.length) {
+      carry = new Uint8Array(0);
+    } else if (cursor === 0) {
+      carry = buf;
+    } else {
+      carry = buf.slice(cursor);
+    }
+  }
+}

package/src/_shared/tarball.ts

@@ -0,0 +1,46 @@
+/**
+ * tarball.ts — a whole tarball, in memory, as a path→bytes map.
+ *
+ * The streaming primitives it walks with (`parseTarHeader`, `streamTarEntries`,
+ * `readableStreamToAsyncIterable`) live in `./tarball-stream.ts` — a
+ * dependency-free leaf, because `bundle-facet-workers.mjs` esbuilds that file
+ * into a string the loader pool injects into dynamic workers, where an import
+ * would not resolve.
+ *
+ * This half is for a caller that already holds the bytes: `gem install`, which
+ * fetches a `.gem` (a tar holding a gzipped tar) and needs both layers open at
+ * once. An installer streaming a tarball it is still downloading should drive
+ * `streamTarEntries` itself and never hold the whole thing.
+ */
+
+import {
+  streamTarEntries,
+  readableStreamToAsyncIterable,
+} from './tarball-stream.js';
+
+/** Extract every regular file. Gzipped input is decompressed first. */
+export async function extractTarball(
+  tarball: ArrayBuffer,
+): Promise<Map<string, Uint8Array>> {
+  const files = new Map<string, Uint8Array>();
+  const raw = new Uint8Array(tarball);
+
+  // Adapter: wrap the single buffer as an async iterable. If gzipped, pipe
+  // through DecompressionStream so the streaming parser still sees tar bytes.
+  let source: AsyncIterable<Uint8Array>;
+  if (raw[0] === 0x1f && raw[1] === 0x8b) {
+    const rs = new Blob([tarball]).stream().pipeThrough(new DecompressionStream('gzip'));
+    source = readableStreamToAsyncIterable(rs);
+  } else {
+    source = (async function* () { yield raw; })();
+  }
+
+  try {
+    for await (const entry of streamTarEntries(source)) {
+      files.set(entry.name, entry.data);
+    }
+  } catch {
+    return files;
+  }
+  return files;
+}

package/src/_shared/w7-frame.ts

@@ -893,7 +893,8 @@ const CRC_SEED = 0xffff_ffff;
 let crcTable: Uint32Array | null = null;
 
 function crc32Update(check: number, bytes: Uint8Array): number {
-  const table = crcTable ??= createCrcTable();
+  crcTable ??= createCrcTable();
+  const table = crcTable;
   let value = check;
   for (const byte of bytes) value = table[(value ^ byte) & 0xff] ^ (value >>> 8);
   return value >>> 0;

package/src/constants.ts

@@ -142,10 +142,13 @@ export const FACET_TIMEOUT_MS = 30_000;  // 30s execution timeout
 // JavaScript, so workerd's per-module text-size limit applies to the
 // JSON-escaped form (each `\n` / `\"` / `\u` adds bytes, plus the
 // per-key string-quote overhead).
-// raw → boots, 8 MiB raw → fails. Encoded as JSON that's roughly 18-25 MiB
-// of module text. We target 22 MiB encoded as the hard ceiling, leaving
-// ~2-3 MiB of headroom for the rest of the worker module (shims, runner
-// boot code) and any minor drift in the eviction loop's accounting.
+// Measured: 8 MiB raw failed to boot; the passing raw bound was measured
+// too, but the number was lost to comment truncation before it ever
+// reached git, so only the failing side is known. Encoded as JSON that's
+// roughly 18-25 MiB of module text. We target 22 MiB encoded as the hard
+// ceiling, leaving ~2-3 MiB of headroom for the rest of the worker module
+// (shims, runner boot code) and any minor drift in the eviction loop's
+// accounting.
 //
 // facet-manager.ts:buildPrefetchBundle uses TextEncoder().encode().length
 // to measure exact UTF-8 bytes (not JS string .length, which counts UTF-16
@@ -224,20 +227,31 @@ export const CF_COMPAT_DATE = '2026-04-01';
 
 // ── Supervisor heap budget [C'.1] ───────────────────────────────────────
 //
-// The supervisor isolate's 128 MiB workerd cap is a HARD platform ceiling
-// (per docs/research/cf-primitives-dossier.md §6 invariant I1 — 128 MiB
-// per V8 isolate, may be shared across same-class peer DOs co-tenanting in
-// one process). Nimbus targets HALF of that as a self-imposed soft ceiling
-// so the supervisor always has runway when workerd LRU-evicts neighbours
-// or AIR (Asynchronous Isolate Recreation) folds growing isolates.
+// Three distinct memory regimes, not one shared pool:
 //
-// 64 MiB is a budget, not a measurement, and nothing enforces it.
+//   - Supervisor DO isolate: 128 MiB HARD platform ceiling ("Each isolate
+//     can consume up to 128 MB of memory" —
+//     https://developers.cloudflare.com/workers/platform/limits/).
+//   - Facets: ~208-256 MiB EACH, measured — memory independent of the
+//     supervisor and of each other (1,664 MiB live across 8 facets +
+//     parent) but CPU SHARED across the actor thread. See
+//     packages/worker/src/loaders/process-fabric.ts.
+//   - Peer DOs: independent memory AND CPU budgets, measured 1.2 GiB
+//     live across 8 peers.
+//
+// Nimbus targets HALF of the supervisor's 128 MiB as a self-imposed soft
+// ceiling so the supervisor always has runway when workerd LRU-evicts
+// neighbours or AIR (Asynchronous Isolate Recreation) folds growing
+// isolates.
+//
+// 64 MiB is a soft admission budget, not a measurement — the right value
+// is unmeasured — and nothing enforces it directly.
 // src/observability/heap-estimate.ts sums the INSTRUMENTED contributors —
-// the supervisor baseline, VFS LRU and in-flight writes, pre-bundle slices,
-// and streaming RPC buffers — which is a lower bound, not full coverage.
-// The prefetch-bundle path in facets/manager.ts allocates against this
-// budget without accounting for it; see HEAP_BLIND_SPOTS for the current
-// gap. Read a low percentOfCeiling accordingly.
+// the supervisor baseline, VFS LRU and in-flight writes, pre-bundle
+// slices, streaming RPC buffers, and the prefetch-bundle build lease and
+// cache gauge — which is a lower bound, not full coverage; allocation
+// sites known to be missing are listed in HEAP_BLIND_SPOTS. Read a low
+// percentOfCeiling accordingly.
 export const SUPERVISOR_HEAP_CEILING_BYTES = 64 * 1024 * 1024;
 
 // Shared allowance for transient allocations in the supervisor DO. With the

package/src/observability/oom-classify.ts

@@ -68,6 +68,7 @@ const OOM_CAUSES = [
   'clone_refused',
   'rpc_timeout',
   'subrequest_cap',
+  'dynamic_worker_cap',
   'condemnation',
   'hard_evict',
   'unknown',
@@ -117,6 +118,12 @@ export function classifyMessage(msg: string): OomCause {
   // Subrequest cap (Cloudflare platform)
   if (m.includes('too many subrequests')) return 'subrequest_cap';
 
+  // Worker Loader concurrency cap (Cloudflare platform): a Durable Object
+  // admits ~5-6 concurrent dynamic workers, and loader-cache entries are
+  // never released — every distinct loader.get(id) permanently consumes a
+  // slot, so this cap recurs until the DO itself is replaced.
+  if (m.includes('too many concurrent dynamic workers')) return 'dynamic_worker_cap';
+
   // Memory exhaustion. workerd trace outcome `exceededMemory`; in-band it
   // arrives as a `broken.exceededMemory` actor break, a "…exceeded [its]
   // memory limit" kill, or the "Memory limit exceeded" RangeError.

package/src/observability/oom-discriminator.ts

@@ -128,7 +128,7 @@ export interface DiagRecoveryEvent {
   toState: SessionState;
   /** Human-readable trigger ('ws-close', 'isolate-evicted', etc.). */
   trigger: string;
-  /** _w9IsolateGen at the time of the event. Lets a probe detect the
+  /** _isolateGen at the time of the event. Lets a probe detect the
    *  difference between an in-isolate transition and a cross-isolate one
    *  (the latter implies workerd recycled the DO). */
   isolateGen: number;