@nimbus-sh/core 0.5.0 → 0.7.0

package/src/_shared/tarball.ts

@@ -1,5 +1,5 @@
 /**
- * tarball.ts — a whole tarball, in memory, as a path→bytes map.
+ * Tarball extraction for streaming installers and buffered archive consumers.
  *
  * The streaming primitives it walks with (`parseTarHeader`, `streamTarEntries`,
  * `readableStreamToAsyncIterable`) live in `./tarball-stream.ts` — a
@@ -7,10 +7,8 @@
  * into a string the loader pool injects into dynamic workers, where an import
  * would not resolve.
  *
- * This half is for a caller that already holds the bytes: `gem install`, which
- * fetches a `.gem` (a tar holding a gzipped tar) and needs both layers open at
- * once. An installer streaming a tarball it is still downloading should drive
- * `streamTarEntries` itself and never hold the whole thing.
+ * Installers use writeTarballStream. extractTarball retains a map for callers
+ * such as gem install, which must open an archive nested inside another one.
  */
 
 import {
@@ -18,6 +16,59 @@ import {
   readableStreamToAsyncIterable,
 } from './tarball-stream.js';
 
+export interface TarballWriteTarget {
+  exists(path: string): boolean;
+  mkdir(path: string, options?: { recursive?: boolean }): void;
+  writeFile(path: string, data: Uint8Array | string): unknown;
+}
+
+export interface TarballWriteResult {
+  /** Regular files written, including the manifest. */
+  files: number;
+  /** Total decompressed bytes written. */
+  bytes: number;
+}
+
+const PACKAGE_MANIFEST = 'package.json';
+
+/**
+ * Stream a gzipped npm archive into a package directory. Entry names are
+ * already canonical and prefix-stripped by streamTarEntries. Hold only the
+ * current entry and the manifest; write the manifest last so a failed install
+ * is not mistaken for a complete package on retry. Filesystem failures reject.
+ */
+export async function writeTarballStream(
+  body: ReadableStream<Uint8Array>,
+  targetDir: string,
+  vfs: TarballWriteTarget,
+): Promise<TarballWriteResult> {
+  const ensureDir = (path: string): void => {
+    if (!vfs.exists(path)) vfs.mkdir(path, { recursive: true });
+  };
+  ensureDir(targetDir);
+  let files = 0;
+  let bytes = 0;
+  let manifest: Uint8Array | null = null;
+  const entries = streamTarEntries(
+    readableStreamToAsyncIterable(body.pipeThrough(new DecompressionStream('gzip'))),
+  );
+  for await (const entry of entries) {
+    if (entry.name === PACKAGE_MANIFEST) {
+      manifest = entry.data;
+      continue;
+    }
+    const fullPath = `${targetDir}/${entry.name}`;
+    const cut = fullPath.lastIndexOf('/');
+    if (cut > 0) ensureDir(fullPath.slice(0, cut));
+    await vfs.writeFile(fullPath, entry.data);
+    files++;
+    bytes += entry.data.length;
+  }
+  if (!manifest) throw new Error(`tarball for ${targetDir} carried no ${PACKAGE_MANIFEST}`);
+  await vfs.writeFile(`${targetDir}/${PACKAGE_MANIFEST}`, manifest);
+  return { files: files + 1, bytes: bytes + manifest.length };
+}
+
 /** Extract every regular file. Gzipped input is decompressed first. */
 export async function extractTarball(
   tarball: ArrayBuffer,

package/src/_shared/vfs-write-ledger.ts

@@ -317,26 +317,32 @@ async function __nimbusPersistVfsWrite(supervisor, path, content, snapshot) {
 }
 
 /**
- * Write back everything parked, keeping any failure for the exit drain.
+ * Write back the cells parked at THIS instant, and only those.
  *
- * The write-back sites that are not the exit drain — the debounce below and
- * the RELEASE barrier ahead of egress — have no caller who could act on a
- * failure: there is no user frame to throw into, and rejecting the fetch that
- * happened to trigger the flush would blame the wrong operation. Retaining
- * the failure in the channel \`__nimbusDrainVfsMutations\` already drains means
- * the exit path still reports it, so a lost write is loud exactly once and
- * never silent.
+ * Bounded on purpose, and deliberately not routed through
+ * \`__nimbusDrainVfsWrites\`, whose \`while (pending > 0)\` waits for the mutation
+ * queue to be EMPTY. That wait is correct at process exit, where no new writes
+ * are coming. Anywhere else it is a livelock: a facet unpacking a tarball adds
+ * mutations faster than the loop retires them, so the loop never returns.
+ * Sited ahead of egress — where it was — that stopped the request from ever
+ * leaving the facet, and \`npx sv create\` ran, printed its intro, and then
+ * never reported an exit at all. A barrier may delay a request; it may not
+ * wait on a condition a busy process never reaches.
+ *
+ * Failures are retained rather than thrown. The two callers — the debounce
+ * below, and the RELEASE barrier ahead of egress — have no frame that could
+ * act on one: rejecting the fetch that happened to trigger the flush would
+ * blame the wrong operation. The exit drain reports what is retained, so a
+ * lost write is loud exactly once and never silent.
  */
 async function __nimbusFlushVfsWriteBack(supervisor) {
   if (!supervisor) return;
-  try {
-    await __nimbusDrainVfsWrites(supervisor);
-  } catch (error) {
-    if (!__nimbusHasPendingVfsMutationFailure) {
-      __nimbusHasPendingVfsMutationFailure = true;
-      __nimbusPendingVfsMutationFailure = error;
-    }
-  }
+  const paths = Object.keys(__vfsWrites);
+  if (paths.length === 0) return;
+  await Promise.allSettled(paths.map((path) => __nimbusFlushVfsWrite(
+    path,
+    (content, snapshot) => __nimbusPersistVfsWrite(supervisor, path, content, snapshot),
+  )));
 }
 
 /**

package/src/constants.ts

@@ -1,7 +1,32 @@
 /**
  * constants.ts — Single source of truth for all Nimbus configuration.
+ *
+ * Nimbus POLICY only. The measured platform limits these policies are
+ * derived from (tx bounds, the RPC envelope, supervisor budgets) live in
+ * `@nimbus-sh/platform/limits.js`.
  */
 
+import { MAX_RPC_SAFE_PAYLOAD_BYTES } from '@nimbus-sh/platform/limits.js';
+
+// Published-surface forwarding: @nimbus-sh/core@0.5.0 exports these from
+// this module and real consumers import them here (Proteus merge-back.ts
+// pulls CHUNK_SIZE and the MAX_TX_* set). The platform split moved the
+// definitions to @nimbus-sh/platform/limits.js, which stays the single
+// source of truth; core's own modules import platform directly.
+export {
+  CHUNK_SIZE,
+  MAX_GLOBAL_WRITE_STREAM_CREDIT_BYTES,
+  MAX_RPC_SAFE_PAYLOAD_BYTES,
+  MAX_TX_BLOB_BYTES,
+  MAX_TX_LOGICAL_ROWS,
+  MAX_TX_SQL_EXECS,
+  PRE_BUNDLE_CONCURRENCY,
+  PRE_BUNDLE_SLICE_CAP_BYTES,
+  SUPERVISOR_HEAP_CEILING_BYTES,
+  SUPERVISOR_IN_FLIGHT_ALLOCATION_BUDGET_BYTES,
+  SUPERVISOR_READ_RESERVE_BYTES,
+} from '@nimbus-sh/platform/limits.js';
+
 // ── Versions ────────────────────────────────────────────────────────────
 export const NIMBUS_VERSION = '2.0.0';
 //
@@ -55,19 +80,11 @@ export const SQLJS_VERSION = '1.14.1';
 export const OPENCODE_VERSION = '1.16.2';
 
 // ── VFS Constants ───────────────────────────────────────────────────────
-export const CHUNK_SIZE = 65_536;        // 64KB per content chunk
+// The chunk size itself (CHUNK_SIZE) and the transaction bounds are
+// platform limits — see @nimbus-sh/platform/limits.js.
 export const LRU_MAX_ENTRIES = 512;      // 512 × 64KB = 32MB hot cache
 export const BATCH_SIZE = 64;            // rows per batch INSERT
 export const VFS_CAPACITY = 10 * 1024 * 1024 * 1024; // 10 GB
-export const MAX_TX_BLOB_BYTES = 1 * 1024 * 1024;
-export const MAX_TX_LOGICAL_ROWS = 256;
-export const MAX_TX_SQL_EXECS = 64;
-export const MAX_GLOBAL_WRITE_STREAM_CREDIT_BYTES = 8 * 1024 * 1024;
-
-// Largest byte payload Nimbus sends through an ordinary Workers RPC value.
-// The platform limit is 32 MiB; 28 MiB leaves room for structured-clone
-// metadata and matches the proven on-demand facet transfer envelope.
-export const MAX_RPC_SAFE_PAYLOAD_BYTES = 28 * 1024 * 1024;
 
 // ── Batched filesystem reads ────────────────────────────────────────────
 // A round trip costs an order of magnitude more than the SQLite lookup
@@ -201,11 +218,12 @@ export const PREFETCH_CACHE_MAX_BYTES = 16 * 1024 * 1024;
 export const CWD_SNAPSHOT_MAX_FILE_BYTES = 2 * 1024 * 1024;
 
 // ── npm Constants ───────────────────────────────────────────────────────
+// The pre-bundle admission envelope (PRE_BUNDLE_SLICE_CAP_BYTES,
+// PRE_BUNDLE_CONCURRENCY) is a measured platform envelope — see
+// @nimbus-sh/platform/limits.js.
 export const NPM_REGISTRY = 'https://registry.npmjs.org';
 export const NPM_CONCURRENCY = 12;
 export const NPM_DECOMPRESS_TIMEOUT = 15_000;
-export const PRE_BUNDLE_SLICE_CAP_BYTES = MAX_RPC_SAFE_PAYLOAD_BYTES;
-export const PRE_BUNDLE_CONCURRENCY = 1;
 
 // ── Dev Server Constants ────────────────────────────────────────────────
 export const DEFAULT_VITE_PORT = 5173;
@@ -225,54 +243,6 @@ export const NIMBUS_AI_GATEWAY_PORT = 8790;
 // ── Compatibility ───────────────────────────────────────────────────────
 export const CF_COMPAT_DATE = '2026-04-01';
 
-// ── Supervisor heap budget [C'.1] ───────────────────────────────────────
-//
-// Three distinct memory regimes, not one shared pool:
-//
-//   - Supervisor DO isolate: 128 MiB HARD platform ceiling ("Each isolate
-//     can consume up to 128 MB of memory" —
-//     https://developers.cloudflare.com/workers/platform/limits/).
-//   - Facets: ~208-256 MiB EACH, measured — memory independent of the
-//     supervisor and of each other (1,664 MiB live across 8 facets +
-//     parent) but CPU SHARED across the actor thread. See
-//     packages/worker/src/loaders/process-fabric.ts.
-//   - Peer DOs: independent memory AND CPU budgets, measured 1.2 GiB
-//     live across 8 peers.
-//
-// Nimbus targets HALF of the supervisor's 128 MiB as a self-imposed soft
-// ceiling so the supervisor always has runway when workerd LRU-evicts
-// neighbours or AIR (Asynchronous Isolate Recreation) folds growing
-// isolates.
-//
-// 64 MiB is a soft admission budget, not a measurement — the right value
-// is unmeasured — and nothing enforces it directly.
-// src/observability/heap-estimate.ts sums the INSTRUMENTED contributors —
-// the supervisor baseline, VFS LRU and in-flight writes, pre-bundle
-// slices, streaming RPC buffers, and the prefetch-bundle build lease and
-// cache gauge — which is a lower bound, not full coverage; allocation
-// sites known to be missing are listed in HEAP_BLIND_SPOTS. Read a low
-// percentOfCeiling accordingly.
-export const SUPERVISOR_HEAP_CEILING_BYTES = 64 * 1024 * 1024;
-
-// Shared allowance for transient allocations in the supervisor DO. With the
-// VFS LRU shrunk to 8 MiB during an active reservation, 40 MiB of admitted
-// payload plus the 9 MiB bundle baseline stays below the 64 MiB soft ceiling
-// with 7 MiB left for metadata, structured-clone overhead, and runtime state.
-// This is 31.25% of the platform's 128 MiB hard isolate ceiling.
-export const SUPERVISOR_IN_FLIGHT_ALLOCATION_BUDGET_BYTES = 40 * 1024 * 1024;
-
-// Reserved slice of that allowance for chunk-sized filesystem reads, so a
-// read is never queued behind a multi-megabyte owner for a wait that has
-// nothing to do with its own cost. A read only draws on this when the shared
-// budget is already contended — which is exactly when the VFS LRU is shrunk
-// to 8 MiB — so peak accounting is 40 + 1 admitted, 8 LRU, 9 bundle baseline:
-// 58 MiB, still under the 64 MiB soft ceiling.
-//
-// 1 MiB is 16 concurrent READ_STREAM_CHUNK_BYTES reads. The point is not
-// depth, it is that a read loop keeps moving while a heavy owner works
-// instead of stopping dead for the duration.
-export const SUPERVISOR_READ_RESERVE_BYTES = 1024 * 1024;
-
 // ── OS Defaults ─────────────────────────────────────────────────────────
 export const DEFAULT_HOSTNAME = 'nimbus';
 export const DEFAULT_HOME = '/home/user';

package/src/runtime/assets-loader.ts

@@ -35,7 +35,7 @@
  * same content.
  */
 
-import { disposeRpcResource } from '../_shared/rpc-dispose.js';
+import { disposeRpcResource } from '@nimbus-sh/platform/rpc-dispose.js';
 
 /**
  * Per-isolate cache. Keys are asset paths (e.g. `/_assets/foo.js`).

package/src/runtime/comment-strip.ts

@@ -27,47 +27,66 @@
  * so it's a minor wasted-work cost — see IMPORT_RE header comment in
  * `require-resolver.ts:84-86`.
  *
+ * Memory: the output is assembled from slices of the input — one per
+ * comment-free span — and joined once. It used to be built one character
+ * at a time with `out += c`, which every JS engine represents as a rope
+ * of one node per append: ~30 bytes of heap per character of output, all
+ * of it live until the string is first read. The require walk runs this
+ * over every file in the closure, and on typescript's 6.15 MB `lib/_tsc.js`
+ * that rope measured 174 MB (V8) / 172 MB (JSC) against a 128 MB isolate —
+ * the session Durable Object was killed inside `tsc`'s spawn before the
+ * facet existed. A span-sliced build holds the input, the output and a
+ * short array of slices, and nothing else.
+ *
  * TODO(CLN-X): when the `src/runtime/esbuild-service.ts` anti-touch
  * window opens, unify with the canonical
  * `stripCommentsAndStrings(src)` helper there. Both implement the
  * same logical pass over comments; only the string/regex-literal
  * handling differs.
  */
+const SLASH = 0x2f;
+const STAR = 0x2a;
+const NEWLINE = 0x0a;
+
 export function stripCommentsForImports(src: string): string {
-  let out = '';
-  let i = 0;
   const N = src.length;
+  const parts: string[] = [];
+  // Start of the comment-free span not yet copied to `parts`.
+  let spanStart = 0;
+  let i = 0;
   while (i < N) {
-    const c = src[i];
-    // Line comment: `//` to end-of-line.
-    if (c === '/' && src[i + 1] === '/') {
-      out += ' ';
+    if (src.charCodeAt(i) !== SLASH) { i++; continue; }
+    const next = src.charCodeAt(i + 1);
+    // Line comment: `//` to end-of-line, replaced by one space. The
+    // newline is not part of the comment and stays in the next span.
+    if (next === SLASH) {
+      parts.push(src.slice(spanStart, i), ' ');
       i += 2;
-      while (i < N && src[i] !== '\n') i++;
-      // Newline preserved by falling through; do NOT consume it.
+      while (i < N && src.charCodeAt(i) !== NEWLINE) i++;
+      spanStart = i;
       continue;
     }
-    // Block comment: `/*` to `*/`.
-    if (c === '/' && src[i + 1] === '*') {
+    // Block comment: `/*` to `*/`, replaced by its own newlines (so line
+    // numbers stay aligned) and one space. An unterminated one runs to
+    // the end of input and gets no closing space.
+    if (next === STAR) {
+      parts.push(src.slice(spanStart, i));
       i += 2;
+      let newlines = 0;
+      let closed = false;
       while (i < N) {
-        if (src[i] === '\n') {
-          // Preserve newlines so line numbers stay aligned.
-          out += '\n';
-          i++;
-          continue;
-        }
-        if (src[i] === '*' && src[i + 1] === '/') {
-          i += 2;
-          out += ' ';
-          break;
-        }
+        const c = src.charCodeAt(i);
+        if (c === NEWLINE) newlines++;
+        else if (c === STAR && src.charCodeAt(i + 1) === SLASH) { i += 2; closed = true; break; }
         i++;
       }
+      if (newlines > 0) parts.push('\n'.repeat(newlines));
+      if (closed) parts.push(' ');
+      spanStart = i;
       continue;
     }
-    out += c;
     i++;
   }
-  return out;
+  parts.push(src.slice(spanStart));
+  return parts.join('');
 }