@nimbus-sh/worker 0.5.0 → 0.7.0

package/dist/npm/install-batch-facet.js

@@ -33,16 +33,17 @@
  * Stability invariants (cloudflare-parallel):
  *   - No `this` references.
  *   - No closure capture other than args + preamble names.
- *   - Preamble symbols (streamTarEntries, readableStreamToAsyncIterable,
- *     MAX_FILE_BYTES) referenced via @ts-ignore.
+ *   - Preamble symbols (streamPackageEntries, streamTarEntries,
+ *     readableStreamToAsyncIterable, MAX_FILE_BYTES) referenced via
+ *     @ts-ignore.
  */
 // ── Facet function ──────────────────────────────────────────────────────
 //
 // Runs inside a IsolatePool isolate. Serialised via fn.toString();
-// the helpers it references at top-level scope (streamTarEntries,
-// readableStreamToAsyncIterable, MAX_FILE_BYTES) are NOT in the facet's
-// lexical scope — the pool injects them via the preamble. No static
-// imports of those names; references are bare identifiers.
+// the helpers it references at top-level scope (streamPackageEntries,
+// streamTarEntries, readableStreamToAsyncIterable, MAX_FILE_BYTES) are NOT
+// in the facet's lexical scope — the pool injects them via the preamble.
+// No static imports of those names; references are bare identifiers.
 export const installPackagesInFacet = async function installPackagesInFacet(batch, env) {
     const tBatchStart = Date.now();
     if (!batch || typeof batch !== 'object' || !Array.isArray(batch.packages)) {
@@ -121,6 +122,10 @@ export const installPackagesInFacet = async function installPackagesInFacet(batc
     let sharedChunks = [];
     let sharedBufferedBytes = 0;
     let sharedOwners = new Set();
+    // Directories staged into a wave, shard-wide. A directory is staged once
+    // per shard, not once per package — and a wave that fails removes its
+    // directories from this set so they are staged again (see doSharedFlush).
+    const landedDirs = new Set();
     const ownerWaves = new Map();
     const ownersWithCompletionMarker = new Set();
     const completionMarkers = new Map();
@@ -135,12 +140,20 @@ export const installPackagesInFacet = async function installPackagesInFacet(batc
     // for too long", so the schedule has to outlast a queue that deep; the
     // whole-batch timeout is 10 minutes, which bounds it.
     const WAVE_RETRY_BACKOFF_MS = [250, 1000, 3000, 6000, 12000, 20000];
+    // "Network connection lost" is the RPC transport between this shard and
+    // the coordinator DO dropping under load — the same class: the wave may
+    // or may not have run, and re-sending identical path-keyed bytes is safe
+    // either way. It was the dominant shard failure on a 629-package install
+    // (Markflow): one lost wave failed every package that had contributed to
+    // it, and the directories that wave carried never landed, so later waves
+    // of unrelated packages failed with ENOENT on those parents.
     const isSheddableWaveError = (message) => {
         const m = message.toLowerCase();
         return m.includes('overloaded')
             || m.includes('reset because its code was updated')
             || m.includes('starting up durable object storage')
-            || (m.includes('storage operation') && m.includes('reset'));
+            || (m.includes('storage operation') && m.includes('reset'))
+            || m.includes('network connection lost');
     };
     // Mutex: only one flush runs at a time. Concurrent installs awaiting
     // flush() will line up behind this promise and resolve in arrival
@@ -177,7 +190,7 @@ export const installPackagesInFacet = async function installPackagesInFacet(batc
         // with every contributing package before awaiting the RPC, so a package
         // cannot report success while another package happens to be the caller
         // that triggered its shared flush.
-        const wave = (async () => {
+        const sendWave = async () => {
             for (let attempt = 0;; attempt++) {
                 try {
                     // Each attempt encodes its OWN bytes. The encoder hands chunk
@@ -212,6 +225,19 @@ export const installPackagesInFacet = async function installPackagesInFacet(batc
                     await new Promise((rs) => setTimeout(rs, delayMs));
                 }
             }
+        };
+        const dirsInWave = inodesNow.filter((inode) => inode.isDir).map((inode) => inode.path);
+        const wave = (async () => {
+            const outcome = await sendWave();
+            // A wave that did not land took its directory inodes with it. Every
+            // package that staged one of them into this wave believes it exists;
+            // forget them so the next file under such a directory re-stages the
+            // chain into its own wave instead of failing ENOENT on a parent that
+            // was never written.
+            if (!outcome.ok)
+                for (const dir of dirsInWave)
+                    landedDirs.delete(dir);
+            return outcome;
         })();
         for (const owner of ownersNow) {
             let waves = ownerWaves.get(owner);
@@ -647,7 +673,6 @@ export const installPackagesInFacet = async function installPackagesInFacet(batc
             // not per package. Per-package totals stay local to the result object.
             let totalFileInodes = 0;
             let totalBytesWritten = 0;
-            const dirSet = new Set();
             let completionMarker = null;
             // Stage `installRoot` and every directory down to `dirPath` (inclusive),
             // root-to-leaf, BEFORE any file that needs them. The credentialed
@@ -663,9 +688,13 @@ export const installPackagesInFacet = async function installPackagesInFacet(batc
                 const segs = suffix ? suffix.split('/') : [];
                 for (let i = 0; i <= segs.length; i++) {
                     const d = i === 0 ? installRoot : installRoot + '/' + segs.slice(0, i).join('/');
-                    if (dirSet.has(d))
+                    if (landedDirs.has(d)) {
+                        // Staged by an earlier wave of this shard. This owner's files
+                        // depend on it, so this owner must see that wave's outcome too.
+                        sharedOwners.add(ownerId);
                         continue;
-                    dirSet.add(d);
+                    }
+                    landedDirs.add(d);
                     await enqueueSharedDirectory(ownerId, d, spec.mtime);
                 }
             };
@@ -685,10 +714,11 @@ export const installPackagesInFacet = async function installPackagesInFacet(batc
             // packages whose only member is package.json.
             await enqueueDirsUpTo(pkgDir);
             // @ts-ignore — preamble symbol.
-            for await (const entry of streamTarEntries(asyncIter, onSkip)) {
-                // entry.name is already canonicalized (no "."/".." segments) by
-                // the tar parser, so joining under the canonical pkgDir yields a
-                // canonical path the w7-frame writer accepts.
+            for await (const entry of streamPackageEntries(asyncIter, onSkip)) {
+                // entry.name is canonicalized (no "."/".." segments) and stripped of
+                // the archive's single top-level directory by streamPackageEntries,
+                // so joining under the canonical pkgDir yields a canonical path the
+                // w7-frame writer accepts.
                 const filePath = pkgDir + '/' + entry.name;
                 // Stage this file's parent-dir chain before the file itself.
                 await enqueueDirsUpTo(filePath.substring(0, filePath.lastIndexOf('/')));

package/dist/npm/installer.d.ts

@@ -23,7 +23,8 @@ import type { SqliteVFS } from '@nimbus-sh/core/vfs/sqlite-vfs.js';
 import type { EsbuildService } from '@nimbus-sh/core/runtime/esbuild-service.js';
 import { NpmCache } from './cache.js';
 import { type FetchFn } from './resolver.js';
-import { type NpmLogEmitter } from './npm-log.js';
+import { type NpmLogEmitter } from '@nimbus-sh/core/substrate/lifo/commands/system/npm-log.js';
+import type { BundlePoolProvider } from '../facets/esbuild-bundle-pool.js';
 import type { InstallPhase } from '@nimbus-sh/platform/install-phase.js';
 export interface InstallProgress {
     phase: InstallPhase;
@@ -49,6 +50,8 @@ export declare class NpmInstaller {
     private readonly vfs;
     private cache;
     private esbuild;
+    /** The session's esbuild facet pool, shared with the on-demand dev-server path. */
+    private bundlePool;
     private ctx;
     private env;
     private onProgress;
@@ -68,6 +71,7 @@ export declare class NpmInstaller {
     private npmLog;
     constructor(vfs: SqliteVFS, sql: SqlStorage, opts?: {
         esbuild?: EsbuildService;
+        bundlePool?: BundlePoolProvider;
         ctx?: DurableObjectState;
         env?: any;
         onProgress?: (msg: string) => void;
@@ -86,6 +90,7 @@ export declare class NpmInstaller {
         production?: boolean;
         pid?: number;
         npmLog?: NpmLogEmitter;
+        onProgress?: (msg: string) => void;
     }): Promise<NpmInstallResult>;
     private _installInner;
     /**
@@ -104,7 +109,13 @@ export declare class NpmInstaller {
      *   - X.5-G G1 optional-native silent-skip,
      *   - X.5-drizzle best-effort tagging on optional-peer subtrees,
      *   - W6 swap / warn / reject decisions (top-level enforcement; the
-     *     per-package task ALSO checks these for transitive correctness),
+     *     per-package task ALSO checks these for transitive correctness).
+     *     A package is required iff it is reachable from a required root
+     *     (every selected spec except optionalDependencies roots) through
+     *     a chain of required edges (`dependencies` minus the package's
+     *     own optionalDependencies, required peers); W6 refusals are
+     *     classified against the closure computed over the resolved
+     *     graph at end of walk.
      *   - cache flushing (one batched putRegistryEntries at end).
      *
      * The per-package task (resolveOnePackumentInFacet) owns ONLY the
@@ -139,24 +150,18 @@ export declare class NpmInstaller {
      *   the V8 cap risk.
      */
     private fetchViaBatchFacet;
-    /**
-     * Build the dependency specs from package.json + explicit packages.
-     */
-    /**
-     * W11: framework detection at install time. Reads package.json from the
-     * project root and runs detectFramework() against its deps + the basenames
-     * we can see in node_modules-adjacent siblings.
-     *
-     * Returns true if the project is detected as one of {next, astro, nuxt,
-     * remix, sveltekit, vite, wrangler}. False for 'unknown'.
-     */
-    private detectFrameworkAware;
     private buildSpecs;
     /**
      * W6: apply the PACKAGE_ABI_POLICY swap rewrites and reject deny list
-     * to a top-level spec map. Emits `[swap]` notices via onProgress; throws
-     * a multi-line error on any reject (with `transitive='warn'` rejects
-     * also failing at top level — they only soften at depth>0).
+     * to a top-level spec map. Emits `[swap]` notices via onProgress.
+     *
+     * G2: rejects are advisories — a listed package stays in the spec map
+     * and installs like any other (npm parity: it cannot run here, but
+     * install is the wrong place to say so). Each gets one `[npm] note:`
+     * line per install and an `advisory` registry event; the require-side
+     * refusal carries the reason when the artifact is invoked.
+     * `advised` dedupes across the top-level pass and the walk (the
+     * package still resolves and can surface transitively).
      *
      * Idempotent: running on already-swapped specs is a no-op.
      */

package/dist/npm/installer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"installer.d.ts","sourceRoot":"","sources":["../../src/npm/installer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,KAAK,EACV,SAAS,EAGV,MAAM,mCAAmC,CAAC;AAM3C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,4CAA4C,CAAC;AAEjF,OAAO,EAAE,QAAQ,EAAsB,MAAM,YAAY,CAAC;AAC1D,OAAO,EAEiC,KAAK,OAAO,EACnD,MAAM,eAAe,CAAC;AAEvB,OAAO,EAEL,KAAK,aAAa,EACnB,MAAM,cAAc,CAAC;AAuEtB,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sCAAsC,CAAC;AAEzE,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,YAAY,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;CACjB;AAaD,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAChC;AAID,qBAAa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAY;IAClC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAkB;IACtC,OAAO,CAAC,KAAK,CAAW;IACxB,OAAO,CAAC,OAAO,CAAwB;IACvC,OAAO,CAAC,GAAG,CAAiC;IAC5C,OAAO,CAAC,GAAG,CAAM;IACjB,OAAO,CAAC,UAAU,CAAsC;IACxD;;;;;;OAMG;IACH,OAAO,CAAC,OAAO,CAAsB;IACrC;;;;OAIG;IACH,OAAO,CAAC,MAAM,CAA2B;gBAGvC,GAAG,EAAE,SAAS,EACd,GAAG,EAAE,UAAU,EACf,IAAI,CAAC,EAAE;QACL,OAAO,CAAC,EAAE,cAAc,CAAC;QACzB,GAAG,CAAC,EAAE,kBAAkB,CAAC;QACzB,GAAG,CAAC,EAAE,GAAG,CAAC;QACV,UAAU,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;QACnC,OAAO,CAAC,EAAE,OAAO,CAAC;KACnB;IAYH,4EAA4E;IAC5E,IAAI,QAAQ,IAAI,QAAQ,CAAuB;IAI/C;;;;;OAKG;IACG,OAAO,CACX,UAAU,EAAE,MAAM,EAClB,IAAI,CAAC,EAAE;QACL,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;QACpB,UAAU,CAAC,EAAE,OAAO,CAAC;QACrB,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,MAAM,CAAC,EAAE,aAAa,CAAC;KACxB,GACA,OAAO,CAAC,gBAAgB,CAAC;YAkBd,aAAa;IAoV3B;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;YACW,oBAAoB;IAiVlC;;;;;;;;;;;;;;;;;;;;;;;OAuBG;YACW,kBAAkB;IAyNhC;;OAEG;IACH;;;;;;;OAOG;YACW,oBAAoB;YAgCpB,UAAU;IA6CxB;;;;;;;OAOG;IACH,OAAO,CAAC,eAAe;IAiCvB;;OAEG;IACH,OAAO,CAAC,eAAe;IAgCvB;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAqC1B;;OAEG;IACH,OAAO,CAAC,aAAa;IAqBrB;;OAEG;YACW,QAAQ;YA4DR,kBAAkB;IAehC,OAAO,CAAC,iBAAiB;IAuCzB;;;;;;;;;;OAUG;YACW,oBAAoB;IA8jBlC;;;;;;;;;;OAUG;IACH,OAAO,CAAC,eAAe;IAgFvB;;OAEG;IACH;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,uBAAuB;IAsD/B,2EAA2E;IAC3E,OAAO,CAAC,kBAAkB;IAI1B;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,OAAO,CAAC,0BAA0B;CAenC"}
+{"version":3,"file":"installer.d.ts","sourceRoot":"","sources":["../../src/npm/installer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,KAAK,EACV,SAAS,EAGV,MAAM,mCAAmC,CAAC;AAM3C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,4CAA4C,CAAC;AAEjF,OAAO,EAAE,QAAQ,EAAsB,MAAM,YAAY,CAAC;AAC1D,OAAO,EAEiC,KAAK,OAAO,EACnD,MAAM,eAAe,CAAC;AAGvB,OAAO,EAEL,KAAK,aAAa,EACnB,MAAM,2DAA2D,CAAC;AASnE,OAAO,KAAK,EAAc,kBAAkB,EAAE,MAAM,kCAAkC,CAAC;AA2DvF,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sCAAsC,CAAC;AAEzE,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,YAAY,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;CACjB;AAiCD,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAChC;AAID,qBAAa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAY;IAClC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAkB;IACtC,OAAO,CAAC,KAAK,CAAW;IACxB,OAAO,CAAC,OAAO,CAAwB;IACvC,mFAAmF;IACnF,OAAO,CAAC,UAAU,CAA4B;IAC9C,OAAO,CAAC,GAAG,CAAiC;IAC5C,OAAO,CAAC,GAAG,CAAM;IACjB,OAAO,CAAC,UAAU,CAAsC;IACxD;;;;;;OAMG;IACH,OAAO,CAAC,OAAO,CAAsB;IACrC;;;;OAIG;IACH,OAAO,CAAC,MAAM,CAA2B;gBAGvC,GAAG,EAAE,SAAS,EACd,GAAG,EAAE,UAAU,EACf,IAAI,CAAC,EAAE;QACL,OAAO,CAAC,EAAE,cAAc,CAAC;QACzB,UAAU,CAAC,EAAE,kBAAkB,CAAC;QAChC,GAAG,CAAC,EAAE,kBAAkB,CAAC;QACzB,GAAG,CAAC,EAAE,GAAG,CAAC;QACV,UAAU,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;QACnC,OAAO,CAAC,EAAE,OAAO,CAAC;KACnB;IAaH,4EAA4E;IAC5E,IAAI,QAAQ,IAAI,QAAQ,CAAuB;IAI/C;;;;;OAKG;IACG,OAAO,CACX,UAAU,EAAE,MAAM,EAClB,IAAI,CAAC,EAAE;QACL,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;QACpB,UAAU,CAAC,EAAE,OAAO,CAAC;QACrB,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,MAAM,CAAC,EAAE,aAAa,CAAC;QACvB,UAAU,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;KACpC,GACA,OAAO,CAAC,gBAAgB,CAAC;YAuBd,aAAa;IAyV3B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA8BG;YACW,oBAAoB;IAoXlC;;;;;;;;;;;;;;;;;;;;;;;OAuBG;YACW,kBAAkB;YAyNlB,UAAU;IAwDxB;;;;;;;;;;;;;OAaG;IACH,OAAO,CAAC,eAAe;IA2BvB;;OAEG;IACH,OAAO,CAAC,eAAe;IA+CvB;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAqC1B;;OAEG;IACH,OAAO,CAAC,aAAa;IAqBrB;;OAEG;YACW,QAAQ;YA4DR,kBAAkB;IAehC,OAAO,CAAC,iBAAiB;IAuCzB;;;;;;;;;;OAUG;YACW,oBAAoB;IAkiBlC;;;;;;;;;;OAUG;IACH,OAAO,CAAC,eAAe;IAgFvB;;OAEG;IACH;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,uBAAuB;IAsD/B,2EAA2E;IAC3E,OAAO,CAAC,kBAAkB;IAI1B;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,OAAO,CAAC,0BAA0B;CAenC"}