@nimbus-sh/worker 0.1.0

package/dist/npm/installer.d.ts

@@ -0,0 +1,239 @@
+/**
+ * npm-installer.ts — Unified npm installer for Nimbus.
+ *
+ * Designed for bun/pnpm-level performance on Cloudflare DO + SQLite.
+ *
+ * Pipeline:
+ *   Phase 0: Lock-check     — instant if lockfile valid
+ *   Phase 1: Resolve        — pipelined, 12-wide concurrency, registry cached
+ *   Phase 2: Hoist          — compute flat node_modules layout
+ *   Phase 3: Diff           — skip packages already cached
+ *   Phase 4: Fetch+Extract  — wave-based, 15 pkgs/wave, cache results
+ *   Phase 5: Write          — ONE transactionSync() per wave via writeBatch()
+ *   Phase 6: Link bins      — create node_modules/.bin/ entries
+ *   Phase 7: Pre-bundle     — scan source, esbuild used packages (background)
+ *
+ * Key invariants:
+ *   - All VFS writes go through writeBatch() (never individual writeFile)
+ *   - Tarball cache is per-package (name, version) — no cross-package dedup
+ *   - Lockfile stored in SQLite (not JSON file)
+ *   - ESM pre-bundles cached in SQLite for /@modules/ serving
+ */
+import type { SqliteVFS } from '../vfs/sqlite-vfs.js';
+import type { EsbuildService } from '../runtime/esbuild-service.js';
+import { NpmCache } from './cache.js';
+import { type FetchFn } from './resolver.js';
+export type { InstallPhase } from '../_shared/install-phase.js';
+import type { InstallPhase } from '../_shared/install-phase.js';
+export interface InstallProgress {
+    phase: InstallPhase;
+    resolved: number;
+    totalToResolve: number;
+    fetched: number;
+    totalToFetch: number;
+    written: number;
+    totalToWrite: number;
+    cachedHits: number;
+    elapsed: number;
+}
+export interface NpmInstallResult {
+    installed: string[];
+    failed: string[];
+    totalFiles: number;
+    elapsed: number;
+    cachedHits: number;
+    phases: Record<string, number>;
+}
+export declare class NpmInstaller {
+    private vfs;
+    private cache;
+    private esbuild;
+    private ctx;
+    private env;
+    private onProgress;
+    /**
+     * Injectable fetch function. Required because DO fetch() hangs in
+     * wrangler local dev. The caller (NimbusSession) provides a function
+     * that routes fetches through a facet worker. Used only by the resolve
+     * path (packument JSON) — tarball fetches happen inside the facet pool
+     * when the feature flag is on, using the facet's own global fetch.
+     */
+    private fetchFn;
+    constructor(vfs: SqliteVFS, sql: SqlStorage, opts?: {
+        esbuild?: EsbuildService;
+        ctx?: DurableObjectState;
+        env?: any;
+        onProgress?: (msg: string) => void;
+        fetchFn?: FetchFn;
+    });
+    /** Expose cache for external use (e.g., serveModule in vite-dev-server). */
+    get npmCache(): NpmCache;
+    /**
+     * Install packages for a project. Handles:
+     * - Lockfile-based fast path (no network if lock is valid)
+     * - Full resolution + fetch + write pipeline
+     * - Incremental: only fetches/writes what changed
+     */
+    install(projectDir: string, opts?: {
+        packages?: string[];
+        production?: boolean;
+    }): Promise<NpmInstallResult>;
+    private _installInner;
+    /**
+     * F-2 frontier-coordinator path. Replaces the single-resolve-facet
+     * dispatch with a per-package fanout: each BFS layer becomes ONE
+     * `NimbusFanoutPool.submitMany` call, layer N+1 builds from the
+     * resolved metadata of layer N.
+     *
+     * Topology auto-routes per layer:
+     *   width <  IN_DO_THRESHOLD (5)  → POC C in-DO loader-pool
+     *   width >= IN_DO_THRESHOLD       → POC B peer-DO (sibling NimbusSession DOs)
+     *
+     * The supervisor still owns:
+     *   - cycle detection (`seen`),
+     *   - X.5-F top-level / required-peer policy,
+     *   - X.5-G G1 optional-native silent-skip,
+     *   - X.5-drizzle best-effort tagging on optional-peer subtrees,
+     *   - W6 swap / warn / reject decisions (top-level enforcement; the
+     *     per-package task ALSO checks these for transitive correctness),
+     *   - cache flushing (one batched putRegistryEntries at end).
+     *
+     * The per-package task (resolveOnePackumentInFacet) owns ONLY the
+     * fetch + version pick + edge extraction. See
+     * audit/sections/F2-RESOLVER-FANOUT-plan.md.
+     *
+     * Anti-requirements (cleanup-not-done charter): NO setTimeout
+     * between layers, NO fallback to single-facet on missing bindings.
+     */
+    private resolveTreeViaFanout;
+    /**
+     * Batch install via two-tier fan-out (NimbusFanoutPool).
+     *
+     * Topology selected automatically based on the spec count:
+     *   specs.length <  IN_DO_THRESHOLD (5)  → POC C in-DO
+     *     1 NimbusLoaderPool with concurrency = specs.length, capped at
+     *     4 by V8 invariant. Each spec is one facet running its own
+     *     installPackagesInFacet over a single-element shard.
+     *   specs.length >= IN_DO_THRESHOLD       → POC B peer-DO
+     *     N peer NimbusSession sibling DOs (N = min(specs.length,
+     *     MAX_PEER_FANOUT = 32)), each running ONE installPackagesInFacet
+     *     against its own shard with internal pLimit(3).
+     *
+     * Sharding strategy: round-robin (`pkgIdx % N`) so every peer DO
+     *   receives roughly equal work. Stable-id router maps each
+     *   `shard-${i}` task key deterministically (tests can predict
+     *   placement).
+     *
+     * Pre-fix lineage: this site previously ran ONE NimbusLoaderPool
+     *   with concurrency=1, internal pLimit(3) — the explicit "collapses
+     *   what was 4 concurrent dynamic workers (pool.map slots) into 1"
+     *   YELLOW retreat documented in audit/sections/FANOUT-AUDIT.md (P1).
+     *   Two-tier topology re-expands the fan-out without re-introducing
+     *   the V8 cap risk.
+     */
+    private fetchViaBatchFacet;
+    /**
+     * Build the dependency specs from package.json + explicit packages.
+     */
+    /**
+     * W11: framework detection at install time. Reads package.json from the
+     * project root and runs detectFramework() against its deps + the basenames
+     * we can see in node_modules-adjacent siblings.
+     *
+     * Returns true if the project is detected as one of {next, astro, nuxt,
+     * remix, sveltekit, vite, wrangler}. False for 'unknown'.
+     */
+    private detectFrameworkAware;
+    private buildSpecs;
+    /**
+     * W6: apply WASM_SWAPS rewrites and REJECT_INSTALL deny list to a
+     * top-level spec map. Emits `[swap]` notices via onProgress; throws
+     * a multi-line error on any reject (with `transitive='warn'` rejects
+     * also failing at top level — they only soften at depth>0).
+     *
+     * Idempotent: running on already-swapped specs is a no-op.
+     */
+    private applyW6Registry;
+    /**
+     * Check if a lockfile is still valid against current package.json specs.
+     */
+    private isLockfileValid;
+    /**
+     * Convert a lockfile back to resolved packages (for cache restore).
+     */
+    private lockfileToResolved;
+    /**
+     * Write lockfile to SQLite.
+     */
+    private writeLockfile;
+    /**
+     * Create node_modules/.bin/ entries for packages with "bin" fields.
+     */
+    private linkBins;
+    private updatePackageJson;
+    /**
+     * Pre-bundle ESM modules that are actually imported by the project source.
+     * Scans .ts/.tsx/.jsx/.js files for bare import specifiers (including
+     * subpaths like `react/jsx-runtime`), then bundles each via esbuild.build()
+     * with the VFS plugin.
+     *
+     * Critical: each bundle must externalize react / scheduler / jsx-runtime
+     * as appropriate for the specifier being built. Without this, react-dom
+     * and jsx-runtime each get their own embedded React copy and cross-bundle
+     * JSX elements get rejected as "alien" — silent render failure.
+     */
+    private prebundleUsedModules;
+    /**
+     * Scan project source files for bare import specifiers.
+     * Returns unique bare specifiers including subpaths (e.g., both `react`
+     * AND `react/jsx-runtime` so each can be pre-bundled separately with the
+     * correct externals for shared-runtime isolation).
+     *
+     * Also injects common JSX-runtime subpaths derived from esbuild's automatic
+     * JSX transform: if `react` is imported, we also queue `react/jsx-runtime`
+     * and `react/jsx-dev-runtime` because the compiled JSX output imports from
+     * them even if the source never wrote `import ... from "react/jsx-runtime"`.
+     */
+    private scanBareImports;
+    /**
+     * Resolve a package's entry point to a VFS path.
+     */
+    /**
+     * Resolve a specifier (possibly with subpath, e.g. "react/jsx-runtime" or
+     * "react-dom/client") to an entry-point VFS path under node_modules.
+     *
+     * Algorithm:
+     *   1. Split specifier into pkgName and subpath
+     *      (e.g. "react-dom/client" → pkgName="react-dom", subpath="client";
+     *       "@scope/pkg/sub" → pkgName="@scope/pkg", subpath="sub")
+     *   2. Read node_modules/<pkgName>/package.json
+     *   3. Use resolvePackageEntry(pkg, './' + subpath) to consult exports field
+     *      with ESM conditions
+     *   4. Try extensions and index-file fallbacks
+     */
+    private resolvePackageEntryPath;
+    /**
+     * P5 (prod-bugs-2) — deterministic supervisor-heap estimate in MiB.
+     *
+     * Routes through observability/heap-estimate.ts which sums KNOWN
+     * supervisor heap allocation sources from runtime counters that
+     * ARE accurate inside a DO context (DiagCounters singleton +
+     * SqliteVFS.getStats()). This replaces a previous use of
+     * process.memoryUsage() which returned 0 for every field inside a
+     * Durable Object class context, making the printed
+     * "supervisor heap N MiB" lines actively misleading (they always
+     * read 0.0 MiB regardless of actual heap state).
+     *
+     * The estimator is INTENTIONALLY conservative — peak-or-current
+     * components, sum may overestimate — but it never under-reports.
+     * Returns 0 if the estimator throws (defensive: a counter
+     * regression must not block install completion).
+     *
+     * Same call shape used by /api/_diag/memory in
+     * src/session/routes.ts:247, so the value printed in the
+     * pre-bundle banner is comparable to the value the diag endpoint
+     * reports for the same isolate.
+     */
+    private _estimateSupervisorHeapMiB;
+}
+//# sourceMappingURL=installer.d.ts.map

package/dist/npm/installer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"installer.d.ts","sourceRoot":"","sources":["../../src/npm/installer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAmB,MAAM,sBAAsB,CAAC;AACvE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAC;AAEpE,OAAO,EAAE,QAAQ,EAAsB,MAAM,YAAY,CAAC;AAC1D,OAAO,EAEiC,KAAK,OAAO,EACnD,MAAM,eAAe,CAAC;AAmEvB,YAAY,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAChE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAEhE,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,YAAY,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAChC;AAID,qBAAa,YAAY;IACvB,OAAO,CAAC,GAAG,CAAY;IACvB,OAAO,CAAC,KAAK,CAAW;IACxB,OAAO,CAAC,OAAO,CAAwB;IACvC,OAAO,CAAC,GAAG,CAAiC;IAC5C,OAAO,CAAC,GAAG,CAAM;IACjB,OAAO,CAAC,UAAU,CAAsC;IACxD;;;;;;OAMG;IACH,OAAO,CAAC,OAAO,CAAsB;gBAGnC,GAAG,EAAE,SAAS,EACd,GAAG,EAAE,UAAU,EACf,IAAI,CAAC,EAAE;QACL,OAAO,CAAC,EAAE,cAAc,CAAC;QACzB,GAAG,CAAC,EAAE,kBAAkB,CAAC;QACzB,GAAG,CAAC,EAAE,GAAG,CAAC;QACV,UAAU,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;QACnC,OAAO,CAAC,EAAE,OAAO,CAAC;KACnB;IAWH,4EAA4E;IAC5E,IAAI,QAAQ,IAAI,QAAQ,CAAuB;IAI/C;;;;;OAKG;IACG,OAAO,CACX,UAAU,EAAE,MAAM,EAClB,IAAI,CAAC,EAAE;QACL,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;QACpB,UAAU,CAAC,EAAE,OAAO,CAAC;KACtB,GACA,OAAO,CAAC,gBAAgB,CAAC;YAcd,aAAa;IA+T3B;;;;;;;;;;;;;;;;;;;;;;;;;OAyBG;YACW,oBAAoB;IAoRlC;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;YACW,kBAAkB;IAmQhC;;OAEG;IACH;;;;;;;OAOG;YACW,oBAAoB;YAgCpB,UAAU;IAkDxB;;;;;;;OAOG;IACH,OAAO,CAAC,eAAe;IA8BvB;;OAEG;IACH,OAAO,CAAC,eAAe;IAgCvB;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAqC1B;;OAEG;IACH,OAAO,CAAC,aAAa;IAqBrB;;OAEG;IACH,OAAO,CAAC,QAAQ;IAsDhB,OAAO,CAAC,iBAAiB;IA+CzB;;;;;;;;;;OAUG;YACW,oBAAoB;IAuiBlC;;;;;;;;;;OAUG;IACH,OAAO,CAAC,eAAe;IAgFvB;;OAEG;IACH;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,uBAAuB;IAsD/B;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,OAAO,CAAC,0BAA0B;CAkBnC"}