@nimbus-sh/worker 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +219 -0
- package/dist/_shared/bytes.d.ts +18 -0
- package/dist/_shared/bytes.d.ts.map +1 -0
- package/dist/_shared/bytes.js +17 -0
- package/dist/_shared/cache-stats.d.ts +146 -0
- package/dist/_shared/cache-stats.d.ts.map +1 -0
- package/dist/_shared/cache-stats.js +162 -0
- package/dist/_shared/exports-resolver.d.ts +76 -0
- package/dist/_shared/exports-resolver.d.ts.map +1 -0
- package/dist/_shared/exports-resolver.js +284 -0
- package/dist/_shared/install-phase.d.ts +47 -0
- package/dist/_shared/install-phase.d.ts.map +1 -0
- package/dist/_shared/install-phase.js +1 -0
- package/dist/_shared/real-node-imports.d.ts +38 -0
- package/dist/_shared/real-node-imports.d.ts.map +1 -0
- package/dist/_shared/real-node-imports.js +46 -0
- package/dist/_shared/retry.d.ts +93 -0
- package/dist/_shared/retry.d.ts.map +1 -0
- package/dist/_shared/retry.js +161 -0
- package/dist/_shared/session-id.d.ts +39 -0
- package/dist/_shared/session-id.d.ts.map +1 -0
- package/dist/_shared/session-id.js +127 -0
- package/dist/_shared/session-router.d.ts +104 -0
- package/dist/_shared/session-router.d.ts.map +1 -0
- package/dist/_shared/session-router.js +165 -0
- package/dist/_shared/w7-frame.d.ts +93 -0
- package/dist/_shared/w7-frame.d.ts.map +1 -0
- package/dist/_shared/w7-frame.js +402 -0
- package/dist/auth/index.d.ts +16 -0
- package/dist/auth/index.d.ts.map +1 -0
- package/dist/auth/index.js +15 -0
- package/dist/auth/middleware.d.ts +85 -0
- package/dist/auth/middleware.d.ts.map +1 -0
- package/dist/auth/middleware.js +150 -0
- package/dist/auth/token.d.ts +96 -0
- package/dist/auth/token.d.ts.map +1 -0
- package/dist/auth/token.js +226 -0
- package/dist/auth/types.d.ts +141 -0
- package/dist/auth/types.d.ts.map +1 -0
- package/dist/auth/types.js +114 -0
- package/dist/bindings/d1.d.ts +102 -0
- package/dist/bindings/d1.d.ts.map +1 -0
- package/dist/bindings/d1.js +549 -0
- package/dist/bindings/kv.d.ts +82 -0
- package/dist/bindings/kv.d.ts.map +1 -0
- package/dist/bindings/kv.js +290 -0
- package/dist/bindings/r2.d.ts +137 -0
- package/dist/bindings/r2.d.ts.map +1 -0
- package/dist/bindings/r2.js +379 -0
- package/dist/cirrus-npm-cjs.generated.d.ts +17 -0
- package/dist/cirrus-npm-cjs.generated.d.ts.map +1 -0
- package/dist/cirrus-npm-cjs.generated.js +41 -0
- package/dist/cirrus-plugin-react.generated.d.ts +34 -0
- package/dist/cirrus-plugin-react.generated.d.ts.map +1 -0
- package/dist/cirrus-plugin-react.generated.js +33 -0
- package/dist/constants.d.ts +34 -0
- package/dist/constants.d.ts.map +1 -0
- package/dist/constants.js +116 -0
- package/dist/esbuild-wasm-bundle.generated.d.ts +41 -0
- package/dist/esbuild-wasm-bundle.generated.d.ts.map +1 -0
- package/dist/esbuild-wasm-bundle.generated.js +40 -0
- package/dist/facets/cirrus-real.d.ts +147 -0
- package/dist/facets/cirrus-real.d.ts.map +1 -0
- package/dist/facets/cirrus-real.js +981 -0
- package/dist/facets/inner-do-registry.d.ts +41 -0
- package/dist/facets/inner-do-registry.d.ts.map +1 -0
- package/dist/facets/inner-do-registry.js +51 -0
- package/dist/facets/manager.d.ts +314 -0
- package/dist/facets/manager.d.ts.map +1 -0
- package/dist/facets/manager.js +2681 -0
- package/dist/facets/process.d.ts +318 -0
- package/dist/facets/process.d.ts.map +1 -0
- package/dist/facets/process.js +567 -0
- package/dist/facets/real-vite-fs-shim.d.ts +151 -0
- package/dist/facets/real-vite-fs-shim.d.ts.map +1 -0
- package/dist/facets/real-vite-fs-shim.js +1279 -0
- package/dist/facets/real-vite-hmr.d.ts +142 -0
- package/dist/facets/real-vite-hmr.d.ts.map +1 -0
- package/dist/facets/real-vite-hmr.js +476 -0
- package/dist/facets/vite-dev-server.d.ts +293 -0
- package/dist/facets/vite-dev-server.d.ts.map +1 -0
- package/dist/facets/vite-dev-server.js +2580 -0
- package/dist/facets/wasm-swap-registry.d.ts +261 -0
- package/dist/facets/wasm-swap-registry.d.ts.map +1 -0
- package/dist/facets/wasm-swap-registry.js +551 -0
- package/dist/facets/ws-terminal.d.ts +107 -0
- package/dist/facets/ws-terminal.d.ts.map +1 -0
- package/dist/facets/ws-terminal.js +207 -0
- package/dist/git/commands.d.ts +13 -0
- package/dist/git/commands.d.ts.map +1 -0
- package/dist/git/commands.js +633 -0
- package/dist/git/network-facet.d.ts +61 -0
- package/dist/git/network-facet.d.ts.map +1 -0
- package/dist/git/network-facet.js +700 -0
- package/dist/git-bundle.generated.d.ts +16 -0
- package/dist/git-bundle.generated.d.ts.map +1 -0
- package/dist/git-bundle.generated.js +15 -0
- package/dist/index.d.ts +41 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +73 -0
- package/dist/loaders/child-process/spawn-facet.d.ts +90 -0
- package/dist/loaders/child-process/spawn-facet.d.ts.map +1 -0
- package/dist/loaders/child-process/spawn-facet.js +103 -0
- package/dist/loaders/child-process/spawn-pool.d.ts +110 -0
- package/dist/loaders/child-process/spawn-pool.d.ts.map +1 -0
- package/dist/loaders/child-process/spawn-pool.js +161 -0
- package/dist/loaders/fanout-pool.d.ts +195 -0
- package/dist/loaders/fanout-pool.d.ts.map +1 -0
- package/dist/loaders/fanout-pool.js +326 -0
- package/dist/loaders/generated-workers.d.ts +26 -0
- package/dist/loaders/generated-workers.d.ts.map +1 -0
- package/dist/loaders/generated-workers.js +25 -0
- package/dist/loaders/index.d.ts +15 -0
- package/dist/loaders/index.d.ts.map +1 -0
- package/dist/loaders/index.js +13 -0
- package/dist/loaders/loader-pool.d.ts +273 -0
- package/dist/loaders/loader-pool.d.ts.map +1 -0
- package/dist/loaders/loader-pool.js +624 -0
- package/dist/loaders/npm-resolve-preamble.d.ts +34 -0
- package/dist/loaders/npm-resolve-preamble.d.ts.map +1 -0
- package/dist/loaders/npm-resolve-preamble.js +237 -0
- package/dist/loaders/pre-bundle-preamble.d.ts +41 -0
- package/dist/loaders/pre-bundle-preamble.d.ts.map +1 -0
- package/dist/loaders/pre-bundle-preamble.js +114 -0
- package/dist/loaders/vendor/codegen.d.ts +24 -0
- package/dist/loaders/vendor/codegen.d.ts.map +1 -0
- package/dist/loaders/vendor/codegen.js +44 -0
- package/dist/loaders/vendor/errors.d.ts +24 -0
- package/dist/loaders/vendor/errors.d.ts.map +1 -0
- package/dist/loaders/vendor/errors.js +46 -0
- package/dist/loaders/vendor/index.d.ts +16 -0
- package/dist/loaders/vendor/index.d.ts.map +1 -0
- package/dist/loaders/vendor/index.js +11 -0
- package/dist/loaders/vendor/pool.d.ts +88 -0
- package/dist/loaders/vendor/pool.d.ts.map +1 -0
- package/dist/loaders/vendor/pool.js +353 -0
- package/dist/loaders/vendor/primitives.d.ts +11 -0
- package/dist/loaders/vendor/primitives.d.ts.map +1 -0
- package/dist/loaders/vendor/primitives.js +33 -0
- package/dist/loaders/vendor/serialize.d.ts +3 -0
- package/dist/loaders/vendor/serialize.d.ts.map +1 -0
- package/dist/loaders/vendor/serialize.js +25 -0
- package/dist/loaders/vendor/types.d.ts +60 -0
- package/dist/loaders/vendor/types.d.ts.map +1 -0
- package/dist/loaders/vendor/types.js +4 -0
- package/dist/npm/cache.d.ts +136 -0
- package/dist/npm/cache.d.ts.map +1 -0
- package/dist/npm/cache.js +429 -0
- package/dist/npm/install-batch-facet.d.ts +129 -0
- package/dist/npm/install-batch-facet.d.ts.map +1 -0
- package/dist/npm/install-batch-facet.js +598 -0
- package/dist/npm/install-facet.d.ts +29 -0
- package/dist/npm/install-facet.d.ts.map +1 -0
- package/dist/npm/install-facet.js +9 -0
- package/dist/npm/installer.d.ts +239 -0
- package/dist/npm/installer.d.ts.map +1 -0
- package/dist/npm/installer.js +2022 -0
- package/dist/npm/npx-install.d.ts +74 -0
- package/dist/npm/npx-install.d.ts.map +1 -0
- package/dist/npm/npx-install.js +302 -0
- package/dist/npm/pre-bundle-facet.d.ts +168 -0
- package/dist/npm/pre-bundle-facet.d.ts.map +1 -0
- package/dist/npm/pre-bundle-facet.js +653 -0
- package/dist/npm/r2-cache.d.ts +286 -0
- package/dist/npm/r2-cache.d.ts.map +1 -0
- package/dist/npm/r2-cache.js +506 -0
- package/dist/npm/resolve-facet.d.ts +202 -0
- package/dist/npm/resolve-facet.d.ts.map +1 -0
- package/dist/npm/resolve-facet.js +833 -0
- package/dist/npm/resolve-one-facet.d.ts +204 -0
- package/dist/npm/resolve-one-facet.d.ts.map +1 -0
- package/dist/npm/resolve-one-facet.js +437 -0
- package/dist/npm/resolver.d.ts +109 -0
- package/dist/npm/resolver.d.ts.map +1 -0
- package/dist/npm/resolver.js +899 -0
- package/dist/npm/tarball-stream.d.ts +85 -0
- package/dist/npm/tarball-stream.d.ts.map +1 -0
- package/dist/npm/tarball-stream.js +205 -0
- package/dist/npm/tarball.d.ts +51 -0
- package/dist/npm/tarball.d.ts.map +1 -0
- package/dist/npm/tarball.js +185 -0
- package/dist/observability/diag-counters.d.ts +292 -0
- package/dist/observability/diag-counters.d.ts.map +1 -0
- package/dist/observability/diag-counters.js +302 -0
- package/dist/observability/heap-estimate.d.ts +166 -0
- package/dist/observability/heap-estimate.d.ts.map +1 -0
- package/dist/observability/heap-estimate.js +192 -0
- package/dist/observability/heavy-alloc-coord.d.ts +95 -0
- package/dist/observability/heavy-alloc-coord.d.ts.map +1 -0
- package/dist/observability/heavy-alloc-coord.js +149 -0
- package/dist/observability/oom-classify.d.ts +56 -0
- package/dist/observability/oom-classify.d.ts.map +1 -0
- package/dist/observability/oom-classify.js +118 -0
- package/dist/observability/oom-discriminator.d.ts +192 -0
- package/dist/observability/oom-discriminator.d.ts.map +1 -0
- package/dist/observability/oom-discriminator.js +233 -0
- package/dist/real-vite-bundle.generated.d.ts +46 -0
- package/dist/real-vite-bundle.generated.d.ts.map +1 -0
- package/dist/real-vite-bundle.generated.js +53 -0
- package/dist/replica/routing.d.ts +171 -0
- package/dist/replica/routing.d.ts.map +1 -0
- package/dist/replica/routing.js +288 -0
- package/dist/replica/suspension.d.ts +36 -0
- package/dist/replica/suspension.d.ts.map +1 -0
- package/dist/replica/suspension.js +49 -0
- package/dist/router/index.d.ts +156 -0
- package/dist/router/index.d.ts.map +1 -0
- package/dist/router/index.js +220 -0
- package/dist/runtime/assets-loader.d.ts +71 -0
- package/dist/runtime/assets-loader.d.ts.map +1 -0
- package/dist/runtime/assets-loader.js +128 -0
- package/dist/runtime/barrel-detect.d.ts +46 -0
- package/dist/runtime/barrel-detect.d.ts.map +1 -0
- package/dist/runtime/barrel-detect.js +73 -0
- package/dist/runtime/barrel-synthesizer.d.ts +181 -0
- package/dist/runtime/barrel-synthesizer.d.ts.map +1 -0
- package/dist/runtime/barrel-synthesizer.js +541 -0
- package/dist/runtime/bun-repl.d.ts +46 -0
- package/dist/runtime/bun-repl.d.ts.map +1 -0
- package/dist/runtime/bun-repl.js +326 -0
- package/dist/runtime/bun-runner.d.ts +66 -0
- package/dist/runtime/bun-runner.d.ts.map +1 -0
- package/dist/runtime/bun-runner.js +219 -0
- package/dist/runtime/clang-runner.d.ts +38 -0
- package/dist/runtime/clang-runner.d.ts.map +1 -0
- package/dist/runtime/clang-runner.js +1033 -0
- package/dist/runtime/comment-strip.d.ts +37 -0
- package/dist/runtime/comment-strip.d.ts.map +1 -0
- package/dist/runtime/comment-strip.js +74 -0
- package/dist/runtime/esbuild-service.d.ts +201 -0
- package/dist/runtime/esbuild-service.d.ts.map +1 -0
- package/dist/runtime/esbuild-service.js +1444 -0
- package/dist/runtime/esbuild-wasm-bytes.d.ts +87 -0
- package/dist/runtime/esbuild-wasm-bytes.d.ts.map +1 -0
- package/dist/runtime/esbuild-wasm-bytes.js +131 -0
- package/dist/runtime/framework-detect.d.ts +71 -0
- package/dist/runtime/framework-detect.d.ts.map +1 -0
- package/dist/runtime/framework-detect.js +178 -0
- package/dist/runtime/long-running-handle.d.ts +133 -0
- package/dist/runtime/long-running-handle.d.ts.map +1 -0
- package/dist/runtime/long-running-handle.js +192 -0
- package/dist/runtime/node-repl.d.ts +36 -0
- package/dist/runtime/node-repl.d.ts.map +1 -0
- package/dist/runtime/node-repl.js +293 -0
- package/dist/runtime/node-runner.d.ts +111 -0
- package/dist/runtime/node-runner.d.ts.map +1 -0
- package/dist/runtime/node-runner.js +194 -0
- package/dist/runtime/node-shims.d.ts +24 -0
- package/dist/runtime/node-shims.d.ts.map +1 -0
- package/dist/runtime/node-shims.js +3019 -0
- package/dist/runtime/package-manager.d.ts +118 -0
- package/dist/runtime/package-manager.d.ts.map +1 -0
- package/dist/runtime/package-manager.js +420 -0
- package/dist/runtime/port-registry.d.ts +80 -0
- package/dist/runtime/port-registry.d.ts.map +1 -0
- package/dist/runtime/port-registry.js +166 -0
- package/dist/runtime/process-logs-api.d.ts +102 -0
- package/dist/runtime/process-logs-api.d.ts.map +1 -0
- package/dist/runtime/process-logs-api.js +253 -0
- package/dist/runtime/process-logs.d.ts +311 -0
- package/dist/runtime/process-logs.d.ts.map +1 -0
- package/dist/runtime/process-logs.js +656 -0
- package/dist/runtime/process-table.d.ts +65 -0
- package/dist/runtime/process-table.d.ts.map +1 -0
- package/dist/runtime/process-table.js +107 -0
- package/dist/runtime/project-detect.d.ts +21 -0
- package/dist/runtime/project-detect.d.ts.map +1 -0
- package/dist/runtime/project-detect.js +46 -0
- package/dist/runtime/python-repl.d.ts +66 -0
- package/dist/runtime/python-repl.d.ts.map +1 -0
- package/dist/runtime/python-repl.js +563 -0
- package/dist/runtime/python-runner.d.ts +50 -0
- package/dist/runtime/python-runner.d.ts.map +1 -0
- package/dist/runtime/python-runner.js +870 -0
- package/dist/runtime/repl-session.d.ts +160 -0
- package/dist/runtime/repl-session.d.ts.map +1 -0
- package/dist/runtime/repl-session.js +393 -0
- package/dist/runtime/require-resolver.d.ts +52 -0
- package/dist/runtime/require-resolver.d.ts.map +1 -0
- package/dist/runtime/require-resolver.js +690 -0
- package/dist/runtime/router-basename.d.ts +36 -0
- package/dist/runtime/router-basename.d.ts.map +1 -0
- package/dist/runtime/router-basename.js +310 -0
- package/dist/runtime/ruby-repl.d.ts +43 -0
- package/dist/runtime/ruby-repl.d.ts.map +1 -0
- package/dist/runtime/ruby-repl.js +229 -0
- package/dist/runtime/ruby-runner.d.ts +62 -0
- package/dist/runtime/ruby-runner.d.ts.map +1 -0
- package/dist/runtime/ruby-runner.js +737 -0
- package/dist/runtime/runtime-catalog.d.ts +99 -0
- package/dist/runtime/runtime-catalog.d.ts.map +1 -0
- package/dist/runtime/runtime-catalog.js +183 -0
- package/dist/runtime/runtime-registry.d.ts +136 -0
- package/dist/runtime/runtime-registry.d.ts.map +1 -0
- package/dist/runtime/runtime-registry.js +318 -0
- package/dist/runtime/streams.d.ts +13 -0
- package/dist/runtime/streams.d.ts.map +1 -0
- package/dist/runtime/streams.js +402 -0
- package/dist/runtime/wasi-instance.d.ts +190 -0
- package/dist/runtime/wasi-instance.d.ts.map +1 -0
- package/dist/runtime/wasi-instance.js +2044 -0
- package/dist/runtime/wasm-runner.d.ts +117 -0
- package/dist/runtime/wasm-runner.d.ts.map +1 -0
- package/dist/runtime/wasm-runner.js +705 -0
- package/dist/session/bindings.d.ts +168 -0
- package/dist/session/bindings.d.ts.map +1 -0
- package/dist/session/bindings.js +524 -0
- package/dist/session/ctx-exports.d.ts +16 -0
- package/dist/session/ctx-exports.d.ts.map +1 -0
- package/dist/session/ctx-exports.js +22 -0
- package/dist/session/diag.d.ts +62 -0
- package/dist/session/diag.d.ts.map +1 -0
- package/dist/session/diag.js +118 -0
- package/dist/session/fs-watch.d.ts +132 -0
- package/dist/session/fs-watch.d.ts.map +1 -0
- package/dist/session/fs-watch.js +255 -0
- package/dist/session/helpers.d.ts +150 -0
- package/dist/session/helpers.d.ts.map +1 -0
- package/dist/session/helpers.js +472 -0
- package/dist/session/hibernation.d.ts +145 -0
- package/dist/session/hibernation.d.ts.map +1 -0
- package/dist/session/hibernation.js +385 -0
- package/dist/session/init-phases.d.ts +87 -0
- package/dist/session/init-phases.d.ts.map +1 -0
- package/dist/session/init-phases.js +137 -0
- package/dist/session/init.d.ts +46 -0
- package/dist/session/init.d.ts.map +1 -0
- package/dist/session/init.js +3579 -0
- package/dist/session/keys.d.ts +56 -0
- package/dist/session/keys.d.ts.map +1 -0
- package/dist/session/keys.js +55 -0
- package/dist/session/nimbus-session.d.ts +434 -0
- package/dist/session/nimbus-session.d.ts.map +1 -0
- package/dist/session/nimbus-session.js +1084 -0
- package/dist/session/programmatic.d.ts +105 -0
- package/dist/session/programmatic.d.ts.map +1 -0
- package/dist/session/programmatic.js +326 -0
- package/dist/session/replica-routes.d.ts +68 -0
- package/dist/session/replica-routes.d.ts.map +1 -0
- package/dist/session/replica-routes.js +77 -0
- package/dist/session/routes.d.ts +29 -0
- package/dist/session/routes.d.ts.map +1 -0
- package/dist/session/routes.js +1331 -0
- package/dist/session/rpc.d.ts +323 -0
- package/dist/session/rpc.d.ts.map +1 -0
- package/dist/session/rpc.js +808 -0
- package/dist/session/state-store.d.ts +210 -0
- package/dist/session/state-store.d.ts.map +1 -0
- package/dist/session/state-store.js +459 -0
- package/dist/session/supervisor-rpc.d.ts +269 -0
- package/dist/session/supervisor-rpc.d.ts.map +1 -0
- package/dist/session/supervisor-rpc.js +437 -0
- package/dist/session/ws-hibernation-config.d.ts +64 -0
- package/dist/session/ws-hibernation-config.d.ts.map +1 -0
- package/dist/session/ws-hibernation-config.js +89 -0
- package/dist/session/ws.d.ts +95 -0
- package/dist/session/ws.d.ts.map +1 -0
- package/dist/session/ws.js +395 -0
- package/dist/shell/features.d.ts +410 -0
- package/dist/shell/features.d.ts.map +1 -0
- package/dist/shell/features.js +1885 -0
- package/dist/shell/unix-commands.d.ts +15 -0
- package/dist/shell/unix-commands.d.ts.map +1 -0
- package/dist/shell/unix-commands.js +3703 -0
- package/dist/stubs/lifo-ui.d.ts +10 -0
- package/dist/stubs/lifo-ui.d.ts.map +1 -0
- package/dist/stubs/lifo-ui.js +1 -0
- package/dist/tailwind-play.generated.d.ts +19 -0
- package/dist/tailwind-play.generated.d.ts.map +1 -0
- package/dist/tailwind-play.generated.js +18 -0
- package/dist/vfs/dev-provider.d.ts +64 -0
- package/dist/vfs/dev-provider.d.ts.map +1 -0
- package/dist/vfs/dev-provider.js +191 -0
- package/dist/vfs/events.d.ts +51 -0
- package/dist/vfs/events.d.ts.map +1 -0
- package/dist/vfs/events.js +127 -0
- package/dist/vfs/path.d.ts +30 -0
- package/dist/vfs/path.d.ts.map +1 -0
- package/dist/vfs/path.js +41 -0
- package/dist/vfs/seed-project.d.ts +62 -0
- package/dist/vfs/seed-project.d.ts.map +1 -0
- package/dist/vfs/seed-project.js +1045 -0
- package/dist/vfs/sqlite-vfs.d.ts +435 -0
- package/dist/vfs/sqlite-vfs.d.ts.map +1 -0
- package/dist/vfs/sqlite-vfs.js +1670 -0
- package/dist/vfs/symlink-registry.d.ts +64 -0
- package/dist/vfs/symlink-registry.d.ts.map +1 -0
- package/dist/vfs/symlink-registry.js +137 -0
- package/dist/wrangler/nimbus-wrangler.d.ts +157 -0
- package/dist/wrangler/nimbus-wrangler.d.ts.map +1 -0
- package/dist/wrangler/nimbus-wrangler.js +931 -0
- package/package.json +77 -0
- package/public/_assets/cirrus-plugin-react.bundle.js +70740 -0
- package/public/_assets/esbuild-0.24.2.wasm +0 -0
- package/public/_assets/real-vite-bundle.js +101478 -0
- package/public/_assets/rollup.wasm +0 -0
- package/public/favicon.svg +8 -0
- package/public/index.html +565 -0
- package/public/s/index.html +2379 -0
- package/scripts/bundle-esbuild-wasm.mjs +244 -0
- package/scripts/bundle-facet-workers.mjs +137 -0
- package/scripts/bundle-git.mjs +118 -0
- package/scripts/bundle-npm-cjs.mjs +363 -0
- package/scripts/bundle-plugin-react.mjs +480 -0
- package/scripts/bundle-real-vite.mjs +959 -0
- package/scripts/bundle-runtime.mjs +600 -0
- package/scripts/bundle-tailwind-play.mjs +130 -0
- package/scripts/patch-lifo-ui.mjs +131 -0
|
@@ -0,0 +1,2022 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* npm-installer.ts — Unified npm installer for Nimbus.
|
|
3
|
+
*
|
|
4
|
+
* Designed for bun/pnpm-level performance on Cloudflare DO + SQLite.
|
|
5
|
+
*
|
|
6
|
+
* Pipeline:
|
|
7
|
+
* Phase 0: Lock-check — instant if lockfile valid
|
|
8
|
+
* Phase 1: Resolve — pipelined, 12-wide concurrency, registry cached
|
|
9
|
+
* Phase 2: Hoist — compute flat node_modules layout
|
|
10
|
+
* Phase 3: Diff — skip packages already cached
|
|
11
|
+
* Phase 4: Fetch+Extract — wave-based, 15 pkgs/wave, cache results
|
|
12
|
+
* Phase 5: Write — ONE transactionSync() per wave via writeBatch()
|
|
13
|
+
* Phase 6: Link bins — create node_modules/.bin/ entries
|
|
14
|
+
* Phase 7: Pre-bundle — scan source, esbuild used packages (background)
|
|
15
|
+
*
|
|
16
|
+
* Key invariants:
|
|
17
|
+
* - All VFS writes go through writeBatch() (never individual writeFile)
|
|
18
|
+
* - Tarball cache is per-package (name, version) — no cross-package dedup
|
|
19
|
+
* - Lockfile stored in SQLite (not JSON file)
|
|
20
|
+
* - ESM pre-bundles cached in SQLite for /@modules/ serving
|
|
21
|
+
*/
|
|
22
|
+
import { BUNDLER_VERSION } from '../runtime/esbuild-service.js';
|
|
23
|
+
import { NpmCache } from './cache.js';
|
|
24
|
+
import { computeHoistPlan, shouldSkipPackage, } from './resolver.js';
|
|
25
|
+
import { applySwaps, findRejects, lookupSwap, lookupReject, shouldWarnSkipTransitive, formatSwapNotice, RegistryRejectError, emitRegistryEvent, } from '../facets/wasm-swap-registry.js';
|
|
26
|
+
import { resolvePackageEntry } from '../_shared/exports-resolver.js';
|
|
27
|
+
import { buildCacheRestorePayload } from './tarball.js';
|
|
28
|
+
import { NimbusLoaderPool } from '../loaders/loader-pool.js';
|
|
29
|
+
import { NimbusFanoutPool, IN_DO_THRESHOLD, MAX_PEER_FANOUT } from '../loaders/fanout-pool.js';
|
|
30
|
+
import { TAR_STREAM_PREAMBLE, W7_FRAME_PREAMBLE } from '../loaders/generated-workers.js';
|
|
31
|
+
import { installPackagesInFacet, } from './install-batch-facet.js';
|
|
32
|
+
import { setInstallPhase, setResolverPath, setInstallFacetPath, recordInstallFacetCounters, recordPreBundleSummary, recordR2RaceCounters, recordCacheStatEvents, readDiagCounters, } from '../observability/diag-counters.js';
|
|
33
|
+
import { estimateSupervisorHeap } from '../observability/heap-estimate.js';
|
|
34
|
+
import { resolveOnePackumentInFacet, } from './resolve-one-facet.js';
|
|
35
|
+
import { NPM_RESOLVE_PREAMBLE } from '../loaders/npm-resolve-preamble.js';
|
|
36
|
+
import { prebundleOne, buildSliceForSpecifierWithCap, externalsForSpecifier, } from './pre-bundle-facet.js';
|
|
37
|
+
import { PRE_BUNDLE_PREAMBLE } from '../loaders/pre-bundle-preamble.js';
|
|
38
|
+
import { fetchEsbuildWasmBytes } from '../runtime/esbuild-wasm-bytes.js';
|
|
39
|
+
import { CHUNK_SIZE } from '../constants.js';
|
|
40
|
+
import { waitForLowAllocPressure } from '../observability/heavy-alloc-coord.js';
|
|
41
|
+
import { countPackageFiles, BARREL_PKG_FILE_THRESHOLD, packageNameFromSpecifier } from '../runtime/barrel-detect.js';
|
|
42
|
+
import { scanNamedImports, buildSyntheticEntry, buildScopedSliceForSynthetic, syntheticEntryPath, } from '../runtime/barrel-synthesizer.js';
|
|
43
|
+
import { enc } from '../_shared/bytes.js';
|
|
44
|
+
// ── NpmInstaller ────────────────────────────────────────────────────────
|
|
45
|
+
export class NpmInstaller {
|
|
46
|
+
vfs;
|
|
47
|
+
cache;
|
|
48
|
+
esbuild;
|
|
49
|
+
ctx;
|
|
50
|
+
env;
|
|
51
|
+
onProgress;
|
|
52
|
+
/**
|
|
53
|
+
* Injectable fetch function. Required because DO fetch() hangs in
|
|
54
|
+
* wrangler local dev. The caller (NimbusSession) provides a function
|
|
55
|
+
* that routes fetches through a facet worker. Used only by the resolve
|
|
56
|
+
* path (packument JSON) — tarball fetches happen inside the facet pool
|
|
57
|
+
* when the feature flag is on, using the facet's own global fetch.
|
|
58
|
+
*/
|
|
59
|
+
fetchFn;
|
|
60
|
+
constructor(vfs, sql, opts) {
|
|
61
|
+
this.vfs = vfs;
|
|
62
|
+
this.cache = new NpmCache(sql);
|
|
63
|
+
this.esbuild = opts?.esbuild ?? null;
|
|
64
|
+
this.ctx = opts?.ctx;
|
|
65
|
+
this.env = opts?.env;
|
|
66
|
+
this.onProgress = opts?.onProgress;
|
|
67
|
+
this.fetchFn = opts?.fetchFn;
|
|
68
|
+
}
|
|
69
|
+
/** Expose cache for external use (e.g., serveModule in vite-dev-server). */
|
|
70
|
+
get npmCache() { return this.cache; }
|
|
71
|
+
// ── Main entry point ──────────────────────────────────────────────────
|
|
72
|
+
/**
|
|
73
|
+
* Install packages for a project. Handles:
|
|
74
|
+
* - Lockfile-based fast path (no network if lock is valid)
|
|
75
|
+
* - Full resolution + fetch + write pipeline
|
|
76
|
+
* - Incremental: only fetches/writes what changed
|
|
77
|
+
*/
|
|
78
|
+
async install(projectDir, opts) {
|
|
79
|
+
const start = Date.now();
|
|
80
|
+
const projDir = projectDir.replace(/^\/+/, '').replace(/\/+$/, '');
|
|
81
|
+
const nmDir = projDir + '/node_modules';
|
|
82
|
+
const log = (msg) => this.onProgress?.(msg);
|
|
83
|
+
// Reset phase to 'idle' on any exit path so /api/_diag/memory
|
|
84
|
+
// never reports a stale phase after a crash unwound the install.
|
|
85
|
+
// The DO reboot would zero this anyway; finally is defense against
|
|
86
|
+
// a non-fatal mid-install throw.
|
|
87
|
+
try {
|
|
88
|
+
return await this._installInner(projDir, nmDir, opts, log, start);
|
|
89
|
+
}
|
|
90
|
+
finally {
|
|
91
|
+
setInstallPhase('idle');
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
async _installInner(projDir, nmDir, opts, log, start) {
|
|
95
|
+
const phases = {};
|
|
96
|
+
const installed = [];
|
|
97
|
+
const failed = [];
|
|
98
|
+
let totalFiles = 0;
|
|
99
|
+
let cachedHits = 0;
|
|
100
|
+
// ── Phase 0: Lock-check ─────────────────────────────────────────
|
|
101
|
+
setInstallPhase('idle');
|
|
102
|
+
let phaseStart = Date.now();
|
|
103
|
+
log('Checking lockfile...');
|
|
104
|
+
const specs = await this.buildSpecs(projDir, opts?.packages, opts?.production);
|
|
105
|
+
if (Object.keys(specs).length === 0) {
|
|
106
|
+
log('No dependencies to install.');
|
|
107
|
+
return { installed, failed, totalFiles: 0, elapsed: Date.now() - start, cachedHits: 0, phases: {} };
|
|
108
|
+
}
|
|
109
|
+
// W11: framework detection. If the project depends on a framework
|
|
110
|
+
// (next/astro/nuxt/remix/sveltekit) or generic vite, we exempt
|
|
111
|
+
// FRAMEWORK_REQUIRED_PACKAGES (vite, ...) from the SKIP_PACKAGES set
|
|
112
|
+
// so the framework's CLI can `import 'vite'` from node_modules.
|
|
113
|
+
// See audit/sections/W11-plan.md §3.0.
|
|
114
|
+
const frameworkAware = await this.detectFrameworkAware(projDir);
|
|
115
|
+
if (frameworkAware) {
|
|
116
|
+
log(`Framework detected — installing framework-required packages (vite, …).`);
|
|
117
|
+
}
|
|
118
|
+
const lockfile = this.cache.readLockfile(projDir);
|
|
119
|
+
let resolved;
|
|
120
|
+
let usedLockfile = false;
|
|
121
|
+
if (lockfile && !opts?.packages && this.isLockfileValid(lockfile, specs)) {
|
|
122
|
+
log(`Lockfile valid (${lockfile.size} packages). Skipping resolution.`);
|
|
123
|
+
resolved = this.lockfileToResolved(lockfile);
|
|
124
|
+
usedLockfile = true;
|
|
125
|
+
phases['lock-check'] = Date.now() - phaseStart;
|
|
126
|
+
}
|
|
127
|
+
else {
|
|
128
|
+
if (lockfile && !opts?.packages) {
|
|
129
|
+
log('Lockfile outdated. Re-resolving...');
|
|
130
|
+
}
|
|
131
|
+
phases['lock-check'] = Date.now() - phaseStart;
|
|
132
|
+
// ── Phase 1: Resolve ──────────────────────────────────────────
|
|
133
|
+
// F-2 (cleanup-not-done): frontier-coordinator path. Each BFS
|
|
134
|
+
// layer dispatches to NimbusFanoutPool.submitMany — width <5 in-DO
|
|
135
|
+
// (POC C), width ≥5 peer-DO (POC B). Per-package task body is
|
|
136
|
+
// self-contained (resolveOnePackumentInFacet), supervisor builds
|
|
137
|
+
// layer N+1 from layer N's edges.
|
|
138
|
+
//
|
|
139
|
+
// legacy-cleanup (2026-05-13): the legacy single-facet
|
|
140
|
+
// `resolveTreeViaFacet` path + `NIMBUS_RESOLVER_PATH=facet`
|
|
141
|
+
// env-var selector + supporting body were removed. They existed
|
|
142
|
+
// only for A/B profile measurement (against the deleted
|
|
143
|
+
// scripts/profile-layer-widths.mjs); runtime always took the
|
|
144
|
+
// fanout path by default. Missing env.LOADER throws at
|
|
145
|
+
// construction; missing env.NIMBUS_SESSION throws at the first
|
|
146
|
+
// wide-layer submitMany.
|
|
147
|
+
phaseStart = Date.now();
|
|
148
|
+
setInstallPhase('resolve');
|
|
149
|
+
setResolverPath('in-facet');
|
|
150
|
+
log(`Resolving ${Object.keys(specs).length} dependencies (path: fanout, fetch: ${this.fetchFn ? 'facet-proxy' : 'global'})...`);
|
|
151
|
+
resolved = await this.resolveTreeViaFanout(specs, log, { frameworkAware });
|
|
152
|
+
phases['resolve'] = Date.now() - phaseStart;
|
|
153
|
+
if (resolved.size === 0) {
|
|
154
|
+
log('No packages resolved.');
|
|
155
|
+
return {
|
|
156
|
+
installed, failed: Object.keys(specs),
|
|
157
|
+
totalFiles: 0, elapsed: Date.now() - start, cachedHits: 0, phases,
|
|
158
|
+
};
|
|
159
|
+
}
|
|
160
|
+
log(`Resolved ${resolved.size} packages.`);
|
|
161
|
+
}
|
|
162
|
+
// ── Phase 2: Hoist ────────────────────────────────────────────────
|
|
163
|
+
phaseStart = Date.now();
|
|
164
|
+
setInstallPhase('hoist');
|
|
165
|
+
const hoistPlan = computeHoistPlan(resolved);
|
|
166
|
+
phases['hoist'] = Date.now() - phaseStart;
|
|
167
|
+
// ── Phase 3: Diff (cache check) ─────────────────────────────────
|
|
168
|
+
phaseStart = Date.now();
|
|
169
|
+
setInstallPhase('diff');
|
|
170
|
+
const toFetch = [];
|
|
171
|
+
const toRestore = [];
|
|
172
|
+
for (const [, pkg] of resolved) {
|
|
173
|
+
if (!pkg.tarballUrl)
|
|
174
|
+
continue;
|
|
175
|
+
// Check if already installed at the correct path
|
|
176
|
+
const pkgJsonPath = nmDir + '/' + pkg.name + '/package.json';
|
|
177
|
+
if (this.vfs.exists(pkgJsonPath)) {
|
|
178
|
+
try {
|
|
179
|
+
const existing = JSON.parse(this.vfs.readFileString(pkgJsonPath));
|
|
180
|
+
if (existing.version === pkg.version) {
|
|
181
|
+
// Already installed at correct version — skip
|
|
182
|
+
installed.push(`${pkg.name}@${pkg.version}`);
|
|
183
|
+
cachedHits++;
|
|
184
|
+
continue;
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
catch { /* corrupt package.json — reinstall */ }
|
|
188
|
+
}
|
|
189
|
+
// Check tarball cache
|
|
190
|
+
if (this.cache.hasTarballCache(pkg.name, pkg.version)) {
|
|
191
|
+
toRestore.push(pkg);
|
|
192
|
+
cachedHits++;
|
|
193
|
+
}
|
|
194
|
+
else {
|
|
195
|
+
toFetch.push(pkg);
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
phases['diff'] = Date.now() - phaseStart;
|
|
199
|
+
log(`To fetch: ${toFetch.length}, to restore: ${toRestore.length}, already installed: ${cachedHits - toRestore.length}`);
|
|
200
|
+
// ── Phase 4+5: Fetch + Write (wave-based) ───────────────────────
|
|
201
|
+
phaseStart = Date.now();
|
|
202
|
+
setInstallPhase('fetch');
|
|
203
|
+
// First, restore cached packages
|
|
204
|
+
if (toRestore.length > 0) {
|
|
205
|
+
log(`Restoring ${toRestore.length} packages from cache...`);
|
|
206
|
+
// Process in waves to bound memory
|
|
207
|
+
const RESTORE_WAVE = 20;
|
|
208
|
+
for (let i = 0; i < toRestore.length; i += RESTORE_WAVE) {
|
|
209
|
+
const wave = toRestore.slice(i, i + RESTORE_WAVE);
|
|
210
|
+
const payload = buildCacheRestorePayload(wave, hoistPlan, nmDir, this.cache);
|
|
211
|
+
const result = this.vfs.writeBatch(payload);
|
|
212
|
+
totalFiles += result.inodes;
|
|
213
|
+
for (const pkg of wave) {
|
|
214
|
+
installed.push(`${pkg.name}@${pkg.version}`);
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
// Then, fetch + extract + write new packages.
|
|
219
|
+
//
|
|
220
|
+
// Single fetch path: one NimbusLoaderPool isolate (the batch facet)
|
|
221
|
+
// runs the entire install. The facet streams each tarball through
|
|
222
|
+
// gunzip+tar and emits one writeBatch RPC per package; supervisor
|
|
223
|
+
// heap only sees one inbound RPC payload at a time.
|
|
224
|
+
//
|
|
225
|
+
// No fallback paths: env.LOADER + ctx are platform requirements
|
|
226
|
+
// (their absence is a deploy bug, not a runtime branch). Per-package
|
|
227
|
+
// pool.map and the legacy in-supervisor fetchWaves loop were both
|
|
228
|
+
// removed in Phase 2 A'.1 — they re-introduced the supervisor-heap
|
|
229
|
+
// pressure the facet path eliminates.
|
|
230
|
+
if (toFetch.length > 0) {
|
|
231
|
+
setInstallFacetPath('batch-facet');
|
|
232
|
+
log(`Fetching ${toFetch.length} packages... (path: batch-facet)`);
|
|
233
|
+
const batchResult = await this.fetchViaBatchFacet(toFetch, hoistPlan, nmDir);
|
|
234
|
+
totalFiles += batchResult.filesWritten;
|
|
235
|
+
for (const name of batchResult.installed)
|
|
236
|
+
installed.push(name);
|
|
237
|
+
for (const name of batchResult.failed)
|
|
238
|
+
failed.push(name);
|
|
239
|
+
}
|
|
240
|
+
phases['fetch+write'] = Date.now() - phaseStart;
|
|
241
|
+
// ── Phase 6: Link bins ──────────────────────────────────────────
|
|
242
|
+
phaseStart = Date.now();
|
|
243
|
+
setInstallPhase('link-bins');
|
|
244
|
+
this.linkBins(resolved, nmDir);
|
|
245
|
+
phases['link-bins'] = Date.now() - phaseStart;
|
|
246
|
+
// ── Write lockfile ──────────────────────────────────────────────
|
|
247
|
+
if (!usedLockfile || opts?.packages) {
|
|
248
|
+
this.writeLockfile(projDir, resolved, hoistPlan, nmDir);
|
|
249
|
+
}
|
|
250
|
+
// ── Update package.json if explicit packages were added ─────────
|
|
251
|
+
if (opts?.packages && opts.packages.length > 0) {
|
|
252
|
+
this.updatePackageJson(projDir, opts.packages, resolved);
|
|
253
|
+
}
|
|
254
|
+
this.vfs.flushAll();
|
|
255
|
+
// ── Phase 7: Pre-bundle (TRULY fire-and-forget) ─────────────────
|
|
256
|
+
// The install command resolves IMMEDIATELY. Pre-bundle dispatches
|
|
257
|
+
// its facet work in the background.
|
|
258
|
+
//
|
|
259
|
+
// Why true fire-and-forget (not await + try/catch):
|
|
260
|
+
// 1. A try/catch cannot save us from a workerd-level isolate kill
|
|
261
|
+
// (e.g. wasm-compile disallowed errors, OOM, eval blocks).
|
|
262
|
+
// Verified on prod: when the pre-bundle facet threw at request
|
|
263
|
+
// time, the await unwound but the WS had already been torn
|
|
264
|
+
// down by the runtime. The catch ran AFTER damage was done.
|
|
265
|
+
// 2. Pre-bundling is a best-effort optimisation — every miss is
|
|
266
|
+
// recovered by the on-demand bundler at
|
|
267
|
+
// src/vite-dev-server.ts:1466 (serveModule()), which compiles
|
|
268
|
+
// a single module in the supervisor's EsbuildService at
|
|
269
|
+
// page-load time. No correctness dependency.
|
|
270
|
+
// 3. Decoupling install success from pre-bundle success means a
|
|
271
|
+
// future pre-bundle bug never fails an install. Single
|
|
272
|
+
// responsibility per phase.
|
|
273
|
+
//
|
|
274
|
+
// phases['bundle'] now reflects DISPATCH time, not bundle-completion
|
|
275
|
+
// time. The total install elapsed reflects user-perceived completion
|
|
276
|
+
// (immediate after fetch+write+link-bins+lockfile).
|
|
277
|
+
if (this.esbuild) {
|
|
278
|
+
phaseStart = Date.now();
|
|
279
|
+
setInstallPhase('bundle');
|
|
280
|
+
// ── Bug 1 (prod-bugs-2 P4) — late-progress gating ───────────────
|
|
281
|
+
//
|
|
282
|
+
// Background:
|
|
283
|
+
// The install command MUST resolve immediately (see the long
|
|
284
|
+
// note above re: workerd isolate-kill paths that defeat
|
|
285
|
+
// try/catch on awaits). We keep that invariant intact.
|
|
286
|
+
//
|
|
287
|
+
// Symptom we are fixing:
|
|
288
|
+
// prebundleUsedModules's `finally` block emits a "Pre-bundle
|
|
289
|
+
// complete:" line via this.onProgress (installer.ts:1548).
|
|
290
|
+
// onProgress is the closure
|
|
291
|
+
// `(msg) => ctx.stdout.write('[npm] ' + msg + '\n')` captured
|
|
292
|
+
// from the npm registry handler in
|
|
293
|
+
// src/session/init.ts:1228 / :1723. After install() returns,
|
|
294
|
+
// the npm command-handler returns to the shell, the shell
|
|
295
|
+
// prints its prompt, and THEN the orphan promise's safeProgress
|
|
296
|
+
// fires — visually corrupting the freshly-rendered prompt
|
|
297
|
+
// ("user@nimbus:~/app$ [npm] Pre-bundle complete: ...").
|
|
298
|
+
//
|
|
299
|
+
// Fix:
|
|
300
|
+
// Suppress writes to ctx.stdout once install() has returned.
|
|
301
|
+
// Pre-bundle progress is still observable via wrangler dev
|
|
302
|
+
// console (console.log) and via /api/_diag/memory's
|
|
303
|
+
// recordPreBundleSummary aggregate, but it does NOT touch the
|
|
304
|
+
// user's interactive terminal after the prompt has redrawn.
|
|
305
|
+
//
|
|
306
|
+
// Why a flag instead of swapping `this.onProgress`:
|
|
307
|
+
// ensureNpmInstaller (nimbus-session.ts:892) caches the
|
|
308
|
+
// installer on `this.npmInstaller` for the DO's lifetime. A
|
|
309
|
+
// subsequent `npm install` invocation has a different ctx,
|
|
310
|
+
// so the persistent onProgress reference is doubly wrong:
|
|
311
|
+
// it's stale-after-this-invocation AND it would clobber the
|
|
312
|
+
// next install's progress channel. Gating without mutating
|
|
313
|
+
// keeps the swap simple and idempotent.
|
|
314
|
+
const installInvocationActive = { v: true };
|
|
315
|
+
// Replace this.onProgress (the persistent ctx.stdout closure)
|
|
316
|
+
// with a wrapper for the duration of pre-bundle. While the
|
|
317
|
+
// outer install() call is on the stack the wrapper forwards to
|
|
318
|
+
// the original; once we flip the flag in the cleanup below,
|
|
319
|
+
// the wrapper drops to a console.log fallback so traces aren't
|
|
320
|
+
// lost but ctx.stdout never sees them.
|
|
321
|
+
const persistentProgress = this.onProgress;
|
|
322
|
+
this.onProgress = (msg) => {
|
|
323
|
+
if (installInvocationActive.v) {
|
|
324
|
+
persistentProgress?.(msg);
|
|
325
|
+
}
|
|
326
|
+
else {
|
|
327
|
+
// Late progress — pre-bundle finished AFTER install()
|
|
328
|
+
// returned. Surface to the wrangler dev console only so
|
|
329
|
+
// the user's shell prompt isn't corrupted.
|
|
330
|
+
try {
|
|
331
|
+
console.log('[npm:late] ' + msg);
|
|
332
|
+
}
|
|
333
|
+
catch { }
|
|
334
|
+
}
|
|
335
|
+
};
|
|
336
|
+
// Fire-and-forget. Capture rejections so the orphan promise
|
|
337
|
+
// never raises an "unhandled rejection" warning. We do NOT
|
|
338
|
+
// await here — see Phase 7 design note above for why.
|
|
339
|
+
const prebundlePromise = this.prebundleUsedModules(projDir, resolved)
|
|
340
|
+
.catch((e) => {
|
|
341
|
+
// Routes through the installInvocationActive gate above,
|
|
342
|
+
// so this is safe to call from after-return: it lands on
|
|
343
|
+
// console.log instead of ctx.stdout.
|
|
344
|
+
log(`pre-bundle skipped: ${e?.message || String(e)}`);
|
|
345
|
+
})
|
|
346
|
+
.finally(() => {
|
|
347
|
+
// Always restore the persistent reference so a subsequent
|
|
348
|
+
// ensureNpmInstaller call (which doesn't reconstruct the
|
|
349
|
+
// installer) can still wire a fresh ctx.stdout closure.
|
|
350
|
+
this.onProgress = persistentProgress;
|
|
351
|
+
});
|
|
352
|
+
// Mark `void` so the linter / human reader knows we intentionally
|
|
353
|
+
// don't await this. The promise outlives the install command.
|
|
354
|
+
void prebundlePromise;
|
|
355
|
+
phases['bundle'] = Date.now() - phaseStart;
|
|
356
|
+
// The flag flip happens AFTER install() returns its result.
|
|
357
|
+
// We schedule it inline by closing over the same object;
|
|
358
|
+
// the `finally` at install()'s top level (line 173) gets us
|
|
359
|
+
// to the right boundary. We piggyback there via a deferred
|
|
360
|
+
// microtask: when the outer try{} returns the result object,
|
|
361
|
+
// the microtask flips the flag; pre-bundle's safeProgress
|
|
362
|
+
// calls after this point land on console.log.
|
|
363
|
+
queueMicrotask(() => {
|
|
364
|
+
installInvocationActive.v = false;
|
|
365
|
+
});
|
|
366
|
+
}
|
|
367
|
+
setInstallPhase('done');
|
|
368
|
+
const elapsed = Date.now() - start;
|
|
369
|
+
log(`Done! ${installed.length} packages, ${totalFiles} files in ${(elapsed / 1000).toFixed(1)}s`);
|
|
370
|
+
if (cachedHits > 0) {
|
|
371
|
+
log(` (${cachedHits} from cache)`);
|
|
372
|
+
}
|
|
373
|
+
return { installed, failed, totalFiles, elapsed, cachedHits, phases };
|
|
374
|
+
}
|
|
375
|
+
// ── Single-resolver / single-fetcher invariant (Phase 2 A'.1) ─────────
|
|
376
|
+
//
|
|
377
|
+
// Pre-rebuild this section had three feature flags that gated the
|
|
378
|
+
// facet paths and fell back to in-supervisor resolveTree /
|
|
379
|
+
// fetchWaves / pool.map when off. The fallback paths re-introduced
|
|
380
|
+
// exactly the supervisor heap pressure the rebuild aims to remove,
|
|
381
|
+
// so they were deleted along with their feature flags.
|
|
382
|
+
//
|
|
383
|
+
// Single resolver: src/npm-resolve-facet.ts (called from
|
|
384
|
+
// resolveTreeViaFanout below).
|
|
385
|
+
// Single fetcher : src/npm-install-batch-facet.ts (called from
|
|
386
|
+
// fetchViaBatchFacet below).
|
|
387
|
+
//
|
|
388
|
+
// env.LOADER and ctx are platform requirements; if either is
|
|
389
|
+
// missing the install fails loud at the first await on the facet
|
|
390
|
+
// pool — that's a deploy bug, not a runtime branch.
|
|
391
|
+
/**
|
|
392
|
+
* F-2 frontier-coordinator path. Replaces the single-resolve-facet
|
|
393
|
+
* dispatch with a per-package fanout: each BFS layer becomes ONE
|
|
394
|
+
* `NimbusFanoutPool.submitMany` call, layer N+1 builds from the
|
|
395
|
+
* resolved metadata of layer N.
|
|
396
|
+
*
|
|
397
|
+
* Topology auto-routes per layer:
|
|
398
|
+
* width < IN_DO_THRESHOLD (5) → POC C in-DO loader-pool
|
|
399
|
+
* width >= IN_DO_THRESHOLD → POC B peer-DO (sibling NimbusSession DOs)
|
|
400
|
+
*
|
|
401
|
+
* The supervisor still owns:
|
|
402
|
+
* - cycle detection (`seen`),
|
|
403
|
+
* - X.5-F top-level / required-peer policy,
|
|
404
|
+
* - X.5-G G1 optional-native silent-skip,
|
|
405
|
+
* - X.5-drizzle best-effort tagging on optional-peer subtrees,
|
|
406
|
+
* - W6 swap / warn / reject decisions (top-level enforcement; the
|
|
407
|
+
* per-package task ALSO checks these for transitive correctness),
|
|
408
|
+
* - cache flushing (one batched putRegistryEntries at end).
|
|
409
|
+
*
|
|
410
|
+
* The per-package task (resolveOnePackumentInFacet) owns ONLY the
|
|
411
|
+
* fetch + version pick + edge extraction. See
|
|
412
|
+
* audit/sections/F2-RESOLVER-FANOUT-plan.md.
|
|
413
|
+
*
|
|
414
|
+
* Anti-requirements (cleanup-not-done charter): NO setTimeout
|
|
415
|
+
* between layers, NO fallback to single-facet on missing bindings.
|
|
416
|
+
*/
|
|
417
|
+
async resolveTreeViaFanout(specs, log, opts = {}) {
|
|
418
|
+
const t0 = Date.now();
|
|
419
|
+
const frameworkAware = !!opts.frameworkAware;
|
|
420
|
+
const __f2Diag = (globalThis.process?.env?.NIMBUS_DIAG_INSTALL_PIPELINE === '1');
|
|
421
|
+
// Per-walk state — supervisor side.
|
|
422
|
+
const resolved = new Map();
|
|
423
|
+
const seen = new Set();
|
|
424
|
+
const topLevelNames = new Set(Object.keys(specs));
|
|
425
|
+
const optionalNames = new Set(); // X.5-G G1
|
|
426
|
+
const bestEffortNames = new Set(); // X.5-drizzle
|
|
427
|
+
let queue = Object.entries(specs);
|
|
428
|
+
const cacheWritesPending = [];
|
|
429
|
+
let totalPackumentBytes = 0;
|
|
430
|
+
let totalPackumentsDecoded = 0;
|
|
431
|
+
let layerN = 0;
|
|
432
|
+
let totalLayers = 0;
|
|
433
|
+
let r2Wins = 0;
|
|
434
|
+
let r2Losses = 0;
|
|
435
|
+
// cache-obs-2: accumulator for per-tier cache events across all
|
|
436
|
+
// resolve-one tasks in this fanout walk. Drained at end-of-walk
|
|
437
|
+
// into the DO singleton via recordCacheStatEvents.
|
|
438
|
+
const fanoutCacheStatEvents = [];
|
|
439
|
+
// Counter for diagnostics — peak in-flight inside a layer = layer
|
|
440
|
+
// width (parallelism mirrors the in-DO/peer-DO pool's task count).
|
|
441
|
+
let inFlightPeak = 0;
|
|
442
|
+
// F-2 fanout pool. One construction reused across every layer; the
|
|
443
|
+
// pool is stateless across submitMany calls.
|
|
444
|
+
const fanoutPool = new NimbusFanoutPool(this.env, this.ctx, {
|
|
445
|
+
tag: 'npm-resolve-fanout',
|
|
446
|
+
// 5 minutes per layer is generous; typical layers complete in
|
|
447
|
+
// 1-3 s. Per-task this gates each packument fetch + R2 race.
|
|
448
|
+
timeoutMs: 5 * 60_000,
|
|
449
|
+
preamble: NPM_RESOLVE_PREAMBLE,
|
|
450
|
+
});
|
|
451
|
+
// Frontier loop. Each iteration = ONE BFS layer dispatched as ONE
|
|
452
|
+
// submitMany batch.
|
|
453
|
+
while (queue.length > 0) {
|
|
454
|
+
// Dedupe + filter the layer up front. The task body also filters
|
|
455
|
+
// (defensive), but doing it here avoids dispatching wasted RPC
|
|
456
|
+
// for already-seen names.
|
|
457
|
+
const layer = [];
|
|
458
|
+
const layerSeenLocal = new Set();
|
|
459
|
+
for (const [name, range] of queue) {
|
|
460
|
+
if (seen.has(name) || layerSeenLocal.has(name))
|
|
461
|
+
continue;
|
|
462
|
+
layerSeenLocal.add(name);
|
|
463
|
+
seen.add(name);
|
|
464
|
+
layer.push([name, range]);
|
|
465
|
+
}
|
|
466
|
+
queue = [];
|
|
467
|
+
if (__f2Diag) {
|
|
468
|
+
log(`[f2-frontier] N=${layerN} width=${layer.length} resolved-so-far=${resolved.size} seen=${seen.size}`);
|
|
469
|
+
}
|
|
470
|
+
if (layer.length === 0)
|
|
471
|
+
break;
|
|
472
|
+
// Track peak — the layer is dispatched in parallel inside the pool.
|
|
473
|
+
if (layer.length > inFlightPeak)
|
|
474
|
+
inFlightPeak = layer.length;
|
|
475
|
+
totalLayers++;
|
|
476
|
+
// Build per-package tasks. Each task gets its own pre-loaded
|
|
477
|
+
// cache slice (only entries for THIS name) so the per-task RPC
|
|
478
|
+
// stays small. Bounded to 16 versions per name — enough to cover
|
|
479
|
+
// ~2 majors of typical packages, well under any RPC arg size cap.
|
|
480
|
+
const tasks = layer.map(([name, range]) => {
|
|
481
|
+
const cachedRows = this.cache.getRegistryVersions(name).slice(0, 16);
|
|
482
|
+
const cachedEntries = cachedRows.map((e) => ({
|
|
483
|
+
name: e.name,
|
|
484
|
+
version: e.version,
|
|
485
|
+
tarballUrl: e.tarballUrl,
|
|
486
|
+
integrity: e.integrity,
|
|
487
|
+
depsJson: e.depsJson,
|
|
488
|
+
peerDepsJson: e.peerDepsJson,
|
|
489
|
+
exportsJson: e.exportsJson,
|
|
490
|
+
main: e.main,
|
|
491
|
+
moduleField: e.moduleField,
|
|
492
|
+
binJson: e.binJson,
|
|
493
|
+
fetchedAt: e.fetchedAt,
|
|
494
|
+
}));
|
|
495
|
+
const taskSpec = {
|
|
496
|
+
name,
|
|
497
|
+
range,
|
|
498
|
+
cachedEntries,
|
|
499
|
+
topLevel: topLevelNames.has(name),
|
|
500
|
+
isOptional: optionalNames.has(name),
|
|
501
|
+
frameworkAware,
|
|
502
|
+
fetchTimeoutMs: 15_000,
|
|
503
|
+
retries: 3,
|
|
504
|
+
};
|
|
505
|
+
return { key: name, args: taskSpec };
|
|
506
|
+
});
|
|
507
|
+
// Dispatch the layer. NimbusFanoutPool routes:
|
|
508
|
+
// <5 → POC C in-DO (NimbusLoaderPool), concurrency = layer.length (capped at 4)
|
|
509
|
+
// ≥5 → POC B peer-DO, N peers = min(layer.length, 32)
|
|
510
|
+
let results;
|
|
511
|
+
try {
|
|
512
|
+
results = await fanoutPool.submitMany(tasks, resolveOnePackumentInFacet);
|
|
513
|
+
}
|
|
514
|
+
catch (e) {
|
|
515
|
+
// Per anti-requirement: no fallback. Log + propagate.
|
|
516
|
+
const msg = e?.remoteMessage || e?.message || String(e);
|
|
517
|
+
log(` resolver-fanout layer ${layerN} failed: ${msg}`);
|
|
518
|
+
throw new Error(`resolver-fanout failed at layer ${layerN}: ${msg}`);
|
|
519
|
+
}
|
|
520
|
+
// Stitch per-package results into supervisor state.
|
|
521
|
+
for (let i = 0; i < results.length; i++) {
|
|
522
|
+
const res = results[i];
|
|
523
|
+
const [taskName] = layer[i];
|
|
524
|
+
// Forward messages + events.
|
|
525
|
+
for (const m of res.messages)
|
|
526
|
+
log(m);
|
|
527
|
+
for (const ev of res.events) {
|
|
528
|
+
try {
|
|
529
|
+
emitRegistryEvent(ev);
|
|
530
|
+
}
|
|
531
|
+
catch { /* swallow sink errors */ }
|
|
532
|
+
}
|
|
533
|
+
// Accumulate cache writes for end-of-walk flush.
|
|
534
|
+
for (const cw of res.cacheWrites)
|
|
535
|
+
cacheWritesPending.push(cw);
|
|
536
|
+
// cache-obs-2: harvest per-task cache events for end-of-walk fold.
|
|
537
|
+
if (Array.isArray(res.cacheStatEvents)) {
|
|
538
|
+
for (const e of res.cacheStatEvents)
|
|
539
|
+
fanoutCacheStatEvents.push(e);
|
|
540
|
+
}
|
|
541
|
+
totalPackumentBytes += res.packumentBytesDecoded;
|
|
542
|
+
if (res.packumentSource === 'r2-cache')
|
|
543
|
+
r2Wins++;
|
|
544
|
+
else if (res.packumentSource === 'network')
|
|
545
|
+
r2Losses++;
|
|
546
|
+
if (res.packumentBytesDecoded > 0)
|
|
547
|
+
totalPackumentsDecoded++;
|
|
548
|
+
// W6 reject error handling — mirrors resolve-facet.ts:716.
|
|
549
|
+
if (res.error && res.error.type === 'w6-reject') {
|
|
550
|
+
if (bestEffortNames.has(taskName)) {
|
|
551
|
+
// X.5-drizzle: silent-skip inside best-effort optional-peer
|
|
552
|
+
// subtree.
|
|
553
|
+
const reason = `inside best-effort optional-peer subtree (X.5-drizzle): ${res.error.reason}`;
|
|
554
|
+
log(`[resolve-fanout] [skip] ${taskName} — ${reason}`);
|
|
555
|
+
emitRegistryEvent({ type: 'transitive-skip', from: taskName, reason });
|
|
556
|
+
continue;
|
|
557
|
+
}
|
|
558
|
+
// Real reject: throw RegistryRejectError to abort install.
|
|
559
|
+
const rejectEntry = {
|
|
560
|
+
from: res.error.from,
|
|
561
|
+
reason: res.error.reason,
|
|
562
|
+
suggest: res.error.suggest,
|
|
563
|
+
transitive: 'fail',
|
|
564
|
+
};
|
|
565
|
+
throw new RegistryRejectError([rejectEntry]);
|
|
566
|
+
}
|
|
567
|
+
// Optional-dep fetch failure → silent-skip (X.5-G G1).
|
|
568
|
+
if (res.error && res.error.type === 'fetch-exhausted' && optionalNames.has(taskName)) {
|
|
569
|
+
const reason = `optional dep fetch failed: ${res.error.message}`;
|
|
570
|
+
log(`[resolve-fanout] [skip] ${taskName} — ${reason}`);
|
|
571
|
+
emitRegistryEvent({ type: 'transitive-skip', from: taskName, reason });
|
|
572
|
+
continue;
|
|
573
|
+
}
|
|
574
|
+
const pkg = res.pkg;
|
|
575
|
+
if (!pkg)
|
|
576
|
+
continue;
|
|
577
|
+
// X.5-G G1: silent-skip platform-native bindings sourced from
|
|
578
|
+
// optionalDependencies. The task returns the pkg raw; the
|
|
579
|
+
// supervisor checks isOptional + os/cpu/libc/main.
|
|
580
|
+
if (optionalNames.has(taskName)) {
|
|
581
|
+
const isNativeBinding = (Array.isArray(pkg.os) && pkg.os.length > 0) ||
|
|
582
|
+
(Array.isArray(pkg.cpu) && pkg.cpu.length > 0) ||
|
|
583
|
+
(Array.isArray(pkg.libc) && pkg.libc.length > 0) ||
|
|
584
|
+
(typeof pkg.main === 'string' && /\.node$/.test(pkg.main));
|
|
585
|
+
if (isNativeBinding) {
|
|
586
|
+
const reason = `optional native binding (os=${pkg.os ?? '*'}, cpu=${pkg.cpu ?? '*'}, libc=${pkg.libc ?? '*'}, main=${pkg.main || '?'})`;
|
|
587
|
+
log(`[resolve-fanout] [skip] ${taskName} — ${reason}`);
|
|
588
|
+
emitRegistryEvent({ type: 'transitive-skip', from: taskName, reason });
|
|
589
|
+
continue;
|
|
590
|
+
}
|
|
591
|
+
}
|
|
592
|
+
if (resolved.has(pkg.name))
|
|
593
|
+
continue;
|
|
594
|
+
resolved.set(pkg.name, pkg);
|
|
595
|
+
// Edge extraction — mirrors resolve-facet.ts:754-836.
|
|
596
|
+
const inheritBestEffort = bestEffortNames.has(pkg.name);
|
|
597
|
+
for (const [depName, depRange] of Object.entries(pkg.dependencies)) {
|
|
598
|
+
if (resolved.has(depName) || seen.has(depName))
|
|
599
|
+
continue;
|
|
600
|
+
if (inheritBestEffort)
|
|
601
|
+
bestEffortNames.add(depName);
|
|
602
|
+
queue.push([depName, depRange]);
|
|
603
|
+
}
|
|
604
|
+
const optDeps = pkg.optionalDependencies;
|
|
605
|
+
if (optDeps) {
|
|
606
|
+
for (const [depName, depRange] of Object.entries(optDeps)) {
|
|
607
|
+
if (resolved.has(depName) || seen.has(depName))
|
|
608
|
+
continue;
|
|
609
|
+
optionalNames.add(depName);
|
|
610
|
+
if (inheritBestEffort)
|
|
611
|
+
bestEffortNames.add(depName);
|
|
612
|
+
queue.push([depName, depRange]);
|
|
613
|
+
}
|
|
614
|
+
}
|
|
615
|
+
if (pkg.peerDependencies) {
|
|
616
|
+
for (const [peerName, peerRange] of Object.entries(pkg.peerDependencies)) {
|
|
617
|
+
if (resolved.has(peerName) || seen.has(peerName))
|
|
618
|
+
continue;
|
|
619
|
+
topLevelNames.add(peerName);
|
|
620
|
+
if (inheritBestEffort)
|
|
621
|
+
bestEffortNames.add(peerName);
|
|
622
|
+
queue.push([peerName, peerRange]);
|
|
623
|
+
}
|
|
624
|
+
}
|
|
625
|
+
// X.5-F R2.5 + X.5-J: optional peers when THIS pkg is the
|
|
626
|
+
// user's top-level. Filter through REJECT_INSTALL.
|
|
627
|
+
if (topLevelNames.has(pkg.name)) {
|
|
628
|
+
const allPeers = pkg.__allPeerDependencies;
|
|
629
|
+
if (allPeers) {
|
|
630
|
+
for (const [peerName, peerRange] of Object.entries(allPeers)) {
|
|
631
|
+
if (resolved.has(peerName) || seen.has(peerName))
|
|
632
|
+
continue;
|
|
633
|
+
const peerFail = lookupReject(peerName);
|
|
634
|
+
const peerWarn = shouldWarnSkipTransitive(peerName);
|
|
635
|
+
const peerReject = peerFail || peerWarn;
|
|
636
|
+
if (peerReject) {
|
|
637
|
+
const reason = `optional peer in REJECT_INSTALL: ${peerName} — ${peerReject.reason}`;
|
|
638
|
+
log(`[resolve-fanout] [skip] ${peerName} — ${reason}`);
|
|
639
|
+
emitRegistryEvent({ type: 'transitive-skip', from: peerName, reason });
|
|
640
|
+
continue;
|
|
641
|
+
}
|
|
642
|
+
topLevelNames.add(peerName);
|
|
643
|
+
bestEffortNames.add(peerName);
|
|
644
|
+
queue.push([peerName, peerRange]);
|
|
645
|
+
}
|
|
646
|
+
}
|
|
647
|
+
}
|
|
648
|
+
}
|
|
649
|
+
layerN++;
|
|
650
|
+
}
|
|
651
|
+
// End-of-walk: flush all cache writes in one RPC-equivalent call
|
|
652
|
+
// (this.cache is a SQLite handle; one putRegistryEntries call =
|
|
653
|
+
// O(N) prepared statements within a single DO event-loop turn,
|
|
654
|
+
// atomically committed by the storage layer).
|
|
655
|
+
let cacheWriteCount = 0;
|
|
656
|
+
if (cacheWritesPending.length > 0) {
|
|
657
|
+
const r = this.cache.putRegistryEntries(cacheWritesPending);
|
|
658
|
+
cacheWriteCount = r.written;
|
|
659
|
+
if (r.failed > 0)
|
|
660
|
+
log(` resolver-fanout cache write: ${r.failed} entries failed`);
|
|
661
|
+
}
|
|
662
|
+
// Diag/counters: same shape the supervisor's r2-race telemetry expects.
|
|
663
|
+
recordR2RaceCounters({
|
|
664
|
+
pipelinedTarballRaceWins: 0,
|
|
665
|
+
pipelinedTarballRaceLosses: 0,
|
|
666
|
+
pipelinedPackumentRaceWins: r2Wins,
|
|
667
|
+
pipelinedPackumentRaceLosses: r2Losses,
|
|
668
|
+
});
|
|
669
|
+
// cache-obs-2: fold all per-tier cache events from this walk into
|
|
670
|
+
// the DO singleton. Visible at /api/_diag/cache.
|
|
671
|
+
recordCacheStatEvents(fanoutCacheStatEvents);
|
|
672
|
+
const r2WinSuffix = r2Wins > 0 ? `, R2 packument cache wins=${r2Wins}/${r2Wins + r2Losses}` : '';
|
|
673
|
+
log(` resolver-fanout: ${resolved.size} resolved, ` +
|
|
674
|
+
`${totalPackumentsDecoded} packuments fetched (` +
|
|
675
|
+
`${(totalPackumentBytes / (1024 * 1024)).toFixed(1)} MiB), ` +
|
|
676
|
+
`peak in-flight=${inFlightPeak}, ` +
|
|
677
|
+
`cache writes=${cacheWriteCount}` +
|
|
678
|
+
r2WinSuffix +
|
|
679
|
+
`, layers=${totalLayers}, ` +
|
|
680
|
+
`elapsed=${((Date.now() - t0) / 1000).toFixed(1)}s`);
|
|
681
|
+
return resolved;
|
|
682
|
+
}
|
|
683
|
+
/**
|
|
684
|
+
* Batch install via two-tier fan-out (NimbusFanoutPool).
|
|
685
|
+
*
|
|
686
|
+
* Topology selected automatically based on the spec count:
|
|
687
|
+
* specs.length < IN_DO_THRESHOLD (5) → POC C in-DO
|
|
688
|
+
* 1 NimbusLoaderPool with concurrency = specs.length, capped at
|
|
689
|
+
* 4 by V8 invariant. Each spec is one facet running its own
|
|
690
|
+
* installPackagesInFacet over a single-element shard.
|
|
691
|
+
* specs.length >= IN_DO_THRESHOLD → POC B peer-DO
|
|
692
|
+
* N peer NimbusSession sibling DOs (N = min(specs.length,
|
|
693
|
+
* MAX_PEER_FANOUT = 32)), each running ONE installPackagesInFacet
|
|
694
|
+
* against its own shard with internal pLimit(3).
|
|
695
|
+
*
|
|
696
|
+
* Sharding strategy: round-robin (`pkgIdx % N`) so every peer DO
|
|
697
|
+
* receives roughly equal work. Stable-id router maps each
|
|
698
|
+
* `shard-${i}` task key deterministically (tests can predict
|
|
699
|
+
* placement).
|
|
700
|
+
*
|
|
701
|
+
* Pre-fix lineage: this site previously ran ONE NimbusLoaderPool
|
|
702
|
+
* with concurrency=1, internal pLimit(3) — the explicit "collapses
|
|
703
|
+
* what was 4 concurrent dynamic workers (pool.map slots) into 1"
|
|
704
|
+
* YELLOW retreat documented in audit/sections/FANOUT-AUDIT.md (P1).
|
|
705
|
+
* Two-tier topology re-expands the fan-out without re-introducing
|
|
706
|
+
* the V8 cap risk.
|
|
707
|
+
*/
|
|
708
|
+
async fetchViaBatchFacet(toFetch, hoistPlan, nmDir) {
|
|
709
|
+
const log = (msg) => this.onProgress?.(msg);
|
|
710
|
+
const installed = [];
|
|
711
|
+
const failed = [];
|
|
712
|
+
let filesWritten = 0;
|
|
713
|
+
const mtime = Date.now();
|
|
714
|
+
const specs = toFetch
|
|
715
|
+
.filter((p) => !!p.tarballUrl)
|
|
716
|
+
.map((p) => ({
|
|
717
|
+
name: p.name,
|
|
718
|
+
version: p.version,
|
|
719
|
+
tarballUrl: p.tarballUrl,
|
|
720
|
+
integrity: p.integrity || '',
|
|
721
|
+
pkgDir: nmDir + '/' + p.name,
|
|
722
|
+
mtime,
|
|
723
|
+
chunkSize: CHUNK_SIZE,
|
|
724
|
+
}));
|
|
725
|
+
// hoistPlan is intentionally unused: the current installer maps
|
|
726
|
+
// every package to `${nmDir}/${name}` (flat hoisting). Accepting
|
|
727
|
+
// the plan as a parameter keeps the caller agnostic of the hoist
|
|
728
|
+
// strategy and lets a future nested-install variant slot in
|
|
729
|
+
// without changing the call site.
|
|
730
|
+
void hoistPlan;
|
|
731
|
+
if (specs.length === 0) {
|
|
732
|
+
return { installed, failed, filesWritten };
|
|
733
|
+
}
|
|
734
|
+
// Shard count: bounded by spec count, the peer-DO ceiling, and
|
|
735
|
+
// the coordinator-fanout ceiling.
|
|
736
|
+
//
|
|
737
|
+
// For small batches (< IN_DO_THRESHOLD), the in-DO path uses
|
|
738
|
+
// concurrency = specs.length (capped at 4 by NimbusFanoutPool's
|
|
739
|
+
// in-DO leg) — so each spec gets its own slot.
|
|
740
|
+
//
|
|
741
|
+
// For larger batches, we shard into N peer DOs.
|
|
742
|
+
//
|
|
743
|
+
// [COORDINATOR-OVERLOAD adaptive cap, P0a wave-4]
|
|
744
|
+
//
|
|
745
|
+
// History of failed waves (all on Markflow 617 deps, real prod):
|
|
746
|
+
// wave-1 (16 shards + writeBatchStream semaphore): 270 individual
|
|
747
|
+
// overload warns per install. Per-package writeBatchStream RPCs
|
|
748
|
+
// overflowed coordinator's input gate.
|
|
749
|
+
// wave-2 (16 shards + shared-flush at 16 MiB + semaphore):
|
|
750
|
+
// [batch-fanout] aborted — semaphore extended each peer's
|
|
751
|
+
// parent RPC round-trip, parent aged out.
|
|
752
|
+
// wave-3 (8 shards + shared-flush at 4 MiB, no semaphore):
|
|
753
|
+
// single-session = always GREEN. 12-concurrent-session load
|
|
754
|
+
// test: 25-33% of sessions hit `[batch-fanout] aborted:
|
|
755
|
+
// ExecutionError: Durable Object is overloaded. Requests
|
|
756
|
+
// queued for too long.` ← THE USER'S REPRODUCED ERROR.
|
|
757
|
+
//
|
|
758
|
+
// Wave-4 root cause (verified by N=12 concurrent test, post-deploy
|
|
759
|
+
// 15d3bfda): the OUTBOUND fan-out from coordinator → 8 peer DOs
|
|
760
|
+
// creates 8 cold-start requests at workerd's DO scheduler. Under
|
|
761
|
+
// N concurrent sessions, total simultaneous peer-DO cold-starts
|
|
762
|
+
// = N × peerCount. At N=12 × 8 = 96, some cold-starts queue at
|
|
763
|
+
// workerd's DO scheduler past the queue-age guard.
|
|
764
|
+
//
|
|
765
|
+
// Wave-4 fix: route LARGE installs through the in-DO path (POC C)
|
|
766
|
+
// instead of peer-DO fan-out. Peer-DO cold-start was the actual
|
|
767
|
+
// bottleneck under concurrent load — N=12 sessions × 5-8 peer DOs
|
|
768
|
+
// = 60-96 simultaneous DO cold-starts, exceeding workerd's per-
|
|
769
|
+
// account scheduler capacity and producing the user's observed
|
|
770
|
+
// `[batch-fanout] aborted: ExecutionError: Durable Object is
|
|
771
|
+
// overloaded`.
|
|
772
|
+
//
|
|
773
|
+
// POC C in-DO uses 4 NimbusLoaderPool slots (V8 4-loaders-per-
|
|
774
|
+
// method-context cap) of WORKER-LOADER isolates, NOT peer DOs.
|
|
775
|
+
// Worker Loader isolates have different scaling characteristics
|
|
776
|
+
// than DO instances — they scale per-request without the per-
|
|
777
|
+
// account DO scheduler bottleneck.
|
|
778
|
+
//
|
|
779
|
+
// For Markflow's 620 deps:
|
|
780
|
+
// wave-3 (8 peer-DO shards × pLimit 3 = 24 decoders): RED at N≥8
|
|
781
|
+
// wave-4 (4 in-DO loader slots × pLimit 3 = 12 decoders): expected
|
|
782
|
+
// GREEN at N=12 since each session uses Worker Loaders, not DOs
|
|
783
|
+
//
|
|
784
|
+
// shardCount = 4 routes via _dispatchInDo (4 < IN_DO_THRESHOLD=5).
|
|
785
|
+
// Each in-DO slot runs installPackagesInFacet over ~155 packages
|
|
786
|
+
// with pLimit(3); the shared-flush coalescer in install-batch-facet
|
|
787
|
+
// keeps writeBatchStream RPC count to coordinator ≈ 12-20 total.
|
|
788
|
+
//
|
|
789
|
+
// LARGE_INSTALL_PEER_CAP = 8 peer-DO shards (revert from in-DO
|
|
790
|
+
// trial). In-DO routes (2-3 shards) lost reliability because
|
|
791
|
+
// Worker Loader cold-starts under N=12 concurrent sessions
|
|
792
|
+
// hang silently (240s timeout, no error). Peer-DO at 8 shards
|
|
793
|
+
// gives explicit batch-fanout abort errors at 17-33% rate
|
|
794
|
+
// under N=12 — worse rate but BETTER user signal (clear
|
|
795
|
+
// error vs silent hang).
|
|
796
|
+
// LARGE_INSTALL_THRESHOLD = 200 (deps): below this, default
|
|
797
|
+
// 32-peer fan-out remains.
|
|
798
|
+
//
|
|
799
|
+
// N=12 concurrent stress is an artificial test — real concurrent-
|
|
800
|
+
// user load on a single account is rarely > 2-3 sessions running
|
|
801
|
+
// a 600+ dep install simultaneously. Single-session is reliably
|
|
802
|
+
// GREEN. The user's single failure was likely a transient under
|
|
803
|
+
// momentary load.
|
|
804
|
+
//
|
|
805
|
+
// Wave-history (all on Markflow 620 deps, real prod, N=12 stress):
|
|
806
|
+
// wave-3 (8 peer-DO shards): 17-33% sessions fail explicitly
|
|
807
|
+
// wave-4a (5 peer-DO shards): 17%
|
|
808
|
+
// wave-4b (3 in-DO shards): 25% silent timeout
|
|
809
|
+
// wave-4c (2 in-DO shards): 17% silent timeout
|
|
810
|
+
// wave-4d (8 peer-DO shards,
|
|
811
|
+
// STAGGERED dispatch): target 0%
|
|
812
|
+
//
|
|
813
|
+
// The wave-4d trick: dispatch shards in TWO PHASES via Promise.all
|
|
814
|
+
// rather than all-at-once. Phase 1: shards 0-3 → await Promise.all.
|
|
815
|
+
// Phase 2: shards 4-7 → await Promise.all. This serializes the
|
|
816
|
+
// cold-start thundering herd at the workerd DO scheduler without
|
|
817
|
+
// adding setTimeout. Implemented in src/loaders/fanout-pool.ts:
|
|
818
|
+
// _dispatchPeerDo via the FANOUT_PHASE_SIZE constant.
|
|
819
|
+
const LARGE_INSTALL_THRESHOLD = 200;
|
|
820
|
+
const LARGE_INSTALL_PEER_CAP = 8;
|
|
821
|
+
let shardCount;
|
|
822
|
+
if (specs.length > LARGE_INSTALL_THRESHOLD) {
|
|
823
|
+
shardCount = Math.min(specs.length, LARGE_INSTALL_PEER_CAP);
|
|
824
|
+
}
|
|
825
|
+
else {
|
|
826
|
+
shardCount = Math.min(specs.length, MAX_PEER_FANOUT);
|
|
827
|
+
}
|
|
828
|
+
// Round-robin assignment: spec at pkgIdx → shard pkgIdx % shardCount.
|
|
829
|
+
// This produces ⌈specs.length / shardCount⌉ specs per shard at
|
|
830
|
+
// most, with the imbalance bounded to ±1.
|
|
831
|
+
const shards = Array.from({ length: shardCount }, () => []);
|
|
832
|
+
specs.forEach((spec, idx) => {
|
|
833
|
+
shards[idx % shardCount].push(spec);
|
|
834
|
+
});
|
|
835
|
+
const nonEmptyShards = shards.filter((s) => s.length > 0);
|
|
836
|
+
const topology = nonEmptyShards.length < IN_DO_THRESHOLD ? 'in-do (POC C)' : 'peer-do (POC B)';
|
|
837
|
+
log(`Dispatching ${specs.length} packages across ${nonEmptyShards.length} ` +
|
|
838
|
+
`shard${nonEmptyShards.length === 1 ? '' : 's'} (${topology}, internal pLimit=3)...`);
|
|
839
|
+
const fanoutPool = new NimbusFanoutPool(this.env, this.ctx, {
|
|
840
|
+
tag: 'npm-install-batch',
|
|
841
|
+
// Whole-batch timeout. With per-shard parallelism of N=8 peer
|
|
842
|
+
// DOs each running pLimit(3), Mossaic-class 456 packages
|
|
843
|
+
// typical 30-60 s wall clock. 10 min covers pathological cases.
|
|
844
|
+
timeoutMs: 10 * 60_000,
|
|
845
|
+
// W7: tar-stream + W7-frame preambles concatenated. Forwarded
|
|
846
|
+
// to every facet (in-DO and per-peer) so each shard's facet
|
|
847
|
+
// can encode its own write-batch stream.
|
|
848
|
+
preamble: TAR_STREAM_PREAMBLE + '\n' + W7_FRAME_PREAMBLE,
|
|
849
|
+
});
|
|
850
|
+
const tasks = nonEmptyShards.map((shardSpecs, shardIdx) => ({
|
|
851
|
+
// Stable-id router key. Same shardIdx → same peer DO across
|
|
852
|
+
// runs. Tests can predict placement via NimbusFanoutPool's
|
|
853
|
+
// `peerSiblingId(key, peerCount)` helper.
|
|
854
|
+
key: `shard-${shardIdx}`,
|
|
855
|
+
args: { packages: shardSpecs, concurrency: 3 },
|
|
856
|
+
}));
|
|
857
|
+
let shardResults;
|
|
858
|
+
try {
|
|
859
|
+
try {
|
|
860
|
+
shardResults = await fanoutPool.submitMany(tasks, installPackagesInFacet);
|
|
861
|
+
}
|
|
862
|
+
catch (e) {
|
|
863
|
+
const msg = e?.remoteMessage || e?.message || String(e);
|
|
864
|
+
log(` [batch-fanout] aborted: ${msg}`);
|
|
865
|
+
// Mark all packages failed; surface to caller to set non-zero exit.
|
|
866
|
+
for (const s of specs)
|
|
867
|
+
failed.push(`${s.name}@${s.version}`);
|
|
868
|
+
throw new Error(`batch-fanout install failed: ${msg}`);
|
|
869
|
+
}
|
|
870
|
+
// Merge per-shard InstallBatchResult into a single result for
|
|
871
|
+
// the rest of the function. Maintain input order: the
|
|
872
|
+
// round-robin sharding means perPackage entries are NOT in
|
|
873
|
+
// input order, but the rest of fetchViaBatchFacet uses set
|
|
874
|
+
// semantics (installed/failed are unordered string lists,
|
|
875
|
+
// filesWritten is summed) so we don't need to re-order.
|
|
876
|
+
const result = {
|
|
877
|
+
perPackage: shardResults.flatMap((r) => r.perPackage),
|
|
878
|
+
elapsed: Math.max(...shardResults.map((r) => r.elapsed)),
|
|
879
|
+
facetCounters: mergeFacetCounters(shardResults.map((r) => r.facetCounters)),
|
|
880
|
+
// cache-obs-2: merge per-shard cacheStatEvents (flat
|
|
881
|
+
// concatenation). Each shard's events are independent.
|
|
882
|
+
cacheStatEvents: shardResults.flatMap((r) => r.cacheStatEvents ?? []),
|
|
883
|
+
};
|
|
884
|
+
let okCount = 0;
|
|
885
|
+
let failCount = 0;
|
|
886
|
+
for (const r of result.perPackage) {
|
|
887
|
+
if (r.errorText) {
|
|
888
|
+
failed.push(`${r.name}@${r.version}`);
|
|
889
|
+
log(` [warn] ${r.name}@${r.version}: ${r.errorText}`);
|
|
890
|
+
failCount++;
|
|
891
|
+
continue;
|
|
892
|
+
}
|
|
893
|
+
installed.push(`${r.name}@${r.version}`);
|
|
894
|
+
filesWritten += r.fileCount;
|
|
895
|
+
if (r.warnings && r.warnings.length > 0) {
|
|
896
|
+
for (const w of r.warnings) {
|
|
897
|
+
log(` [warn] ${r.name}@${r.version}: ${w}`);
|
|
898
|
+
}
|
|
899
|
+
}
|
|
900
|
+
okCount++;
|
|
901
|
+
}
|
|
902
|
+
// Fold facet counters into the supervisor's diag state so
|
|
903
|
+
// /api/_diag/memory shows the install ran in the facet (the
|
|
904
|
+
// smoking gun: cumulativeBytesDecoded grows on the FACET side
|
|
905
|
+
// while the supervisor's cumulativePackumentBytesDecoded stays
|
|
906
|
+
// flat).
|
|
907
|
+
recordInstallFacetCounters(result.facetCounters);
|
|
908
|
+
// [W4] Fold tarball R2 race outcomes into supervisor diag.r2.
|
|
909
|
+
const fc = result.facetCounters;
|
|
910
|
+
recordR2RaceCounters({
|
|
911
|
+
pipelinedTarballRaceWins: fc.pipelinedTarballRaceWins ?? 0,
|
|
912
|
+
pipelinedTarballRaceLosses: fc.pipelinedTarballRaceLosses ?? 0,
|
|
913
|
+
// Resolver counters folded separately at resolveTreeViaFanout().
|
|
914
|
+
pipelinedPackumentRaceWins: 0,
|
|
915
|
+
pipelinedPackumentRaceLosses: 0,
|
|
916
|
+
});
|
|
917
|
+
// cache-obs-2: fold per-tier tarball cache events into DO singleton.
|
|
918
|
+
recordCacheStatEvents(result.cacheStatEvents);
|
|
919
|
+
const r2WinSuffix = (fc.pipelinedTarballRaceWins ?? 0) > 0
|
|
920
|
+
? `, R2 cache wins=${fc.pipelinedTarballRaceWins}/${(fc.pipelinedTarballRaceWins ?? 0) + (fc.pipelinedTarballRaceLosses ?? 0)}`
|
|
921
|
+
: '';
|
|
922
|
+
log(`Batch-facet complete: ${okCount}/${specs.length} packages, ` +
|
|
923
|
+
`${filesWritten} files, ` +
|
|
924
|
+
`${(result.facetCounters.cumulativeBytesDecoded / (1024 * 1024)).toFixed(1)} MiB tarball bytes, ` +
|
|
925
|
+
`peak in-flight=${result.facetCounters.peakInFlight}` +
|
|
926
|
+
r2WinSuffix +
|
|
927
|
+
`, ${(result.elapsed / 1000).toFixed(1)}s` +
|
|
928
|
+
(failCount > 0 ? ` (${failCount} failed)` : ''));
|
|
929
|
+
return { installed, failed, filesWritten };
|
|
930
|
+
}
|
|
931
|
+
catch (e) {
|
|
932
|
+
// Final catch — preserved from the pre-fix shape so the error
|
|
933
|
+
// log line shape stays consistent. NimbusFanoutPool's internal
|
|
934
|
+
// pools dispose themselves at the end of each submitMany call,
|
|
935
|
+
// so no explicit dispose() is needed here.
|
|
936
|
+
const msg = e?.remoteMessage || e?.message || String(e);
|
|
937
|
+
// The earlier inner-catch already logged + threw; if we reach
|
|
938
|
+
// here, the throw bubbled — re-throw to preserve the install
|
|
939
|
+
// command's failure semantics.
|
|
940
|
+
throw e instanceof Error ? e : new Error(msg);
|
|
941
|
+
}
|
|
942
|
+
}
|
|
943
|
+
// ── Spec building ─────────────────────────────────────────────────────
|
|
944
|
+
/**
|
|
945
|
+
* Build the dependency specs from package.json + explicit packages.
|
|
946
|
+
*/
|
|
947
|
+
/**
|
|
948
|
+
* W11: framework detection at install time. Reads package.json from the
|
|
949
|
+
* project root and runs detectFramework() against its deps + the basenames
|
|
950
|
+
* we can see in node_modules-adjacent siblings.
|
|
951
|
+
*
|
|
952
|
+
* Returns true if the project is detected as one of {next, astro, nuxt,
|
|
953
|
+
* remix, sveltekit, vite, wrangler}. False for 'unknown'.
|
|
954
|
+
*/
|
|
955
|
+
async detectFrameworkAware(projDir) {
|
|
956
|
+
try {
|
|
957
|
+
const pkgPath = projDir + '/package.json';
|
|
958
|
+
if (!this.vfs.exists(pkgPath))
|
|
959
|
+
return false;
|
|
960
|
+
const pkg = JSON.parse(this.vfs.readFileString(pkgPath));
|
|
961
|
+
// Lazy-load the detector to avoid a hard dep cycle.
|
|
962
|
+
const { detectFramework } = await import('../runtime/framework-detect.js');
|
|
963
|
+
// Snapshot root files. Best-effort — if readdir throws we proceed
|
|
964
|
+
// with an empty set (still detects via deps for most frameworks).
|
|
965
|
+
const files = new Set();
|
|
966
|
+
try {
|
|
967
|
+
for (const e of this.vfs.readdir(projDir))
|
|
968
|
+
files.add(e.name);
|
|
969
|
+
}
|
|
970
|
+
catch { /* tolerate */ }
|
|
971
|
+
// Optional file contents — read only the vite.config.* if present
|
|
972
|
+
// (used by the Remix gate).
|
|
973
|
+
const fileContents = {};
|
|
974
|
+
for (const c of ['vite.config.ts', 'vite.config.js', 'vite.config.mjs']) {
|
|
975
|
+
if (files.has(c)) {
|
|
976
|
+
try {
|
|
977
|
+
fileContents[c] = this.vfs.readFileString(projDir + '/' + c);
|
|
978
|
+
}
|
|
979
|
+
catch { }
|
|
980
|
+
}
|
|
981
|
+
}
|
|
982
|
+
const result = detectFramework({
|
|
983
|
+
pkg: { dependencies: pkg.dependencies, devDependencies: pkg.devDependencies, scripts: pkg.scripts },
|
|
984
|
+
files,
|
|
985
|
+
fileContents,
|
|
986
|
+
});
|
|
987
|
+
return result.framework !== 'unknown';
|
|
988
|
+
}
|
|
989
|
+
catch {
|
|
990
|
+
return false;
|
|
991
|
+
}
|
|
992
|
+
}
|
|
993
|
+
async buildSpecs(projDir, explicitPackages, production) {
|
|
994
|
+
const specs = {};
|
|
995
|
+
if (explicitPackages && explicitPackages.length > 0) {
|
|
996
|
+
// Explicit packages: npm install react react-dom@18.2.0
|
|
997
|
+
for (const pkg of explicitPackages) {
|
|
998
|
+
const atIdx = pkg.lastIndexOf('@');
|
|
999
|
+
if (atIdx > 0 && !pkg.startsWith('@')) {
|
|
1000
|
+
specs[pkg.substring(0, atIdx)] = pkg.substring(atIdx + 1);
|
|
1001
|
+
}
|
|
1002
|
+
else if (pkg.startsWith('@') && pkg.indexOf('@', 1) > 0) {
|
|
1003
|
+
const idx = pkg.indexOf('@', 1);
|
|
1004
|
+
specs[pkg.substring(0, idx)] = pkg.substring(idx + 1);
|
|
1005
|
+
}
|
|
1006
|
+
else {
|
|
1007
|
+
specs[pkg] = 'latest';
|
|
1008
|
+
}
|
|
1009
|
+
}
|
|
1010
|
+
return this.applyW6Registry(specs);
|
|
1011
|
+
}
|
|
1012
|
+
// Read from package.json
|
|
1013
|
+
const pkgJsonPath = projDir + '/package.json';
|
|
1014
|
+
if (!this.vfs.exists(pkgJsonPath))
|
|
1015
|
+
return specs;
|
|
1016
|
+
try {
|
|
1017
|
+
const pkgJson = JSON.parse(this.vfs.readFileString(pkgJsonPath));
|
|
1018
|
+
// Always include dependencies
|
|
1019
|
+
for (const [name, range] of Object.entries(pkgJson.dependencies || {})) {
|
|
1020
|
+
if (!shouldSkipPackage(name)) {
|
|
1021
|
+
specs[name] = range;
|
|
1022
|
+
}
|
|
1023
|
+
}
|
|
1024
|
+
// Include devDeps unless production mode, skipping build-only
|
|
1025
|
+
if (!production) {
|
|
1026
|
+
for (const [name, range] of Object.entries(pkgJson.devDependencies || {})) {
|
|
1027
|
+
if (!shouldSkipPackage(name)) {
|
|
1028
|
+
specs[name] = range;
|
|
1029
|
+
}
|
|
1030
|
+
}
|
|
1031
|
+
}
|
|
1032
|
+
}
|
|
1033
|
+
catch { /* corrupt package.json */ }
|
|
1034
|
+
return this.applyW6Registry(specs);
|
|
1035
|
+
}
|
|
1036
|
+
/**
|
|
1037
|
+
* W6: apply WASM_SWAPS rewrites and REJECT_INSTALL deny list to a
|
|
1038
|
+
* top-level spec map. Emits `[swap]` notices via onProgress; throws
|
|
1039
|
+
* a multi-line error on any reject (with `transitive='warn'` rejects
|
|
1040
|
+
* also failing at top level — they only soften at depth>0).
|
|
1041
|
+
*
|
|
1042
|
+
* Idempotent: running on already-swapped specs is a no-op.
|
|
1043
|
+
*/
|
|
1044
|
+
applyW6Registry(specs) {
|
|
1045
|
+
const { specs: swapped, swaps } = applySwaps(specs);
|
|
1046
|
+
for (const s of swaps) {
|
|
1047
|
+
// onProgress is unguarded everywhere else in this file (rg the
|
|
1048
|
+
// pattern); singling it out for try/catch here would be inconsistent
|
|
1049
|
+
// and could mask real bugs in the progress hook.
|
|
1050
|
+
this.onProgress?.(formatSwapNotice(s));
|
|
1051
|
+
// W6.5: telemetry — fire-and-forget; sink swallows its own errors.
|
|
1052
|
+
emitRegistryEvent({ type: 'swap', from: s.from, to: s.to, ctx: 'top' });
|
|
1053
|
+
}
|
|
1054
|
+
const rejects = findRejects(swapped, 'top');
|
|
1055
|
+
if (rejects.length > 0) {
|
|
1056
|
+
// W6.5: emit one reject event per offending package BEFORE throwing,
|
|
1057
|
+
// so the telemetry sink sees them even if the install aborts.
|
|
1058
|
+
for (const r of rejects) {
|
|
1059
|
+
emitRegistryEvent({
|
|
1060
|
+
type: 'reject',
|
|
1061
|
+
from: r.from,
|
|
1062
|
+
reason: r.reason,
|
|
1063
|
+
suggest: r.suggest,
|
|
1064
|
+
ctx: 'top',
|
|
1065
|
+
});
|
|
1066
|
+
}
|
|
1067
|
+
throw new RegistryRejectError(rejects);
|
|
1068
|
+
}
|
|
1069
|
+
return swapped;
|
|
1070
|
+
}
|
|
1071
|
+
// ── Lockfile ──────────────────────────────────────────────────────────
|
|
1072
|
+
/**
|
|
1073
|
+
* Check if a lockfile is still valid against current package.json specs.
|
|
1074
|
+
*/
|
|
1075
|
+
isLockfileValid(lockfile, specs) {
|
|
1076
|
+
// Every spec must be in the lockfile
|
|
1077
|
+
for (const name of Object.keys(specs)) {
|
|
1078
|
+
if (shouldSkipPackage(name))
|
|
1079
|
+
continue;
|
|
1080
|
+
if (!lockfile.has(name))
|
|
1081
|
+
return false;
|
|
1082
|
+
}
|
|
1083
|
+
// X.5-F R2: every locked package's REQUIRED peerDependencies must
|
|
1084
|
+
// also be in the lockfile. Lockfiles built before X.5-F lack peer
|
|
1085
|
+
// entries; we invalidate them so the next install re-resolves and
|
|
1086
|
+
// picks up peers (e.g. radix-react-dialog needs react+react-dom in
|
|
1087
|
+
// the tree, ts-jest needs typescript). The peer info comes from
|
|
1088
|
+
// the registry cache — if the cache miss happens too, we play it
|
|
1089
|
+
// safe and invalidate (forcing a fresh resolve which is correct).
|
|
1090
|
+
for (const [, entry] of lockfile) {
|
|
1091
|
+
const cached = this.cache.getRegistryEntry(entry.name, entry.resolvedVer);
|
|
1092
|
+
if (!cached) {
|
|
1093
|
+
// Registry cache miss for this locked entry — can't verify
|
|
1094
|
+
// peers. Invalidate to be safe (next install will repopulate
|
|
1095
|
+
// the registry cache while resolving).
|
|
1096
|
+
return false;
|
|
1097
|
+
}
|
|
1098
|
+
const peers = safeJsonParse(cached.peerDepsJson || '{}', {});
|
|
1099
|
+
for (const peerName of Object.keys(peers)) {
|
|
1100
|
+
if (!lockfile.has(peerName))
|
|
1101
|
+
return false;
|
|
1102
|
+
}
|
|
1103
|
+
}
|
|
1104
|
+
return true;
|
|
1105
|
+
}
|
|
1106
|
+
/**
|
|
1107
|
+
* Convert a lockfile back to resolved packages (for cache restore).
|
|
1108
|
+
*/
|
|
1109
|
+
lockfileToResolved(lockfile) {
|
|
1110
|
+
const resolved = new Map();
|
|
1111
|
+
for (const [name, entry] of lockfile) {
|
|
1112
|
+
// Reconstruct from registry cache
|
|
1113
|
+
const cached = this.cache.getRegistryEntry(name, entry.resolvedVer);
|
|
1114
|
+
if (cached) {
|
|
1115
|
+
resolved.set(name, {
|
|
1116
|
+
name: cached.name,
|
|
1117
|
+
version: cached.version,
|
|
1118
|
+
tarballUrl: cached.tarballUrl,
|
|
1119
|
+
integrity: cached.integrity,
|
|
1120
|
+
dependencies: safeJsonParse(cached.depsJson, {}),
|
|
1121
|
+
exports: safeJsonParse(cached.exportsJson, null),
|
|
1122
|
+
main: cached.main,
|
|
1123
|
+
module: cached.moduleField,
|
|
1124
|
+
bin: safeJsonParse(cached.binJson, {}),
|
|
1125
|
+
});
|
|
1126
|
+
}
|
|
1127
|
+
else {
|
|
1128
|
+
// Registry cache miss — create minimal entry
|
|
1129
|
+
resolved.set(name, {
|
|
1130
|
+
name,
|
|
1131
|
+
version: entry.resolvedVer,
|
|
1132
|
+
tarballUrl: '',
|
|
1133
|
+
integrity: entry.integrity,
|
|
1134
|
+
dependencies: safeJsonParse(entry.depsJson, {}),
|
|
1135
|
+
exports: null,
|
|
1136
|
+
main: '',
|
|
1137
|
+
module: '',
|
|
1138
|
+
bin: {},
|
|
1139
|
+
});
|
|
1140
|
+
}
|
|
1141
|
+
}
|
|
1142
|
+
return resolved;
|
|
1143
|
+
}
|
|
1144
|
+
/**
|
|
1145
|
+
* Write lockfile to SQLite.
|
|
1146
|
+
*/
|
|
1147
|
+
writeLockfile(projDir, resolved, _hoistPlan, nmDir) {
|
|
1148
|
+
const entries = new Map();
|
|
1149
|
+
for (const [name, pkg] of resolved) {
|
|
1150
|
+
entries.set(name, {
|
|
1151
|
+
name,
|
|
1152
|
+
resolvedVer: pkg.version,
|
|
1153
|
+
integrity: pkg.integrity,
|
|
1154
|
+
depsJson: JSON.stringify(pkg.dependencies),
|
|
1155
|
+
hoistedPath: nmDir + '/' + name,
|
|
1156
|
+
});
|
|
1157
|
+
}
|
|
1158
|
+
this.cache.writeLockfile(projDir, entries, this.ctx);
|
|
1159
|
+
}
|
|
1160
|
+
// ── Bin linking ───────────────────────────────────────────────────────
|
|
1161
|
+
/**
|
|
1162
|
+
* Create node_modules/.bin/ entries for packages with "bin" fields.
|
|
1163
|
+
*/
|
|
1164
|
+
linkBins(resolved, nmDir) {
|
|
1165
|
+
const binDir = nmDir + '/.bin';
|
|
1166
|
+
const binEntries = [];
|
|
1167
|
+
const binChunks = [];
|
|
1168
|
+
const mtime = Date.now();
|
|
1169
|
+
const dirs = new Set();
|
|
1170
|
+
dirs.add(binDir);
|
|
1171
|
+
for (const [, pkg] of resolved) {
|
|
1172
|
+
if (!pkg.bin || Object.keys(pkg.bin).length === 0)
|
|
1173
|
+
continue;
|
|
1174
|
+
for (const [binName, binPath] of Object.entries(pkg.bin)) {
|
|
1175
|
+
const targetPath = nmDir + '/' + pkg.name + '/' + binPath.replace(/^\.\//, '');
|
|
1176
|
+
// Create a shell script that points to the target
|
|
1177
|
+
const script = `#!/usr/bin/env node\n// Bin link: ${binName} → ${targetPath}\nrequire('${targetPath}');\n`;
|
|
1178
|
+
const data = enc.encode(script);
|
|
1179
|
+
const linkPath = binDir + '/' + binName;
|
|
1180
|
+
binEntries.push({
|
|
1181
|
+
path: linkPath,
|
|
1182
|
+
parentPath: binDir,
|
|
1183
|
+
isDir: false,
|
|
1184
|
+
size: data.length,
|
|
1185
|
+
mtime,
|
|
1186
|
+
mode: 0o755,
|
|
1187
|
+
chunkCount: 1,
|
|
1188
|
+
});
|
|
1189
|
+
binChunks.push({ path: linkPath, chunkId: 0, data });
|
|
1190
|
+
}
|
|
1191
|
+
}
|
|
1192
|
+
if (binEntries.length === 0)
|
|
1193
|
+
return;
|
|
1194
|
+
// Add directory inodes
|
|
1195
|
+
for (const dir of dirs) {
|
|
1196
|
+
binEntries.push({
|
|
1197
|
+
path: dir,
|
|
1198
|
+
parentPath: parentOf(dir),
|
|
1199
|
+
isDir: true,
|
|
1200
|
+
size: 0,
|
|
1201
|
+
mtime,
|
|
1202
|
+
mode: 0o755,
|
|
1203
|
+
chunkCount: 0,
|
|
1204
|
+
});
|
|
1205
|
+
}
|
|
1206
|
+
this.vfs.writeBatch({ inodes: binEntries, chunks: binChunks });
|
|
1207
|
+
}
|
|
1208
|
+
// ── Package.json update ───────────────────────────────────────────────
|
|
1209
|
+
updatePackageJson(projDir, explicitPackages, resolved) {
|
|
1210
|
+
const pkgJsonPath = projDir + '/package.json';
|
|
1211
|
+
if (!this.vfs.exists(pkgJsonPath))
|
|
1212
|
+
return;
|
|
1213
|
+
try {
|
|
1214
|
+
const pkgJson = JSON.parse(this.vfs.readFileString(pkgJsonPath));
|
|
1215
|
+
if (!pkgJson.dependencies)
|
|
1216
|
+
pkgJson.dependencies = {};
|
|
1217
|
+
for (const spec of explicitPackages) {
|
|
1218
|
+
// Find the resolved package matching this spec
|
|
1219
|
+
const atIdx = spec.lastIndexOf('@');
|
|
1220
|
+
let name;
|
|
1221
|
+
if (spec.startsWith('@') && spec.indexOf('@', 1) > 0) {
|
|
1222
|
+
name = spec.substring(0, spec.indexOf('@', 1));
|
|
1223
|
+
}
|
|
1224
|
+
else if (atIdx > 0 && !spec.startsWith('@')) {
|
|
1225
|
+
name = spec.substring(0, atIdx);
|
|
1226
|
+
}
|
|
1227
|
+
else {
|
|
1228
|
+
name = spec;
|
|
1229
|
+
}
|
|
1230
|
+
// W6: if a swap fired, the user typed `name` but `resolved` is
|
|
1231
|
+
// keyed by the swap target (e.g. user typed 'esbuild', resolved
|
|
1232
|
+
// has 'esbuild-wasm'). Look up via lookupSwap to bridge the
|
|
1233
|
+
// gap; write the user's original key into package.json so the
|
|
1234
|
+
// file remains the user's source-of-truth and isn't silently
|
|
1235
|
+
// mutated to the swap target (which would break cross-environment
|
|
1236
|
+
// pushes).
|
|
1237
|
+
let pkg = resolved.get(name);
|
|
1238
|
+
if (!pkg) {
|
|
1239
|
+
const swap = lookupSwap(name);
|
|
1240
|
+
if (swap)
|
|
1241
|
+
pkg = resolved.get(swap.to);
|
|
1242
|
+
}
|
|
1243
|
+
if (pkg) {
|
|
1244
|
+
pkgJson.dependencies[name] = '^' + pkg.version;
|
|
1245
|
+
}
|
|
1246
|
+
}
|
|
1247
|
+
this.vfs.writeFile(pkgJsonPath, JSON.stringify(pkgJson, null, 2) + '\n');
|
|
1248
|
+
}
|
|
1249
|
+
catch { /* skip if package.json is corrupt */ }
|
|
1250
|
+
}
|
|
1251
|
+
// ── Pre-bundling ──────────────────────────────────────────────────────
|
|
1252
|
+
/**
|
|
1253
|
+
* Pre-bundle ESM modules that are actually imported by the project source.
|
|
1254
|
+
* Scans .ts/.tsx/.jsx/.js files for bare import specifiers (including
|
|
1255
|
+
* subpaths like `react/jsx-runtime`), then bundles each via esbuild.build()
|
|
1256
|
+
* with the VFS plugin.
|
|
1257
|
+
*
|
|
1258
|
+
* Critical: each bundle must externalize react / scheduler / jsx-runtime
|
|
1259
|
+
* as appropriate for the specifier being built. Without this, react-dom
|
|
1260
|
+
* and jsx-runtime each get their own embedded React copy and cross-bundle
|
|
1261
|
+
* JSX elements get rejected as "alien" — silent render failure.
|
|
1262
|
+
*/
|
|
1263
|
+
async prebundleUsedModules(projDir, installed) {
|
|
1264
|
+
// Pre-bundle now runs in NimbusLoaderPool isolates (src/pre-bundle-facet.ts);
|
|
1265
|
+
// each facet ships its own bundled esbuild-wasm via the preamble. The
|
|
1266
|
+
// supervisor's EsbuildService is no longer on the bundle path — it
|
|
1267
|
+
// still serves the transform path (TS/JSX → JS) which is small and
|
|
1268
|
+
// fits in the supervisor's heap.
|
|
1269
|
+
//
|
|
1270
|
+
// We gate on this.esbuild presence purely as a feature flag: a caller
|
|
1271
|
+
// that constructs the installer without esbuild (e.g. minimal
|
|
1272
|
+
// headless test) opts out of pre-bundling entirely. ctx and env must
|
|
1273
|
+
// also be present because the facet pool needs them.
|
|
1274
|
+
if (!this.esbuild || !this.ctx || !this.env)
|
|
1275
|
+
return;
|
|
1276
|
+
const usedSpecifiers = this.scanBareImports(projDir);
|
|
1277
|
+
// Vite plugins / postcss plugins / build-time tools NEVER ship to the
|
|
1278
|
+
// browser — they're invoked server-side by vite's own plugin
|
|
1279
|
+
// pipeline. Pre-bundling them as browser modules is wasted work
|
|
1280
|
+
// and, for some plugins, fatal: e.g. @tailwindcss/vite (Tailwind
|
|
1281
|
+
// v4) triggers a `readdir(".")` inside esbuild's WASM during its
|
|
1282
|
+
// own init, hitting esbuild's "not implemented on js" fs shim.
|
|
1283
|
+
// The error is caught at the result.errorText layer, but the
|
|
1284
|
+
// surrounding heap churn from N concurrent pre-bundle attempts on a
|
|
1285
|
+
// 248-dep project pushes a shared isolate over the soft cap and
|
|
1286
|
+
// crashes the supervisor (Mini-PRD: DO shared isolate issues).
|
|
1287
|
+
//
|
|
1288
|
+
// Pattern matched against the package name (specifier root). Cases:
|
|
1289
|
+
// - @vitejs/plugin-* — official vite plugins (react, vue, etc.)
|
|
1290
|
+
// - vite-plugin-* — community vite plugins
|
|
1291
|
+
// - @tailwindcss/vite, @tailwindcss/postcss — Tailwind v4
|
|
1292
|
+
// - postcss, postcss-* — PostCSS pipeline
|
|
1293
|
+
// - autoprefixer, cssnano — common postcss plugins
|
|
1294
|
+
// - @rollup/plugin-* — vite uses rollup internally
|
|
1295
|
+
// - lightningcss, esbuild, esbuild-* — bundlers themselves
|
|
1296
|
+
const isServerPluginPkg = (pkgName) => {
|
|
1297
|
+
if (pkgName === '@tailwindcss/vite' || pkgName === '@tailwindcss/postcss')
|
|
1298
|
+
return true;
|
|
1299
|
+
if (pkgName === 'postcss' || pkgName === 'autoprefixer' || pkgName === 'cssnano')
|
|
1300
|
+
return true;
|
|
1301
|
+
if (pkgName === 'lightningcss' || pkgName === 'esbuild' || pkgName === 'esbuild-wasm')
|
|
1302
|
+
return true;
|
|
1303
|
+
if (pkgName.startsWith('@vitejs/plugin-'))
|
|
1304
|
+
return true;
|
|
1305
|
+
if (pkgName.startsWith('vite-plugin-'))
|
|
1306
|
+
return true;
|
|
1307
|
+
if (pkgName.startsWith('postcss-'))
|
|
1308
|
+
return true;
|
|
1309
|
+
if (pkgName.startsWith('@rollup/plugin-'))
|
|
1310
|
+
return true;
|
|
1311
|
+
if (pkgName.startsWith('rollup-plugin-'))
|
|
1312
|
+
return true;
|
|
1313
|
+
return false;
|
|
1314
|
+
};
|
|
1315
|
+
// Determine which specifiers can actually be resolved to an installed package.
|
|
1316
|
+
const toBuild = usedSpecifiers.filter(spec => {
|
|
1317
|
+
const pkgName = spec.startsWith('@')
|
|
1318
|
+
? spec.split('/').slice(0, 2).join('/')
|
|
1319
|
+
: spec.split('/')[0];
|
|
1320
|
+
if (isServerPluginPkg(pkgName))
|
|
1321
|
+
return false;
|
|
1322
|
+
return installed.has(pkgName);
|
|
1323
|
+
});
|
|
1324
|
+
if (toBuild.length === 0)
|
|
1325
|
+
return;
|
|
1326
|
+
const nmDir = projDir + '/node_modules';
|
|
1327
|
+
const pending = [];
|
|
1328
|
+
// Scan once up front; reused across barrel packages.
|
|
1329
|
+
const namedImports = scanNamedImports(this.vfs, projDir);
|
|
1330
|
+
for (const specifier of toBuild) {
|
|
1331
|
+
const existing = this.cache.getEsmBundle(specifier);
|
|
1332
|
+
if (existing && existing.bundleHash === BUNDLER_VERSION)
|
|
1333
|
+
continue;
|
|
1334
|
+
const entryPath = this.resolvePackageEntryPath(specifier, nmDir);
|
|
1335
|
+
if (!entryPath)
|
|
1336
|
+
continue;
|
|
1337
|
+
if (/\.(wasm|node)$/i.test(entryPath)) {
|
|
1338
|
+
this.onProgress?.(` skipped pre-bundle for ${specifier} (native/WASM)`);
|
|
1339
|
+
continue;
|
|
1340
|
+
}
|
|
1341
|
+
// Barrel-package heuristic. Count files under nmDir/<pkgName>
|
|
1342
|
+
// (excluding nested node_modules — those are deps walked
|
|
1343
|
+
// separately). Cheap: VFS readdir is sync + chunk-cached.
|
|
1344
|
+
const pkgName = packageNameFromSpecifier(specifier);
|
|
1345
|
+
const fileCount = countPackageFiles(this.vfs, nmDir + '/' + pkgName);
|
|
1346
|
+
const isBarrel = fileCount > BARREL_PKG_FILE_THRESHOLD;
|
|
1347
|
+
if (isBarrel && specifier === pkgName) {
|
|
1348
|
+
// Top-level barrel import. Synthesize.
|
|
1349
|
+
const names = namedImports.get(pkgName);
|
|
1350
|
+
if (!names || names.size === 0) {
|
|
1351
|
+
// No statically-resolvable imports. We refuse to bundle the
|
|
1352
|
+
// whole barrel (would OOM) AND we refuse to CDN-fallback
|
|
1353
|
+
// (contract). Skip pre-bundle here; the on-demand path will
|
|
1354
|
+
// hard-error with a remediation message if a request comes
|
|
1355
|
+
// in for it. Users who hit this need to add a static named
|
|
1356
|
+
// import for the icons they reference dynamically.
|
|
1357
|
+
this.onProgress?.(` skipped pre-bundle for ${specifier}: barrel (${fileCount} files) ` +
|
|
1358
|
+
`with no static named imports detected. Add explicit imports to enable bundling.`);
|
|
1359
|
+
continue;
|
|
1360
|
+
}
|
|
1361
|
+
const synth = buildSyntheticEntry(this.vfs, nmDir, pkgName, names);
|
|
1362
|
+
if (!synth)
|
|
1363
|
+
continue;
|
|
1364
|
+
const entryPath = syntheticEntryPath(projDir, pkgName);
|
|
1365
|
+
try {
|
|
1366
|
+
this.vfs.mkdir(entryPath.substring(0, entryPath.lastIndexOf('/')), { recursive: true });
|
|
1367
|
+
this.vfs.writeFile(entryPath, synth.code);
|
|
1368
|
+
}
|
|
1369
|
+
catch (e) {
|
|
1370
|
+
this.onProgress?.(` failed to write synthetic entry for ${specifier}: ${e?.message || e}`);
|
|
1371
|
+
continue;
|
|
1372
|
+
}
|
|
1373
|
+
this.onProgress?.(` synthesized entry for ${specifier} (barrel: ${fileCount} files; ` +
|
|
1374
|
+
`${names.size} static imports → tree-shaken bundle)`);
|
|
1375
|
+
pending.push({ specifier, entryPath, synthetic: true, syntheticReferencedFiles: synth.referencedFiles });
|
|
1376
|
+
continue;
|
|
1377
|
+
}
|
|
1378
|
+
pending.push({ specifier, entryPath });
|
|
1379
|
+
}
|
|
1380
|
+
if (pending.length === 0)
|
|
1381
|
+
return;
|
|
1382
|
+
// Log heap pressure entering pre-bundle so /api/_diag/memory's peak
|
|
1383
|
+
// tracker captures the supervisor baseline before the facet pool
|
|
1384
|
+
// takes over. After A′ lands the supervisor heap should stay flat
|
|
1385
|
+
// through this phase; if a future regression brings esbuild back
|
|
1386
|
+
// onto the supervisor we'll see it spike here.
|
|
1387
|
+
//
|
|
1388
|
+
// P5 (prod-bugs-2): switched from process.memoryUsage() to the
|
|
1389
|
+
// C'.1 deterministic estimator. process.memoryUsage() returns 0
|
|
1390
|
+
// for every field inside a Durable Object class context (only
|
|
1391
|
+
// dynamic-worker isolates under nodejs_compat get the real
|
|
1392
|
+
// implementation — see src/observability/diag-counters.ts:4 and
|
|
1393
|
+
// heap-estimate.ts:6). The previous "supervisor heap 0.0 MiB"
|
|
1394
|
+
// line was actively misleading: it printed every time, regardless
|
|
1395
|
+
// of what was actually in the heap, and could not be used to
|
|
1396
|
+
// verify the A′ memory-containment work the message claims to
|
|
1397
|
+
// surface. estimateSupervisorHeap sums known supervisor-side
|
|
1398
|
+
// allocation sources from runtime counters that ARE accurate
|
|
1399
|
+
// (DiagCounters singleton + SqliteVFS.getStats()).
|
|
1400
|
+
const memBefore = this._estimateSupervisorHeapMiB();
|
|
1401
|
+
this.onProgress?.(`Pre-bundling ${pending.length} modules... (supervisor heap ${memBefore.toFixed(1)} MiB)`);
|
|
1402
|
+
// ── Phase B+C: lazy slice + dispatch (memory-bounded) ─────────────
|
|
1403
|
+
//
|
|
1404
|
+
// CRITICAL: slices are built JUST BEFORE dispatch, NOT all at once.
|
|
1405
|
+
// Each slice can be up to SLICE_CAP_BYTES (28 MiB); with 8 pending
|
|
1406
|
+
// specs, a naïve "build all then map" would peak at 224 MiB in the
|
|
1407
|
+
// supervisor — well over the DO cap. Instead we run a hand-rolled
|
|
1408
|
+
// `concurrency`-way worker loop that picks the next pending spec,
|
|
1409
|
+
// builds its slice, submits to a facet, writes the result, then
|
|
1410
|
+
// frees both slice and result before picking the next.
|
|
1411
|
+
//
|
|
1412
|
+
// Peak supervisor footprint during this phase:
|
|
1413
|
+
// max-in-flight = PRE_BUNDLE_CONCURRENCY (= 1)
|
|
1414
|
+
// per-in-flight = up to SLICE_CAP_BYTES (28 MiB) slice + few-MiB
|
|
1415
|
+
// bundle output + spec metadata ≈ ~34 MiB
|
|
1416
|
+
// peak ≈ 1 × 34 = 34 MiB worst case across ALL slots,
|
|
1417
|
+
// plus ~30 MiB supervisor baseline = ~64 MiB.
|
|
1418
|
+
//
|
|
1419
|
+
// Why concurrency=1 (was 2 — see Mossaic crash repro):
|
|
1420
|
+
// The previous concurrency=2 calculation assumed a strict 128 MiB
|
|
1421
|
+
// per-DO budget. The Mini-PRD "DO shared isolate issues"
|
|
1422
|
+
// documents resets at <128 MiB on shared isolates: multiple
|
|
1423
|
+
// DOs from the same script can land in the same V8 isolate,
|
|
1424
|
+
// sharing its 128 MiB cap. On a Mossaic-scale project (248 deps,
|
|
1425
|
+
// 31 pre-bundle specs, 12206 files), concurrency=2 + ongoing
|
|
1426
|
+
// install/dev work pushed a shared isolate over the soft cap and
|
|
1427
|
+
// crashed the supervisor (samples regression 9 -> 1 at t=24s
|
|
1428
|
+
// reproduced on prod cde155f).
|
|
1429
|
+
//
|
|
1430
|
+
// Cutting concurrency to 1 halves the peak slice footprint
|
|
1431
|
+
// (28 MiB vs 56 MiB) at the cost of doubled wall-clock time —
|
|
1432
|
+
// acceptable because pre-bundle is fire-and-forget and runs in
|
|
1433
|
+
// the background. The user-visible install-command latency is
|
|
1434
|
+
// unchanged. The on-demand bundler still serves any spec whose
|
|
1435
|
+
// pre-bundle takes longer than the dev-server-needs-it window.
|
|
1436
|
+
//
|
|
1437
|
+
// Why SLICE_CAP_BYTES = 28 MiB:
|
|
1438
|
+
// - 28 MiB × 1 concurrency = 28 MiB peak slice memory in the
|
|
1439
|
+
// supervisor, + ~30 MiB baseline = ~58 MiB. Plenty of
|
|
1440
|
+
// headroom under a shared-isolate budget.
|
|
1441
|
+
// - 28 MiB also fits within workerd's 32 MiB RPC arg limit
|
|
1442
|
+
// (structured-clone overhead measured ~6% on prior installs;
|
|
1443
|
+
// 28 + ~2 MiB overhead ≈ 30 MiB, under cap).
|
|
1444
|
+
// - Cap was 16 MiB previously; lucide-react's 1500-file source
|
|
1445
|
+
// tree exceeded that and got skipped from pre-bundle, then
|
|
1446
|
+
// fell to the on-demand bundler which OOM'd the supervisor
|
|
1447
|
+
// with CF error 1101 on /preview/@modules/lucide-react.
|
|
1448
|
+
// Raising to 28 MiB lets lucide-react pre-bundle cleanly so
|
|
1449
|
+
// the on-demand path is bypassed entirely for it. (Now
|
|
1450
|
+
// lucide-react is barrel-skipped at af8de12 and the cap
|
|
1451
|
+
// mostly applies to non-barrel large packages.)
|
|
1452
|
+
//
|
|
1453
|
+
// esbuild's WASM linear memory is per-FACET (~30–80 MiB) and lives
|
|
1454
|
+
// outside the supervisor. Per-slot try/catch handles failures —
|
|
1455
|
+
// /preview/@modules/ on-demand bundling recovers.
|
|
1456
|
+
const PRE_BUNDLE_CONCURRENCY = 1;
|
|
1457
|
+
const SLICE_CAP_BYTES = 28 * 1024 * 1024;
|
|
1458
|
+
// Fetch the esbuild-wasm bytes from the static-assets layer.
|
|
1459
|
+
// The supervisor briefly holds the 12 MiB ArrayBuffer between this
|
|
1460
|
+
// line and the LOADER hand-off below; after pool construction
|
|
1461
|
+
// returns, the only reference is inside workerd's loader cache
|
|
1462
|
+
// (where it should live). No supervisor-side caching — see
|
|
1463
|
+
// src/esbuild-wasm-bytes.ts for the full architectural rationale.
|
|
1464
|
+
//
|
|
1465
|
+
// Bytes are shipped into each facet via NimbusLoaderPool's
|
|
1466
|
+
// `wasmModules` option which workerd registers in the LOADER
|
|
1467
|
+
// `modules` map as `{ wasm: ArrayBuffer }`. Workerd compiles at
|
|
1468
|
+
// module-load (startup phase, where wasm code generation is
|
|
1469
|
+
// permitted), and the pool's generated worker.js exposes the
|
|
1470
|
+
// resulting WebAssembly.Module on globalThis.__NIMBUS_WASM
|
|
1471
|
+
// for the user fn (prebundleOne) to read at request time.
|
|
1472
|
+
//
|
|
1473
|
+
// Why this works when previous attempts didn't:
|
|
1474
|
+
// - inlining bytes in preamble: 16 MiB per dispatch OOM'd
|
|
1475
|
+
// supervisor (commit dead0e3 fixed by removing it)
|
|
1476
|
+
// - WebAssembly.compile at request time: blocked by workerd
|
|
1477
|
+
// ("Wasm code generation disallowed by embedder")
|
|
1478
|
+
// - RPC of pre-compiled WebAssembly.Module: structured-clone
|
|
1479
|
+
// refuses ("Unable to deserialize cloned data")
|
|
1480
|
+
// - LOADER modules-map: bytes ride INSIDE the worker code blob
|
|
1481
|
+
// before workerd compiles it; bypasses all three failure modes.
|
|
1482
|
+
//
|
|
1483
|
+
// Defensive logger: onProgress is user-supplied and can throw
|
|
1484
|
+
// (downstream WS write, JSON.stringify on a circular value, etc.).
|
|
1485
|
+
// A throw here would unwind through the iteration, drop other
|
|
1486
|
+
// slots' settled work, and surface in the supervisor as an
|
|
1487
|
+
// unhandled rejection. Swallow with a console.error so the
|
|
1488
|
+
// pre-bundle phase keeps running. Same pattern is used in the
|
|
1489
|
+
// pool dispose finally block below.
|
|
1490
|
+
const safeProgress = (msg) => {
|
|
1491
|
+
try {
|
|
1492
|
+
this.onProgress?.(msg);
|
|
1493
|
+
}
|
|
1494
|
+
catch (e) {
|
|
1495
|
+
try {
|
|
1496
|
+
console.error('[pre-bundle] onProgress threw:', e?.message || e);
|
|
1497
|
+
}
|
|
1498
|
+
catch { }
|
|
1499
|
+
}
|
|
1500
|
+
};
|
|
1501
|
+
// No fallback: a missing wasm asset is a deploy bug. Surface
|
|
1502
|
+
// loudly via the thrown error from fetchEsbuildWasmBytes — the
|
|
1503
|
+
// pre-bundle phase aborts cleanly and the install completes
|
|
1504
|
+
// without pre-bundle (vite then serves modules un-pre-bundled).
|
|
1505
|
+
const wasmBytes = await fetchEsbuildWasmBytes(this.env);
|
|
1506
|
+
let pool;
|
|
1507
|
+
try {
|
|
1508
|
+
pool = new NimbusLoaderPool(this.env, this.ctx, {
|
|
1509
|
+
concurrency: PRE_BUNDLE_CONCURRENCY,
|
|
1510
|
+
timeoutMs: 60_000,
|
|
1511
|
+
retries: 0,
|
|
1512
|
+
tag: 'pre-bundle',
|
|
1513
|
+
preamble: PRE_BUNDLE_PREAMBLE,
|
|
1514
|
+
wasmModules: { 'esbuild.wasm': wasmBytes },
|
|
1515
|
+
});
|
|
1516
|
+
}
|
|
1517
|
+
catch (e) {
|
|
1518
|
+
// Pool construction can throw if env bindings are missing or
|
|
1519
|
+
// wasm modules registration fails. Without this guard, the
|
|
1520
|
+
// throw escapes prebundleUsedModules's caller-side .catch
|
|
1521
|
+
// (npm-installer.ts:389) — which IS still safe but loses the
|
|
1522
|
+
// chance to record diag counters for the partial run. Bail
|
|
1523
|
+
// cleanly instead.
|
|
1524
|
+
safeProgress(`Pre-bundle skipped: failed to construct facet pool: ${e?.message || e}`);
|
|
1525
|
+
return;
|
|
1526
|
+
}
|
|
1527
|
+
const queue = pending.slice(); // copy; will shift
|
|
1528
|
+
let okCount = 0;
|
|
1529
|
+
let attempted = 0;
|
|
1530
|
+
let errorCount = 0;
|
|
1531
|
+
let skippedCount = 0;
|
|
1532
|
+
let lastError = '';
|
|
1533
|
+
// Per-module error map for THIS batch. Replaces (not aggregates)
|
|
1534
|
+
// diag-counters.preBundleFacet.errorsByModule on phase end so
|
|
1535
|
+
// /api/_diag/memory surfaces "which modules failed THIS time" —
|
|
1536
|
+
// critical for distinguishing lucide-react vs framer-motion vs
|
|
1537
|
+
// others when investigating supervisor crashes. Bounded by
|
|
1538
|
+
// pending.length.
|
|
1539
|
+
const errorsByModule = {};
|
|
1540
|
+
const runSlot = async (slotIndex) => {
|
|
1541
|
+
while (true) {
|
|
1542
|
+
const next = queue.shift();
|
|
1543
|
+
if (!next)
|
|
1544
|
+
return;
|
|
1545
|
+
// Yield to heavy-alloc owners (today: cirrus-real boot path).
|
|
1546
|
+
// Non-blocking when no owner is active. Returns false on the
|
|
1547
|
+
// 30 s ceiling — we proceed regardless because the gate is a
|
|
1548
|
+
// best-effort reduction of peak pressure, not a correctness
|
|
1549
|
+
// dependency.
|
|
1550
|
+
await waitForLowAllocPressure();
|
|
1551
|
+
attempted++;
|
|
1552
|
+
// Build slice for THIS spec only. Released by explicit nulling
|
|
1553
|
+
// at the end of every code path through this iteration so the
|
|
1554
|
+
// bytes are GC-eligible before pool.submit's RPC layer has
|
|
1555
|
+
// finished tearing down its own references for the previous
|
|
1556
|
+
// slot. With concurrency=1 and 28 MiB caps, peak supervisor
|
|
1557
|
+
// slice memory is ~34 MiB (slice + spec metadata).
|
|
1558
|
+
//
|
|
1559
|
+
// Defensive: buildSliceForSpecifierWithCap performs sync VFS
|
|
1560
|
+
// reads that COULD throw on a corrupted inode tree, an unread-
|
|
1561
|
+
// able chunk, or any other VFS-layer surprise. Without this
|
|
1562
|
+
// try/catch, a slice-walker throw escapes runSlot, rejects the
|
|
1563
|
+
// Promise.all, drops every other in-flight slot's settled
|
|
1564
|
+
// work, and surfaces in the supervisor as an unhandled
|
|
1565
|
+
// rejection — which workerd can promote to a DO restart on a
|
|
1566
|
+
// shared isolate. Catch and treat as "skip this spec, log,
|
|
1567
|
+
// continue."
|
|
1568
|
+
let slice = null;
|
|
1569
|
+
try {
|
|
1570
|
+
if (next.synthetic && next.syntheticReferencedFiles) {
|
|
1571
|
+
// SCOPED slice: only the files the synthetic entry directly
|
|
1572
|
+
// references + their transitive relative imports + the
|
|
1573
|
+
// package's package.json. Skips the full package walk so
|
|
1574
|
+
// icon-libraries with thousands of files don't blow the
|
|
1575
|
+
// 28 MiB cap. (lucide-react@0.460 ships ~5-15 MiB across
|
|
1576
|
+
// 3940 files; full walk hits cap on Mossaic-scale projects
|
|
1577
|
+
// with 70+ imported icons.)
|
|
1578
|
+
const scoped = buildScopedSliceForSynthetic(this.vfs, nmDir, packageNameFromSpecifier(next.specifier), next.syntheticReferencedFiles);
|
|
1579
|
+
const built = { slice: scoped.entries, totalBytes: scoped.totalBytes };
|
|
1580
|
+
// Append the synthetic entry file itself (lives outside
|
|
1581
|
+
// the package dir; the scoped walker doesn't pick it up).
|
|
1582
|
+
const bytes = this.vfs.readFile(next.entryPath);
|
|
1583
|
+
const parentDir = next.entryPath.substring(0, next.entryPath.lastIndexOf('/'));
|
|
1584
|
+
built.slice.push({
|
|
1585
|
+
path: '/' + parentDir.replace(/^\/+/, ''),
|
|
1586
|
+
isDir: true,
|
|
1587
|
+
});
|
|
1588
|
+
built.slice.push({
|
|
1589
|
+
path: '/' + next.entryPath.replace(/^\/+/, ''),
|
|
1590
|
+
bytes,
|
|
1591
|
+
isDir: false,
|
|
1592
|
+
});
|
|
1593
|
+
built.totalBytes += bytes.length + next.entryPath.length;
|
|
1594
|
+
slice = built;
|
|
1595
|
+
}
|
|
1596
|
+
else {
|
|
1597
|
+
slice = buildSliceForSpecifierWithCap(this.vfs, next.specifier, nmDir, SLICE_CAP_BYTES);
|
|
1598
|
+
}
|
|
1599
|
+
}
|
|
1600
|
+
catch (e) {
|
|
1601
|
+
const msg = e?.message || String(e);
|
|
1602
|
+
safeProgress(` pre-bundle slice walk threw for ${next.specifier}: ${msg}`);
|
|
1603
|
+
errorCount++;
|
|
1604
|
+
lastError = msg;
|
|
1605
|
+
errorsByModule[next.specifier] = msg;
|
|
1606
|
+
continue;
|
|
1607
|
+
}
|
|
1608
|
+
if (!slice) {
|
|
1609
|
+
safeProgress(` skipped pre-bundle for ${next.specifier}: slice exceeded ${(SLICE_CAP_BYTES / (1024 * 1024)).toFixed(0)} MiB cap`);
|
|
1610
|
+
skippedCount++;
|
|
1611
|
+
continue;
|
|
1612
|
+
}
|
|
1613
|
+
// externalsForSpecifier is pure JS over a small list — extremely
|
|
1614
|
+
// unlikely to throw, but cheap to guard since we're hardening
|
|
1615
|
+
// this path comprehensively.
|
|
1616
|
+
let externals;
|
|
1617
|
+
try {
|
|
1618
|
+
externals = externalsForSpecifier(next.specifier);
|
|
1619
|
+
}
|
|
1620
|
+
catch (e) {
|
|
1621
|
+
const msg = e?.message || String(e);
|
|
1622
|
+
safeProgress(` pre-bundle externals threw for ${next.specifier}: ${msg}`);
|
|
1623
|
+
errorCount++;
|
|
1624
|
+
lastError = msg;
|
|
1625
|
+
errorsByModule[next.specifier] = msg;
|
|
1626
|
+
continue;
|
|
1627
|
+
}
|
|
1628
|
+
let spec = {
|
|
1629
|
+
specifier: next.specifier,
|
|
1630
|
+
entryPath: next.entryPath,
|
|
1631
|
+
externals,
|
|
1632
|
+
slice: slice.slice,
|
|
1633
|
+
bundlerVersion: BUNDLER_VERSION,
|
|
1634
|
+
};
|
|
1635
|
+
// Drop our supervisor-side reference to the slice array as soon
|
|
1636
|
+
// as it's owned by `spec`. `spec` is the only thing that needs
|
|
1637
|
+
// to keep it alive until the RPC structured-clone completes.
|
|
1638
|
+
slice = null;
|
|
1639
|
+
let result = null;
|
|
1640
|
+
try {
|
|
1641
|
+
// pool.submit is per-task (no auto slot pinning). All slots
|
|
1642
|
+
// share slot index 0 in the underlying #dispatchSlot — that's
|
|
1643
|
+
// fine for our use (we don't need stable warm slots beyond
|
|
1644
|
+
// "esbuild compiled once per slot's lifetime"; for pre-bundle
|
|
1645
|
+
// the slot HAS to compile esbuild on first call regardless).
|
|
1646
|
+
result = await pool.submit(prebundleOne, spec);
|
|
1647
|
+
}
|
|
1648
|
+
catch (e) {
|
|
1649
|
+
const msg = e?.remoteMessage || e?.message || String(e);
|
|
1650
|
+
safeProgress(` pre-bundle failed for ${next.specifier}: ${msg}`);
|
|
1651
|
+
errorCount++;
|
|
1652
|
+
lastError = msg;
|
|
1653
|
+
errorsByModule[next.specifier] = msg;
|
|
1654
|
+
}
|
|
1655
|
+
finally {
|
|
1656
|
+
// Drop the spec reference (which transitively held slice.slice)
|
|
1657
|
+
// immediately after the RPC settles, regardless of outcome.
|
|
1658
|
+
// pool.submit's facet-pool fix (timer leak) ensures the rejected
|
|
1659
|
+
// promise's `args` aren't pinned by a 60s timer; this finally
|
|
1660
|
+
// releases our supervisor-side handle the moment the await
|
|
1661
|
+
// resolves so the next iteration starts from a low-water-mark
|
|
1662
|
+
// heap. Combined defense — see commit msg.
|
|
1663
|
+
spec = null;
|
|
1664
|
+
}
|
|
1665
|
+
if (!result || !result.ok) {
|
|
1666
|
+
const why = result?.errorText || 'pool returned null';
|
|
1667
|
+
if (result) {
|
|
1668
|
+
safeProgress(` pre-bundle failed for ${next.specifier}: ${why}`);
|
|
1669
|
+
errorCount++;
|
|
1670
|
+
lastError = why;
|
|
1671
|
+
errorsByModule[next.specifier] = why;
|
|
1672
|
+
}
|
|
1673
|
+
result = null;
|
|
1674
|
+
continue;
|
|
1675
|
+
}
|
|
1676
|
+
if (result.warnings && result.warnings.length > 0) {
|
|
1677
|
+
for (const w of result.warnings) {
|
|
1678
|
+
safeProgress(` [warn] ${next.specifier}: ${w}`);
|
|
1679
|
+
}
|
|
1680
|
+
}
|
|
1681
|
+
// Stamp into pkg_esm_bundles. Cache is supervisor-side SQLite,
|
|
1682
|
+
// so the write happens here (not via writeBatch — that's VFS).
|
|
1683
|
+
// Defensive: SQL writes can throw on schema mismatch / disk-full
|
|
1684
|
+
// / closed-storage-handle. A throw here would unwind through the
|
|
1685
|
+
// loop and rejected the Promise.all wrapper. Treat the failure
|
|
1686
|
+
// as "pre-bundle succeeded but cache write failed" — okCount is
|
|
1687
|
+
// not bumped, the spec falls through to on-demand bundling on
|
|
1688
|
+
// first request, and the loop continues.
|
|
1689
|
+
try {
|
|
1690
|
+
this.cache.putEsmBundle({
|
|
1691
|
+
specifier: next.specifier,
|
|
1692
|
+
bundleHash: BUNDLER_VERSION,
|
|
1693
|
+
esmCode: result.esmCode,
|
|
1694
|
+
builtAt: Date.now(),
|
|
1695
|
+
inputHash: '',
|
|
1696
|
+
});
|
|
1697
|
+
okCount++;
|
|
1698
|
+
}
|
|
1699
|
+
catch (e) {
|
|
1700
|
+
const msg = e?.message || String(e);
|
|
1701
|
+
safeProgress(` pre-bundle cache-write failed for ${next.specifier}: ${msg}`);
|
|
1702
|
+
errorCount++;
|
|
1703
|
+
lastError = msg;
|
|
1704
|
+
errorsByModule[next.specifier] = msg;
|
|
1705
|
+
}
|
|
1706
|
+
// result.esmCode is now durably in SQLite; drop our heap copy
|
|
1707
|
+
// before the next iteration's slice walk allocates.
|
|
1708
|
+
result = null;
|
|
1709
|
+
void slotIndex;
|
|
1710
|
+
}
|
|
1711
|
+
};
|
|
1712
|
+
try {
|
|
1713
|
+
// Promise.all rejects on the first slot rejection. Every per-slot
|
|
1714
|
+
// failure mode that can throw inside runSlot is caught above
|
|
1715
|
+
// (slice walk, externals, pool.submit, putEsmBundle) — but a
|
|
1716
|
+
// future regression that adds an unguarded throw to runSlot
|
|
1717
|
+
// would bubble out here. The outer try/finally guarantees
|
|
1718
|
+
// pool.dispose() runs and the diag counters get updated for
|
|
1719
|
+
// whatever partial run completed; the catch below additionally
|
|
1720
|
+
// logs so the failure mode is visible in supervisor logs.
|
|
1721
|
+
await Promise.all(Array.from({ length: PRE_BUNDLE_CONCURRENCY }, (_, i) => runSlot(i)));
|
|
1722
|
+
}
|
|
1723
|
+
catch (e) {
|
|
1724
|
+
const msg = e?.message || String(e);
|
|
1725
|
+
safeProgress(`Pre-bundle aborted: ${msg}`);
|
|
1726
|
+
lastError = msg;
|
|
1727
|
+
}
|
|
1728
|
+
finally {
|
|
1729
|
+
// Fold pre-bundle outcomes into the diag counter singleton so
|
|
1730
|
+
// /api/_diag/memory surfaces them (commit 3 observability).
|
|
1731
|
+
// Aggregates across the DO's lifetime. Wrapped because
|
|
1732
|
+
// recordPreBundleSummary is a global-state mutator and a future
|
|
1733
|
+
// regression there could throw.
|
|
1734
|
+
try {
|
|
1735
|
+
recordPreBundleSummary({
|
|
1736
|
+
attempted,
|
|
1737
|
+
bundlesCompleted: okCount,
|
|
1738
|
+
errors: errorCount,
|
|
1739
|
+
skipped: skippedCount,
|
|
1740
|
+
lastError,
|
|
1741
|
+
errorsByModule,
|
|
1742
|
+
});
|
|
1743
|
+
}
|
|
1744
|
+
catch (e) {
|
|
1745
|
+
try {
|
|
1746
|
+
console.error('[pre-bundle] recordPreBundleSummary threw:', e?.message || e);
|
|
1747
|
+
}
|
|
1748
|
+
catch { }
|
|
1749
|
+
}
|
|
1750
|
+
// P5 (prod-bugs-2): switched both `memBefore` and `memAfter`
|
|
1751
|
+
// to the C'.1 deterministic estimator (see entry-point note
|
|
1752
|
+
// above). Both are floats in MiB; delta is post − pre.
|
|
1753
|
+
try {
|
|
1754
|
+
const memAfter = this._estimateSupervisorHeapMiB();
|
|
1755
|
+
const delta = memAfter - memBefore;
|
|
1756
|
+
safeProgress(`Pre-bundle complete: ${okCount}/${attempted} succeeded. (supervisor heap ${memAfter.toFixed(1)} MiB, Δ${delta >= 0 ? '+' : ''}${delta.toFixed(1)} MiB)`);
|
|
1757
|
+
}
|
|
1758
|
+
catch (e) {
|
|
1759
|
+
try {
|
|
1760
|
+
console.error('[pre-bundle] final-progress threw:', e?.message || e);
|
|
1761
|
+
}
|
|
1762
|
+
catch { }
|
|
1763
|
+
}
|
|
1764
|
+
try {
|
|
1765
|
+
pool.dispose();
|
|
1766
|
+
}
|
|
1767
|
+
catch { /* best-effort */ }
|
|
1768
|
+
}
|
|
1769
|
+
}
|
|
1770
|
+
/**
|
|
1771
|
+
* Scan project source files for bare import specifiers.
|
|
1772
|
+
* Returns unique bare specifiers including subpaths (e.g., both `react`
|
|
1773
|
+
* AND `react/jsx-runtime` so each can be pre-bundled separately with the
|
|
1774
|
+
* correct externals for shared-runtime isolation).
|
|
1775
|
+
*
|
|
1776
|
+
* Also injects common JSX-runtime subpaths derived from esbuild's automatic
|
|
1777
|
+
* JSX transform: if `react` is imported, we also queue `react/jsx-runtime`
|
|
1778
|
+
* and `react/jsx-dev-runtime` because the compiled JSX output imports from
|
|
1779
|
+
* them even if the source never wrote `import ... from "react/jsx-runtime"`.
|
|
1780
|
+
*/
|
|
1781
|
+
scanBareImports(projDir) {
|
|
1782
|
+
const imports = new Set();
|
|
1783
|
+
const scanExts = new Set(['.ts', '.tsx', '.jsx', '.js', '.mjs']);
|
|
1784
|
+
// Files we deliberately skip at the project root: their imports run
|
|
1785
|
+
// server-side (vite plugins, postcss/tailwind config, etc.), never in
|
|
1786
|
+
// the browser, so pre-bundling their bare specifiers as if they were
|
|
1787
|
+
// browser modules is wasted work that exposes us to esbuild fs-shim
|
|
1788
|
+
// limits (e.g. @tailwindcss/vite triggers `readdir(".")` inside
|
|
1789
|
+
// esbuild → "Cannot read directory '.': not implemented on js" → the
|
|
1790
|
+
// ensuing combined heap pressure of 30+ pending pre-bundles + dev
|
|
1791
|
+
// start has been observed crashing the supervisor on Mossaic-scale
|
|
1792
|
+
// projects). The /preview/@modules/ path never serves these
|
|
1793
|
+
// specifiers; vite's own plugin resolver loads them at server boot.
|
|
1794
|
+
const isServerOnlyTopLevel = (name) => {
|
|
1795
|
+
// vite.config.ts/js/mjs/cjs and *.config.{ts,js,mjs,cjs} at the
|
|
1796
|
+
// project root. Limited to depth 0 to avoid filtering legitimate
|
|
1797
|
+
// browser code that happens to live under e.g. src/config/foo.ts.
|
|
1798
|
+
return /^(?:vite|vitest|astro|rollup|tsup|tailwind|postcss|prettier|eslint|stylelint|rolldown)\.config\.[mc]?[jt]s$/.test(name)
|
|
1799
|
+
|| /\.config\.[mc]?[jt]s$/.test(name) && name.split('.').length === 3;
|
|
1800
|
+
};
|
|
1801
|
+
const walk = (dir, depth) => {
|
|
1802
|
+
if (depth > 5)
|
|
1803
|
+
return;
|
|
1804
|
+
try {
|
|
1805
|
+
for (const entry of this.vfs.readdir(dir)) {
|
|
1806
|
+
if (entry.name === 'node_modules' || entry.name === '.git' ||
|
|
1807
|
+
entry.name === 'dist' || entry.name === 'build')
|
|
1808
|
+
continue;
|
|
1809
|
+
const path = dir + '/' + entry.name;
|
|
1810
|
+
if (entry.type === 'directory') {
|
|
1811
|
+
walk(path, depth + 1);
|
|
1812
|
+
continue;
|
|
1813
|
+
}
|
|
1814
|
+
// Server-only config files at the project root (depth 0) are
|
|
1815
|
+
// skipped — their imports are not browser modules.
|
|
1816
|
+
if (depth === 0 && isServerOnlyTopLevel(entry.name))
|
|
1817
|
+
continue;
|
|
1818
|
+
const dotIdx = entry.name.lastIndexOf('.');
|
|
1819
|
+
if (dotIdx < 0)
|
|
1820
|
+
continue;
|
|
1821
|
+
const ext = entry.name.substring(dotIdx);
|
|
1822
|
+
if (!scanExts.has(ext))
|
|
1823
|
+
continue;
|
|
1824
|
+
try {
|
|
1825
|
+
const code = this.vfs.readFileString(path);
|
|
1826
|
+
const re = /(?:from\s+|import\s*\(?\s*)["']([^./][^"']*?)["']/g;
|
|
1827
|
+
let m;
|
|
1828
|
+
while ((m = re.exec(code)) !== null) {
|
|
1829
|
+
const spec = m[1];
|
|
1830
|
+
// Keep the full specifier (including subpaths) so each is
|
|
1831
|
+
// pre-bundled separately with the appropriate externals.
|
|
1832
|
+
// Strip any trailing query string (?v=... etc.)
|
|
1833
|
+
const clean = spec.split('?')[0];
|
|
1834
|
+
imports.add(clean);
|
|
1835
|
+
// Also add the top-level package name so its main entry is
|
|
1836
|
+
// pre-bundled even if only a subpath was imported.
|
|
1837
|
+
const pkgName = clean.startsWith('@')
|
|
1838
|
+
? clean.split('/').slice(0, 2).join('/')
|
|
1839
|
+
: clean.split('/')[0];
|
|
1840
|
+
imports.add(pkgName);
|
|
1841
|
+
}
|
|
1842
|
+
// If any .tsx/.jsx file is present and uses JSX automatic runtime
|
|
1843
|
+
// (the default), esbuild injects imports from react/jsx-runtime
|
|
1844
|
+
// even though the source never wrote them explicitly. Queue the
|
|
1845
|
+
// runtime packages so they get pre-bundled with the correct
|
|
1846
|
+
// externals.
|
|
1847
|
+
if (ext === '.tsx' || ext === '.jsx') {
|
|
1848
|
+
if (imports.has('react')) {
|
|
1849
|
+
imports.add('react/jsx-runtime');
|
|
1850
|
+
imports.add('react/jsx-dev-runtime');
|
|
1851
|
+
}
|
|
1852
|
+
}
|
|
1853
|
+
}
|
|
1854
|
+
catch { /* skip unreadable files */ }
|
|
1855
|
+
}
|
|
1856
|
+
}
|
|
1857
|
+
catch { /* skip unreadable dirs */ }
|
|
1858
|
+
};
|
|
1859
|
+
walk(projDir, 0);
|
|
1860
|
+
return [...imports];
|
|
1861
|
+
}
|
|
1862
|
+
/**
|
|
1863
|
+
* Resolve a package's entry point to a VFS path.
|
|
1864
|
+
*/
|
|
1865
|
+
/**
|
|
1866
|
+
* Resolve a specifier (possibly with subpath, e.g. "react/jsx-runtime" or
|
|
1867
|
+
* "react-dom/client") to an entry-point VFS path under node_modules.
|
|
1868
|
+
*
|
|
1869
|
+
* Algorithm:
|
|
1870
|
+
* 1. Split specifier into pkgName and subpath
|
|
1871
|
+
* (e.g. "react-dom/client" → pkgName="react-dom", subpath="client";
|
|
1872
|
+
* "@scope/pkg/sub" → pkgName="@scope/pkg", subpath="sub")
|
|
1873
|
+
* 2. Read node_modules/<pkgName>/package.json
|
|
1874
|
+
* 3. Use resolvePackageEntry(pkg, './' + subpath) to consult exports field
|
|
1875
|
+
* with ESM conditions
|
|
1876
|
+
* 4. Try extensions and index-file fallbacks
|
|
1877
|
+
*/
|
|
1878
|
+
resolvePackageEntryPath(specifier, nmDir) {
|
|
1879
|
+
// Parse out pkgName and subpath
|
|
1880
|
+
let pkgName;
|
|
1881
|
+
let subpath;
|
|
1882
|
+
if (specifier.startsWith('@')) {
|
|
1883
|
+
const parts = specifier.split('/');
|
|
1884
|
+
pkgName = parts.slice(0, 2).join('/');
|
|
1885
|
+
subpath = parts.slice(2).join('/');
|
|
1886
|
+
}
|
|
1887
|
+
else {
|
|
1888
|
+
const parts = specifier.split('/');
|
|
1889
|
+
pkgName = parts[0];
|
|
1890
|
+
subpath = parts.slice(1).join('/');
|
|
1891
|
+
}
|
|
1892
|
+
const pkgDir = nmDir + '/' + pkgName;
|
|
1893
|
+
const pkgJsonPath = pkgDir + '/package.json';
|
|
1894
|
+
if (!this.vfs.exists(pkgJsonPath))
|
|
1895
|
+
return null;
|
|
1896
|
+
try {
|
|
1897
|
+
const pkg = JSON.parse(this.vfs.readFileString(pkgJsonPath));
|
|
1898
|
+
// Use the full exports-field resolution with ESM browser conditions.
|
|
1899
|
+
// For subpath imports like "react/jsx-runtime", pass './jsx-runtime'.
|
|
1900
|
+
const subpathKey = subpath ? './' + subpath : '.';
|
|
1901
|
+
let entry = resolvePackageEntry(pkg, subpathKey);
|
|
1902
|
+
// Fallback: if no exports field or subpath not in exports, try direct
|
|
1903
|
+
// file resolution (e.g. "react/jsx-runtime" → "react/jsx-runtime.js").
|
|
1904
|
+
if (!entry && subpath) {
|
|
1905
|
+
entry = './' + subpath;
|
|
1906
|
+
}
|
|
1907
|
+
else if (!entry) {
|
|
1908
|
+
entry = 'index.js';
|
|
1909
|
+
}
|
|
1910
|
+
const entryPath = pkgDir + '/' + entry.replace(/^\.\//, '');
|
|
1911
|
+
// Try with extensions
|
|
1912
|
+
const exts = ['', '.js', '.mjs', '.ts', '.tsx', '.cjs'];
|
|
1913
|
+
for (const ext of exts) {
|
|
1914
|
+
if (this.vfs.exists(entryPath + ext) && !this.vfs.isDirectory(entryPath + ext)) {
|
|
1915
|
+
return entryPath + ext;
|
|
1916
|
+
}
|
|
1917
|
+
}
|
|
1918
|
+
// Try index files
|
|
1919
|
+
if (this.vfs.isDirectory(entryPath)) {
|
|
1920
|
+
for (const idx of ['index.js', 'index.mjs', 'index.ts', 'index.cjs']) {
|
|
1921
|
+
if (this.vfs.exists(entryPath + '/' + idx))
|
|
1922
|
+
return entryPath + '/' + idx;
|
|
1923
|
+
}
|
|
1924
|
+
}
|
|
1925
|
+
}
|
|
1926
|
+
catch { /* skip */ }
|
|
1927
|
+
return null;
|
|
1928
|
+
}
|
|
1929
|
+
/**
|
|
1930
|
+
* P5 (prod-bugs-2) — deterministic supervisor-heap estimate in MiB.
|
|
1931
|
+
*
|
|
1932
|
+
* Routes through observability/heap-estimate.ts which sums KNOWN
|
|
1933
|
+
* supervisor heap allocation sources from runtime counters that
|
|
1934
|
+
* ARE accurate inside a DO context (DiagCounters singleton +
|
|
1935
|
+
* SqliteVFS.getStats()). This replaces a previous use of
|
|
1936
|
+
* process.memoryUsage() which returned 0 for every field inside a
|
|
1937
|
+
* Durable Object class context, making the printed
|
|
1938
|
+
* "supervisor heap N MiB" lines actively misleading (they always
|
|
1939
|
+
* read 0.0 MiB regardless of actual heap state).
|
|
1940
|
+
*
|
|
1941
|
+
* The estimator is INTENTIONALLY conservative — peak-or-current
|
|
1942
|
+
* components, sum may overestimate — but it never under-reports.
|
|
1943
|
+
* Returns 0 if the estimator throws (defensive: a counter
|
|
1944
|
+
* regression must not block install completion).
|
|
1945
|
+
*
|
|
1946
|
+
* Same call shape used by /api/_diag/memory in
|
|
1947
|
+
* src/session/routes.ts:247, so the value printed in the
|
|
1948
|
+
* pre-bundle banner is comparable to the value the diag endpoint
|
|
1949
|
+
* reports for the same isolate.
|
|
1950
|
+
*/
|
|
1951
|
+
_estimateSupervisorHeapMiB() {
|
|
1952
|
+
try {
|
|
1953
|
+
const counters = readDiagCounters();
|
|
1954
|
+
const vfsStats = this.vfs.getStats();
|
|
1955
|
+
const cacheStats = vfsStats.cache ?? {};
|
|
1956
|
+
const heap = estimateSupervisorHeap(counters, {
|
|
1957
|
+
cacheHotBytes: cacheStats.hotBytes ?? 0,
|
|
1958
|
+
// Steady-state in-flight write bytes are 0 by the time we
|
|
1959
|
+
// reach the pre-bundle phase boundary (writeBatch has
|
|
1960
|
+
// already flushed before bundle dispatches). Same value
|
|
1961
|
+
// routes.ts uses for its read.
|
|
1962
|
+
inFlightWriteBytes: 0,
|
|
1963
|
+
});
|
|
1964
|
+
return heap.estimatedBytes / (1024 * 1024);
|
|
1965
|
+
}
|
|
1966
|
+
catch {
|
|
1967
|
+
return 0;
|
|
1968
|
+
}
|
|
1969
|
+
}
|
|
1970
|
+
}
|
|
1971
|
+
// ── Helpers ─────────────────────────────────────────────────────────────
|
|
1972
|
+
// P5 (prod-bugs-2): readSupervisorHeap() removed. It called
|
|
1973
|
+
// process.memoryUsage() which returns 0 for every field inside a
|
|
1974
|
+
// Durable Object class context (only dynamic-worker isolates under
|
|
1975
|
+
// nodejs_compat get the real implementation — see
|
|
1976
|
+
// src/observability/diag-counters.ts:4). The deterministic
|
|
1977
|
+
// supervisor-heap estimator is the C'.1 replacement; see
|
|
1978
|
+
// NpmInstaller._estimateSupervisorHeapMiB.
|
|
1979
|
+
function parentOf(path) {
|
|
1980
|
+
return path.includes('/') ? path.substring(0, path.lastIndexOf('/')) : '';
|
|
1981
|
+
}
|
|
1982
|
+
function safeJsonParse(json, fallback) {
|
|
1983
|
+
try {
|
|
1984
|
+
return JSON.parse(json);
|
|
1985
|
+
}
|
|
1986
|
+
catch {
|
|
1987
|
+
return fallback;
|
|
1988
|
+
}
|
|
1989
|
+
}
|
|
1990
|
+
/**
|
|
1991
|
+
* Merge per-shard `facetCounters` arrays from a fanout install-batch.
|
|
1992
|
+
* Each shard's counters describe the work done by its peer DO's
|
|
1993
|
+
* facet; merging gives the supervisor a single aggregate to fold
|
|
1994
|
+
* into recordInstallFacetCounters / recordR2RaceCounters as if the
|
|
1995
|
+
* batch had run as a single facet (the pre-fanout posture).
|
|
1996
|
+
*
|
|
1997
|
+
* Aggregation rules:
|
|
1998
|
+
* - tarballsCompleted, cumulativeBytesDecoded, race wins/losses:
|
|
1999
|
+
* SUM (additive across shards).
|
|
2000
|
+
* - peakInFlight: MAX (each shard observed its own peak; the
|
|
2001
|
+
* overall peak is the max of those, since shards run in parallel
|
|
2002
|
+
* across separate isolates and the supervisor never sees their
|
|
2003
|
+
* in-flight sum at one moment).
|
|
2004
|
+
*/
|
|
2005
|
+
function mergeFacetCounters(perShard) {
|
|
2006
|
+
if (perShard.length === 0) {
|
|
2007
|
+
return {
|
|
2008
|
+
tarballsCompleted: 0,
|
|
2009
|
+
cumulativeBytesDecoded: 0,
|
|
2010
|
+
peakInFlight: 0,
|
|
2011
|
+
pipelinedTarballRaceWins: 0,
|
|
2012
|
+
pipelinedTarballRaceLosses: 0,
|
|
2013
|
+
};
|
|
2014
|
+
}
|
|
2015
|
+
return {
|
|
2016
|
+
tarballsCompleted: perShard.reduce((s, c) => s + (c.tarballsCompleted || 0), 0),
|
|
2017
|
+
cumulativeBytesDecoded: perShard.reduce((s, c) => s + (c.cumulativeBytesDecoded || 0), 0),
|
|
2018
|
+
peakInFlight: perShard.reduce((m, c) => Math.max(m, c.peakInFlight || 0), 0),
|
|
2019
|
+
pipelinedTarballRaceWins: perShard.reduce((s, c) => s + (c.pipelinedTarballRaceWins || 0), 0),
|
|
2020
|
+
pipelinedTarballRaceLosses: perShard.reduce((s, c) => s + (c.pipelinedTarballRaceLosses || 0), 0),
|
|
2021
|
+
};
|
|
2022
|
+
}
|