@nimbus-sh/worker 0.9.0 → 0.11.0

package/scripts/bundle-git.mjs

@@ -1,30 +1,37 @@
 #!/usr/bin/env node
 /**
- * bundle-git.mjs — Pre-bundle isomorphic-git for the git-network-facet.
+ * bundle-git.mjs — Build the patched git module for hosts and network facets.
  *
  * WHY this exists:
  *   The git-network-facet runs inside a dynamic worker loaded via
  *   env.LOADER.load(). Dynamic workers have NO access to node_modules —
  *   they only see modules explicitly passed in the `modules` object.
  *
- *   The main Worker's bundler (wrangler) inlines isomorphic-git into the
- *   supervisor, but that bundle is unreachable from the facet isolate.
+ *   Hosts import the executable module shipped in vendor/. Consumers never
+ *   depend on Nimbus' repository postinstall having patched their dependencies.
  *
  *   This script bundles isomorphic-git + isomorphic-git/http/web into ONE
- *   ESM string that we can pass as a module to LOADER.load(). The facet
- *   imports it via `import('./isomorphic-git.js')`.
+ *   ESM module. The same bytes are staged as the asset the facet's
+ *   LOADER.load() receives as `git-bundle.js`.
  *
  * Output:
- *   src/git-bundle.generated.ts — exports GIT_BUNDLE_CODE: string
+ *   vendor/git.generated.mjs + self-contained declarations
+ *   public/_assets/runtime/git-<buildId>.js — byte-identical staged copy
+ *   src/git-bundle.generated.ts — GIT_BUNDLE_ENTRY / _BUILD_ID / _SHA256 pins
  *
  * Runs as a postinstall step AND on every `bun run dev` / `bun run deploy`
  * via the "bundle:git" npm script.
  */
 
 import { build } from 'esbuild';
-import { writeFileSync, mkdirSync, existsSync } from 'node:fs';
+import { createHash } from 'node:crypto';
+import { writeFileSync, readFileSync, readdirSync, mkdirSync, existsSync, unlinkSync } from 'node:fs';
 import { dirname, join } from 'node:path';
 import { fileURLToPath } from 'node:url';
+import ts from 'typescript';
+
+import { assertCfGitPatched } from './cf-git-patch.mjs';
+import { resolvePackageDir } from './resolve-package-dir.mjs';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const root = join(__dirname, '..');
@@ -37,7 +44,43 @@ const ENTRY_CONTENTS = [
 
 const ENTRY_FILE = join(root, '.git-bundle-entry.js');
 
+// Upstream declarations are copied mechanically; only the tracked patch's
+// public signature changes live here. No consumer dependency on the build input.
+const GIT_TYPES = `import type * as Upstream from './git-upstream.generated.js';
+type Add = Parameters<typeof Upstream.add>[0];
+type Remove = Parameters<typeof Upstream.remove>[0];
+type Commit = Parameters<typeof Upstream.commit>[0];
+type Matrix = Parameters<typeof Upstream.statusMatrix>[0];
+type Checkout = Parameters<typeof Upstream.checkout>[0];
+export declare const git: Omit<typeof Upstream, 'add' | 'remove' | 'commit' | 'statusMatrix'> & {
+  add(options: Omit<Add, 'filepath'> & { filepath: string | string[] }): ReturnType<typeof Upstream.add>;
+  remove(options: Omit<Remove, 'filepath'> & { filepath: string | string[] }): ReturnType<typeof Upstream.remove>;
+  commit(options: Commit & { rawMessage?: boolean }): ReturnType<typeof Upstream.commit>;
+  statusMatrix(options: Matrix & { deferRefresh?: boolean }): ReturnType<typeof Upstream.statusMatrix>;
+  stage(options: Pick<Add, 'fs' | 'dir' | 'gitdir' | 'cache' | 'force' | 'parallel'> & {
+    add?: string[]; remove?: string[];
+  }): Promise<void>;
+  checkoutFreshChunk(options: Pick<Checkout, 'fs' | 'dir' | 'gitdir' | 'ref' | 'cache' | 'onProgress'> & {
+    cursor?: object | null; maxEntries: number; maxDecodedBytes: number; maxWallMs: number;
+    deferIndexFragmentCleanup?: boolean;
+  }): Promise<{ nextCursor: object | null; files: number; decodedBytes: number; treeEntriesVisited: number; indexEntries: number }>;
+};
+export * as gitHttp from './git-http.generated.js';
+`;
+
+function declaration(source, destination) {
+  const text = readFileSync(source, 'utf8');
+  if (ts.preProcessFile(text).importedFiles.length) {
+    throw new Error('Git declarations acquired imports; bundle their types before publishing: ' + source);
+  }
+  writeFileSync(destination, '// Generated by scripts/bundle-git.mjs from the pinned cf-git declarations.\n' + text);
+}
+
 async function main() {
+  // A copy patched by another revision of the tracked patch would bundle
+  // without its fixes, and nothing downstream would notice: refuse instead.
+  const gitDir = resolvePackageDir('isomorphic-git', { start: root });
+  assertCfGitPatched(join(root, '..', '..'), [gitDir]);
   try {
     writeFileSync(ENTRY_FILE, ENTRY_CONTENTS);
 
@@ -59,6 +102,12 @@ async function main() {
         'node:*',
         'cloudflare:workers',
       ],
+      // The module ships inside the supervisor Worker, which has a size budget
+      // (tests/behavioral/assets-fetch/new/worker-bundle-size.mjs), and as a
+      // staged asset. Whitespace and syntax minification take ~200 KB off it;
+      // identifiers are kept, so stack traces still name cf-git's functions.
+      minifyWhitespace: true,
+      minifySyntax: true,
       write: false,
       logLevel: 'warning',
     });
@@ -69,43 +118,55 @@ async function main() {
 
     const code = result.outputFiles[0].text;
 
-    // Emit as a TS module that exports the bundle as a string constant.
-    // Using String.raw-equivalent: JSON-encode to preserve backticks/$/etc.
-    const outPath = join(root, 'src', 'git-bundle.generated.ts');
+    const vendor = join(root, 'vendor');
+    mkdirSync(vendor, { recursive: true });
+    writeFileSync(join(vendor, 'git.generated.mjs'), code);
+    writeFileSync(join(vendor, 'git.LICENSE'), readFileSync(join(gitDir, 'LICENSE.md')));
+    declaration(join(gitDir, 'index.d.ts'), join(vendor, 'git-upstream.generated.d.ts'));
+    declaration(join(gitDir, 'http', 'web', 'index.d.ts'), join(vendor, 'git-http.generated.d.ts'));
+    writeFileSync(join(vendor, 'git.generated.d.mts'), '// Generated by scripts/bundle-git.mjs.\n' + GIT_TYPES);
 
-    // We JSON.stringify the code to produce a valid JS string literal.
-    // Then wrap in a TS const export. The supervisor imports this at build.
-    const encoded = JSON.stringify(code);
+    // The facet's copy is a staged asset, not a string in the Worker bundle:
+    // the Worker already carries the executable module above, and a second
+    // inlined copy cost ~310 KB of the bundle's size gate
+    // (tests/behavioral/assets-fetch/new/worker-bundle-size.mjs). Same
+    // mechanism as scripts/bundle-node-shims.mjs; src/runtime/git-bundle-artifact.ts
+    // fetches it sha-verified.
+    const sha256 = createHash('sha256').update(code, 'utf8').digest('hex');
+    const buildId = sha256.slice(0, 16);
+    const assetName = `git-${buildId}.js`;
+    const assetDir = join(root, 'public', '_assets', 'runtime');
+    mkdirSync(assetDir, { recursive: true });
+    // Exactly one staged copy: the entry constant pins which one a deploy serves.
+    for (const file of readdirSync(assetDir)) {
+      if (/^git-[0-9a-f]{16}\.js$/.test(file) && file !== assetName) unlinkSync(join(assetDir, file));
+    }
+    writeFileSync(join(assetDir, assetName), code);
 
+    const outPath = join(root, 'src', 'git-bundle.generated.ts');
     const tsWrapper = [
       '/**',
-      ' * git-bundle.generated.ts — AUTO-GENERATED. DO NOT EDIT.',
-      ' *',
-      ' * Produced by scripts/bundle-git.mjs from:',
-      " *   - isomorphic-git (github:AshishKumar4/cf-git)",
-      " *   - isomorphic-git/http/web",
+      ' * git-bundle.generated.ts — AUTO-GENERATED by scripts/bundle-git.mjs. DO NOT EDIT.',
       ' *',
-      ' * Consumed by git-network-facet.ts: passed to LOADER.load()\'s',
-      ' * `modules` record so the facet can `import` isomorphic-git.',
-      ' *',
-      ` * Size: ${(code.length / 1024).toFixed(1)} KiB`,
+      ' * Pins the staged copy of vendor/git.generated.mjs (isomorphic-git from',
+      ' * github:AshishKumar4/cf-git + isomorphic-git/http/web) that the git network',
+      " * facet imports as `git-bundle.js`. GIT_BUNDLE_SHA256 is verified at fetch time.",
       ' */',
       '',
-      `export const GIT_BUNDLE_CODE: string = ${encoded};`,
-      '',
-      `export const GIT_BUNDLE_SIZE = ${code.length};`,
+      `export const GIT_BUNDLE_ENTRY: string = ${JSON.stringify(`/_assets/runtime/${assetName}`)};`,
+      `export const GIT_BUNDLE_BUILD_ID: string = ${JSON.stringify(buildId)};`,
+      `export const GIT_BUNDLE_SHA256: string = ${JSON.stringify(sha256)};`,
       '',
     ].join('\n');
 
     mkdirSync(dirname(outPath), { recursive: true });
     writeFileSync(outPath, tsWrapper);
 
-    console.log(`[bundle-git] wrote ${outPath} (${(code.length / 1024).toFixed(1)} KiB bundled)`);
+    console.log(`[bundle-git] staged ${assetName} (${(code.length / 1024).toFixed(1)} KiB) and wrote ${outPath}`);
   } finally {
     // Clean up the temp entry file
     try {
       if (existsSync(ENTRY_FILE)) {
-        const { unlinkSync } = await import('node:fs');
         unlinkSync(ENTRY_FILE);
       }
     } catch {}

package/scripts/bundle-node-shims.mjs

@@ -1,31 +1,37 @@
 #!/usr/bin/env node
 /**
- * bundle-node-shims.mjs — stage the node-compat shim source as a static asset.
+ * bundle-node-shims.mjs — stage the node-compat layer's sources as static assets.
  *
  * Why this exists
  * ───────────────
- * `generateShimsCode()` (src/runtime/node-shims.ts) produces the ~230 KiB
- * node-compat shim source that every node facet's generated worker text
- * interpolates (`${SHIMS}` in facets/manager.ts entrypoint codegen and the
- * opencode facet runner). Keeping that template inside the worker bundle
- * pushed the main bundle over the ≤6 MiB gate
- * (tests/behavioral/assets-fetch/new/worker-bundle-size.mjs), whose intended
- * fix is exactly this promote: large facet-runner source strings belong in
- * the static-assets layer, not the supervisor bundle.
+ * Every node facet's generated worker text splices three sources that only
+ * ever run inside the facet:
+ *   - `generateShimsCode()` (src/runtime/node-shims.ts), the ~230 KiB
+ *     node-compat shims;
+ *   - `VFS_WRITE_LEDGER_SOURCE` (@nimbus-sh/core _shared/vfs-write-ledger.ts),
+ *     the write ledger the shims' filesystem writes go through;
+ *   - `FACET_RESIDENT_STORE_SOURCE` (src/vfs/facet-resident-store.ts), a
+ *     resident facet's SQLite-backed resident set the shims read from.
+ * Keeping them inside the worker bundle pushed the main bundle over its size
+ * gate (tests/behavioral/assets-fetch/new/worker-bundle-size.mjs), whose
+ * intended fix is exactly this promote: large facet-runner source strings
+ * belong in the static-assets layer, not the supervisor bundle. Nothing the
+ * Worker bundle imports reaches the three modules any more.
  *
- * This script evaluates the COMMITTED dist's generateShimsCode() — dist is
- * the deployable truth (package "main" points at dist; apps bundle from it),
- * so the staged asset always matches what a deploy would have shipped inline.
- * Drift between src and the staged asset is caught by
- * tests/unit/node-shims-artifact-parity.mjs (asserts asset bytes equal the
- * CURRENT src generateShimsCode() output).
+ * This script evaluates the COMMITTED dist — dist is the deployable truth
+ * (package "main" points at dist; apps bundle from it), so each staged asset
+ * matches what a deploy would have shipped inline. Drift between src and the
+ * staged assets is caught by tests/unit/node-shims-artifact-parity.mjs
+ * (asserts each asset's bytes equal the CURRENT src output).
  *
  * Output:
- *   public/_assets/runtime/node-shims-<buildId>.js   (the shim source)
- *   src/node-shims-artifact.generated.ts
- *     export const NODE_SHIMS_ENTRY: string;     // asset path
- *     export const NODE_SHIMS_BUILD_ID: string;  // content-hash prefix
- *     export const NODE_SHIMS_SHA256: string;    // full digest
+ *   public/_assets/runtime/node-shims-<buildId>.js
+ *   public/_assets/runtime/vfs-write-ledger-<buildId>.js
+ *   public/_assets/runtime/resident-store-<buildId>.js
+ *   src/node-shims-artifact.generated.ts, per source:
+ *     export const <NAME>_ENTRY: string;     // asset path
+ *     export const <NAME>_BUILD_ID: string;  // content-hash prefix
+ *     export const <NAME>_SHA256: string;    // full digest
  *
  * Fetched at runtime by src/runtime/node-shims-artifact.ts (L2 + ASSETS,
  * sha-verified, memoized per isolate).
@@ -42,9 +48,9 @@ import { fileURLToPath } from 'node:url';
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 const ROOT = path.resolve(__dirname, '..');
 
-const { generateShimsCode } = await import(
-  path.join(ROOT, 'dist/runtime/node-shims.js')
-);
+const { generateShimsCode } = await import(path.join(ROOT, 'dist/runtime/node-shims.js'));
+const { FACET_RESIDENT_STORE_SOURCE } = await import(path.join(ROOT, 'dist/vfs/facet-resident-store.js'));
+const { VFS_WRITE_LEDGER_SOURCE } = await import('@nimbus-sh/core/_shared/vfs-write-ledger.js');
 
 const shims = generateShimsCode();
 if (typeof shims !== 'string' || shims.length < 100_000) {
@@ -55,38 +61,58 @@ if (typeof shims !== 'string' || shims.length < 100_000) {
   );
 }
 
-const sha256 = createHash('sha256').update(shims, 'utf8').digest('hex');
-const buildId = sha256.slice(0, 16);
-const assetDir = path.join(ROOT, 'public/_assets/runtime');
-const assetName = `node-shims-${buildId}.js`;
-const entry = `/_assets/runtime/${assetName}`;
+const SOURCES = [
+  { name: 'NODE_SHIMS', family: 'node-shims', source: shims, from: 'dist/runtime/node-shims.js generateShimsCode()' },
+  {
+    name: 'VFS_WRITE_LEDGER',
+    family: 'vfs-write-ledger',
+    source: VFS_WRITE_LEDGER_SOURCE,
+    from: '@nimbus-sh/core dist/_shared/vfs-write-ledger.js VFS_WRITE_LEDGER_SOURCE',
+  },
+  {
+    name: 'RESIDENT_STORE',
+    family: 'resident-store',
+    source: FACET_RESIDENT_STORE_SOURCE,
+    from: 'dist/vfs/facet-resident-store.js FACET_RESIDENT_STORE_SOURCE',
+  },
+];
 
+const assetDir = path.join(ROOT, 'public/_assets/runtime');
 await fs.mkdir(assetDir, { recursive: true });
-// Remove stale same-family assets so the directory carries exactly one
-// node-shims blob (the entry constant pins which one a deploy serves).
-for (const f of await fs.readdir(assetDir)) {
-  if (f.startsWith('node-shims-') && f !== assetName) {
-    await fs.unlink(path.join(assetDir, f));
+const existing = await fs.readdir(assetDir);
+const pins = [];
+for (const { name, family, source, from } of SOURCES) {
+  if (typeof source !== 'string' || source.length === 0) {
+    throw new Error(`[bundle-node-shims] ${from} is ${typeof source === 'string' ? 'empty' : typeof source}, not the source to stage`);
+  }
+  const sha256 = createHash('sha256').update(source, 'utf8').digest('hex');
+  const buildId = sha256.slice(0, 16);
+  const assetName = `${family}-${buildId}.js`;
+  // Remove stale same-family assets so the directory carries exactly one blob
+  // per family (the entry constant pins which one a deploy serves).
+  for (const f of existing) {
+    if (f.startsWith(`${family}-`) && f !== assetName) await fs.unlink(path.join(assetDir, f));
   }
+  await fs.writeFile(path.join(assetDir, assetName), source, 'utf8');
+  pins.push(
+    `/** ${from} */`,
+    `export const ${name}_ENTRY: string = ${JSON.stringify(`/_assets/runtime/${assetName}`)};`,
+    `export const ${name}_BUILD_ID: string = ${JSON.stringify(buildId)};`,
+    `export const ${name}_SHA256: string = ${JSON.stringify(sha256)};`,
+    '',
+  );
+  console.log(`[bundle-node-shims] staged ${assetName} (${(source.length / 1024).toFixed(1)} KiB, sha ${buildId}…)`);
 }
-await fs.writeFile(path.join(assetDir, assetName), shims, 'utf8');
 
 const generated = `/**
  * node-shims-artifact.generated.ts — AUTO-GENERATED by
  * scripts/bundle-node-shims.mjs. DO NOT EDIT.
  *
- * Pins the staged node-compat shim asset (the generateShimsCode() output
- * promoted out of the worker bundle). NODE_SHIMS_BUILD_ID is a content-hash
- * prefix so cache layers never serve stale bytes after a rebuild;
- * NODE_SHIMS_SHA256 is the full digest verified at fetch time.
+ * Pins the staged sources of the node-compat layer, promoted out of the worker
+ * bundle: the shims, the VFS write ledger and the resident store. Each
+ * <NAME>_BUILD_ID is a content-hash prefix so cache layers never serve stale
+ * bytes after a rebuild; <NAME>_SHA256 is the full digest verified at fetch time.
  */
 
-export const NODE_SHIMS_ENTRY: string = ${JSON.stringify(entry)};
-export const NODE_SHIMS_BUILD_ID: string = ${JSON.stringify(buildId)};
-export const NODE_SHIMS_SHA256: string = ${JSON.stringify(sha256)};
-`;
+${pins.join('\n')}`;
 await fs.writeFile(path.join(ROOT, 'src/node-shims-artifact.generated.ts'), generated, 'utf8');
-
-console.log(
-  `[bundle-node-shims] staged ${assetName} (${(shims.length / 1024).toFixed(1)} KiB, sha ${buildId}…)`,
-);