@exodus/stasis 1.0.0-beta.2 → 1.0.0-beta.5

package/src/babel.js

@@ -0,0 +1,131 @@
+// Per-file Babel pre-transforms for `stasis build`, serving two flags:
+//
+// --platform=hermes downlevels every code file to the dialect Hermes (React Native's
+// engine) executes -- no classes, class fields, let/const, arrows, async generators or
+// for-await. Two steps per file, then a constrained final bundle:
+//   1. esbuild.transform lowers class fields / private members / static blocks / `using`
+//      into plain class syntax (stripping TS types and compiling JSX on the way), the
+//      shape Babel's class transform can consume.
+//   2. A Babel worker pool applies transform-block-scoping + transform-classes -- the two
+//      lowerings esbuild refuses to do itself.
+//   3. The final esbuild bundle runs with HERMES_UNSUPPORTED (arrow/async-generator/
+//      for-await lowered by esbuild; class and const-and-let declared unsupported both so
+//      esbuild's own helpers stay Hermes-safe and as a fail-closed check that none
+//      survived Babel).
+//
+// --babel instead runs each code file through the project's own babel.config.* verbatim --
+// no esbuild pre-transform, Babel sees the original source exactly as the project's own
+// toolchain (e.g. Metro) would hand it over.
+//
+// Babel is the PROJECT's dev dependency, not stasis's: @babel/core (and, for hermes, the
+// two transform plugins) resolve from the project directory, so the project pins its own
+// Babel toolchain and config semantics.
+import { existsSync } from 'node:fs'
+import { createRequire } from 'node:module'
+import { join } from 'node:path'
+
+const require = createRequire(import.meta.url)
+
+// esbuild `supported` map for the FINAL hermes bundle (spread into esbuild.build options).
+export const HERMES_UNSUPPORTED = {
+  arrow: false,
+  class: false, // already lowered by Babel; unsupporting it doubles as a safeguard check
+  'async-generator': false,
+  'const-and-let': false, // ditto, and keeps esbuild-emitted helpers off let/const
+  'for-await': false,
+}
+
+// Class-syntax extensions the hermes pre-transform lowers into plain classes for Babel,
+// which handles none of them without extra plugins (and Hermes supports none of them either).
+const PRE_TRANSFORM_UNSUPPORTED = {
+  'class-field': false,
+  'class-private-accessor': false,
+  'class-private-brand-check': false,
+  'class-private-field': false,
+  'class-private-method': false,
+  'class-private-static-accessor': false,
+  'class-private-static-field': false,
+  'class-private-static-method': false,
+  'class-static-blocks': false,
+  'class-static-field': false,
+  'import-attributes': false,
+  using: false,
+}
+
+const HERMES_BABEL_PLUGINS = ['@babel/plugin-transform-block-scoping', '@babel/plugin-transform-classes']
+
+// Only real JS-family sources are transformed; anything else the plugin serves (json, or a
+// --loader override like .svg:text) passes through untouched.
+const TRANSFORMABLE_LOADERS = new Set(['js', 'ts', 'jsx', 'tsx'])
+
+// Resolved up front on the main thread so a missing install fails with instructions, not a
+// worker crash mid-build. Returns absolute paths: the worker loads @babel/core by path, and
+// plugin paths bypass Babel's own name resolution (which would look in the wrong dir).
+function resolveFromProject(projectDir, deps, flag) {
+  const projectRequire = createRequire(join(projectDir, 'noop.js'))
+  try {
+    return deps.map((dep) => projectRequire.resolve(dep))
+  } catch (cause) {
+    throw new Error(
+      `stasis build: ${flag} needs ${deps.join(', ')} installed in the project ` +
+      `(e.g. \`npm i -D ${deps.join(' ')}\`); resolution failed from ${projectDir}`,
+      { cause }
+    )
+  }
+}
+
+// The hermes per-file pre-transform, in StasisEsbuild's load-mode `transform` hook shape:
+// (source, { path, loader }) => { contents, loader } | undefined.
+export function createHermesTransform({ projectDir, esbuild }) {
+  const [babelPath, ...plugins] = resolveFromProject(
+    projectDir, ['@babel/core', ...HERMES_BABEL_PLUGINS], '--platform=hermes'
+  )
+  const { transformAsync } = require('./babel-worker.cjs')
+  return async (source, { path, loader }) => {
+    if (!TRANSFORMABLE_LOADERS.has(loader)) return undefined
+    const { code, warnings } = await esbuild.transform(source, {
+      sourcemap: 'inline',
+      sourcefile: path,
+      loader,
+      supported: PRE_TRANSFORM_UNSUPPORTED,
+    })
+    if (warnings.length > 0) {
+      const formatted = await esbuild.formatMessages(warnings, { kind: 'warning', color: false })
+      for (const line of formatted) process.stderr.write(line)
+    }
+    const result = await transformAsync(babelPath, code, {
+      compact: false,
+      babelrc: false,
+      configFile: false,
+      plugins,
+    })
+    // TS is stripped and JSX compiled by the pre-transform, so the result is always plain js.
+    return { contents: result.code, loader: 'js' }
+  }
+}
+
+// The --babel per-file transform (same hook shape): the project's own babel.config.* over
+// the original source. Its output must be esbuild-parseable js -- a config that leaves TS/JSX
+// in place fails the build in Babel or esbuild, which is the project's config to fix.
+export function createBabelConfigTransform({ projectDir }) {
+  const configFile = ['babel.config.js', 'babel.config.cjs', 'babel.config.mjs', 'babel.config.json']
+    .map((name) => join(projectDir, name))
+    .find((file) => existsSync(file))
+  if (!configFile) {
+    throw new Error(`stasis build: --babel found no babel.config.(js|cjs|mjs|json) in ${projectDir}`)
+  }
+  const [babelPath] = resolveFromProject(projectDir, ['@babel/core'], '--babel')
+  const { transformAsync } = require('./babel-worker.cjs')
+  return async (source, { path, loader }) => {
+    if (!TRANSFORMABLE_LOADERS.has(loader)) return undefined
+    // filename makes per-file config (overrides, .babelrc lookup) behave as in a real run.
+    const result = await transformAsync(babelPath, source, {
+      cwd: projectDir,
+      root: projectDir,
+      configFile,
+      filename: path,
+      compact: false,
+    })
+    return { contents: result.code, loader: 'js' }
+  }
+}

package/src/bundle-from-lockfile.js

@@ -4,17 +4,12 @@ import { resolve } from 'node:path'
 
 import { Bundle } from '@exodus/stasis-core/bundle'
 import { sha512integrity } from '@exodus/stasis-core/state-util'
-import { moduleFileKey } from '@exodus/stasis-core/util'
+import { moduleFileKey, narrowExecutable } from '@exodus/stasis-core/util'
 
-// Reconstruct an in-memory Bundle from a lockfile by reading every file it attests
-// from disk and verifying each against the lockfile's SRI digest -- the exact inverse
-// of `extract`'s lockfileFromBundle. A changed or tampered file fails closed here (the
-// digest won't match), so building from a lockfile is as trustworthy as a
-// `stasis run --lock=frozen` replay: the build only ever sees bytes the lockfile
-// vouches for. `root` is the project root the lockfile's paths are relative to.
-//
-// Directory captures (`stasis run --fs` readdir listings, format 'directory') are
-// skipped: they key a listing at a directory's own path, not a file esbuild could load.
+// Reconstruct an in-memory Bundle from a lockfile, reading every attested file from disk and
+// verifying each against the lockfile's SRI digest -- fail closed on mismatch, so this is as
+// trustworthy as a `stasis run --lock=frozen` replay. `root` is what the lockfile's paths are
+// relative to. Directory captures (format 'directory') are skipped -- not files esbuild can load.
 export function bundleFromLockfile(lockfile, { root }) {
   if (!lockfile.imports) throw new Error('stasis: lockfile has no recorded import graph')
   const formats = lockfile.formats ?? new Map()
@@ -32,18 +27,14 @@ export function bundleFromLockfile(lockfile, { root }) {
       } catch (cause) {
         throw new Error(`stasis: file attested by the lockfile is missing on disk: ${file}`, { cause })
       }
-      // The lockfile digests raw on-disk bytes; verify against them before trusting
-      // the file. (Code and 'resource' files hash identically whether viewed as a
-      // string or bytes, since Node hashes a string as its UTF-8 bytes.)
+      // The lockfile digests raw on-disk bytes; verify before trusting the file.
       const actual = sha512integrity(bytes)
       if (actual !== integrity) {
         throw new Error(`stasis: ${file} does not match the lockfile (expected ${integrity}, got ${actual})`)
       }
-      // Store content the way a Bundle does: 'resource:base64' as base64 text,
-      // everything else (code, 'resource') as a raw UTF-8 string. A non-base64 entry is
-      // digested AND served as UTF-8 text, so a non-UTF-8 byte would transcode lossily
-      // (U+FFFD), diverging the served content from the verified bytes -- mirror the
-      // capture side's isUtf8 guard and fail closed instead.
+      // Store content as a Bundle does: 'resource:base64' as base64, everything else as a UTF-8 string.
+      // A non-UTF-8 byte in a non-base64 entry would transcode lossily (U+FFFD), diverging served
+      // content from the verified bytes -- mirror the capture side's isUtf8 guard and fail closed.
       if (format === 'resource:base64') {
         out[rel] = bytes.toString('base64')
       } else {
@@ -61,5 +52,8 @@ export function bundleFromLockfile(lockfile, { root }) {
     modules,
     imports: lockfile.imports,
     formats,
+    // Narrowed to the files this bundle ended up carrying (a skipped `directory` capture drops out),
+    // keeping Bundle.parse's rules true for a hand-built lockfile too.
+    executable: narrowExecutable(lockfile.executable, { modules, formats, scope: lockfile.config.scope }),
   })
 }

package/src/bundle-reader.js

@@ -0,0 +1,77 @@
+import { createReadStream } from 'node:fs'
+import { pipeline } from 'node:stream/promises'
+import { createBrotliDecompress } from 'node:zlib'
+
+import { Bundle } from '@exodus/stasis-core/bundle'
+import { assert } from '@exodus/stasis-core/util'
+
+import { JsonStreamParser } from './json-stream.js'
+
+// Rejects as soon as `signal` aborts, without waiting for `promise` (onFile gets the signal to stop itself).
+const untilAborted = (promise, signal) => (signal === undefined ? promise : new Promise((resolve, reject) => {
+  const abort = () => reject(signal.reason)
+  if (signal.aborted) abort()
+  else signal.addEventListener('abort', abort, { once: true })
+  Promise.resolve(promise).then(resolve, reject).finally(() => signal.removeEventListener('abort', abort))
+}))
+
+// Left in place of each streamed file; fromJSON's contents-free mode requires one per file.
+const STREAMED = Symbol('streamed')
+
+// Bundle.parse(brotliDecompressSync(source)), without holding the decompressed bytes or the JSON text whole.
+// onFile gets each file in stream order, wherever the bundle puts its files, and before the bundle is
+// validated: what it receives (format included) is provisional until readBundle resolves. See doc/file-formats.md.
+export async function readBundle(source, { onFile, signal } = {}) {
+  assert(onFile === undefined || typeof onFile === 'function', 'readBundle: onFile must be a function')
+  let input = source
+  if (typeof source === 'string' || source instanceof URL) input = createReadStream(source)
+  else if (source instanceof ArrayBuffer) input = [new Uint8Array(source)]
+  else if (ArrayBuffer.isView(source)) input = [new Uint8Array(source.buffer, source.byteOffset, source.byteLength)]
+
+  const formats = new Map() // as far as they have streamed: newer bundles put them before any file
+  const delivered = new Map() // file -> the format onFile was given
+  const pending = []
+  const parser = new JsonStreamParser(onFile && {
+    onString(contents, path) {
+      if (path.length === 2 && path[0] === 'formats') formats.set(path[1] === '' ? '.' : path[1], contents)
+      const file = Bundle.fileKeyAt(path)
+      if (file === undefined) return contents
+      assert(!delivered.has(file), `bundle carries file '${file}' twice`)
+      const format = formats.get(file)
+      delivered.set(file, format)
+      pending.push([file, contents, format])
+      return STREAMED
+    },
+  })
+
+  // Big chunks let zlib's thread pool decompress ahead of the parser; zlib's 16 KiB default runs about 2x slower.
+  const decompress = createBrotliDecompress({ chunkSize: 1024 * 1024, readableHighWaterMark: 4 * 1024 * 1024 })
+  let drained = false
+  await pipeline(input, decompress, async (chunks) => {
+    for await (const chunk of chunks) {
+      parser.write(chunk)
+      for (const [file, contents, format] of pending.splice(0)) {
+        signal?.throwIfAborted() // one chunk can queue many files
+        // eslint-disable-next-line no-await-in-loop -- in stream order, with backpressure on the decompressor
+        await untilAborted(onFile(file, contents, { signal, format }), signal)
+      }
+    }
+    drained = true
+  }, { signal }).catch((error) => {
+    // Once drained, the stream is whole: pipeline reports unread bytes after it as an abort.
+    if (!drained || signal?.aborted) throw error
+  })
+
+  const bundle = Bundle.fromJSON(parser.end(), { contents: !onFile })
+  if (onFile) {
+    // fromJSON required a placeholder per file, so onFile got each one; it must have got nothing else.
+    let files = 0
+    for (const [, info] of bundle.modules) files += Object.keys(info.files).length
+    assert(delivered.size === files, 'bundle carries file contents outside its file list')
+    // A later `formats` key replaces the one onFile was told about.
+    for (const [file, format] of delivered) {
+      if (format !== undefined && bundle.formats.get(file) !== format) assert(false, `bundle file '${file}' changed format after onFile got it`)
+    }
+  }
+  return bundle
+}