@exodus/stasis-plugins 1.0.0-beta.9 → 1.0.0-rc.0

package/src/metro-native.js

@@ -0,0 +1,201 @@
+import { isUtf8 } from 'node:buffer'
+import { basename, dirname, join, posix, relative, sep } from 'node:path'
+
+import { readJson } from '@exodus/stasis-core/bundle-util'
+import { diskHost } from '@exodus/stasis-core/host'
+import { CODE_EXTENSIONS, classifyFormat, isAutoExcludedDir, isBinaryPlist, isDotEnvFile, isExcludedNativeFile, isNativeArtifact, nameFormat, pathExt, toPosix, upsert } from '@exodus/stasis-core/util'
+
+// The native build inputs of a React Native app's packages, shared by the Metro plugin's capture and `stasis bundle
+// --metro` so the two carry the same files. `fs` is a host: readdir(dir) -> Dirent[] (throws where `dir` isn't one),
+// readFile(file) -> Buffer, stat(path) -> Stats, or null where nothing is there.
+
+// Files a React Native build requires that neither Metro's graph nor a run's loader carries, resolved from the project
+// root: setup_env.sh, which the React Native CLI runs as every command starts (`react-native config` included).
+export const AUTO_INCLUDES = ['@react-native-community/cli/setup_env.sh']
+
+export function listing(dir, fs) {
+  try {
+    return fs.readdir(dir)
+  } catch {
+    return []
+  }
+}
+
+// RN core build scripts the native walk would defer to Metro as `.js`, but Metro never sees them (RN's podspecs
+// invoke them at pod-install). Project-relative, so react-native's `exports` can't hide them.
+export const RN_CORE_INCLUDE_FILES = ['sdks/hermes-engine/utils/replace_hermes_version.js']
+
+// By name: RN scatters podspecs in subdirs `react-native config` misses.
+const isPodspec = (name) => name.endsWith('.podspec') || name.endsWith('.podspec.json')
+
+// The off-platform top-level dir of a native package, which no build on this host compiles.
+export function isExcludedNativeDir(name, { win32 = process.platform === 'win32' } = {}) {
+  return !win32 && name === 'windows'
+}
+
+// Nested packages (attested separately) and regenerated build output, on top of the shared set.
+const NATIVE_BUILD_OUTPUT_DIRS = new Set(['node_modules', 'build', '.gradle', '.cxx', 'Pods', 'DerivedData'])
+const isSkippedNativeWalkDir = (name) => NATIVE_BUILD_OUTPUT_DIRS.has(name) || isAutoExcludedDir(name) || isNativeArtifact(name)
+
+// A compiled tool (Hermes' `hermesc`), never source.
+const isExtensionlessBinary = (name, content) => pathExt(name) === '' && Buffer.isBuffer(content) && !isUtf8(content)
+
+// A native capture's view of a file by name: { action: 'code', format } for a native build input, 'skip' for noise
+// or a JS-family file by extension (Metro owns those), else 'resource'.
+export function classifyNativeCapture(name, { win32 = process.platform === 'win32', content } = {}) {
+  // `.env` files carry secrets: an automated capture must never sweep them in.
+  if (isExcludedNativeFile(name, { win32 }) || isDotEnvFile(name)) return { action: 'skip' }
+  // Name-matched code is a native build input even where its tag is a Node format the JS graph would own.
+  const byName = nameFormat(name)
+  if (byName !== undefined) return { action: 'code', format: byName }
+  if (CODE_EXTENSIONS.has(pathExt(name))) return { action: 'skip' }
+  const format = classifyFormat(name, { content })
+  return format === undefined ? { action: 'resource' } : { action: 'code', format }
+}
+
+// classifyNativeCapture refined by the bytes, which only ever demote; a 'resource' gets its format from its bytes.
+export function refineNativeCapture(classified, name, content, resources = new Set()) {
+  if (isExtensionlessBinary(name, content)) return { action: 'skip' }
+  if (isBinaryPlist(name, content)) return { action: resources.has('plist') ? 'resource' : 'skip' }
+  return classified
+}
+
+// A file name as Metro's AssetPaths reads it (parsePlatformFilePath, then parseBaseName): its asset, `${name}.${type}`
+// less any scale (`a@2x.png` is `a.png`'s), its scale (`resolution`, 1 where it names none), and its platform where it
+// names `platform` (`a.ios.png`; `a.android.png` is an asset of its own). -> null for a name with no type
+const METRO_PLATFORM_FILE = /^(.+?)(?:\.([^.]+))?\.([^.]+)$/u
+const METRO_ASSET_SCALE = /^(.+?)(?:@([\d.]+)x)?$/u
+export function parseMetroAssetName(name, platform) {
+  const match = METRO_PLATFORM_FILE.exec(name)
+  if (match === null) return null
+  const [, base, suffix, type] = match
+  const ofPlatform = suffix !== undefined && suffix === platform
+  const [, root, scale] = METRO_ASSET_SCALE.exec(suffix === undefined || ofPlatform ? base : `${base}.${suffix}`)
+  const resolution = Number.parseFloat(scale)
+  return { asset: `${root}.${type}`, name: root, type, resolution: Number.isNaN(resolution) ? 1 : resolution, platform: ofPlatform ? platform : null }
+}
+
+// A file's stat for metroAssetFiles, where a listing names one the disk lacks.
+export const FILE_STAT = { isFile: () => true }
+
+// The files `state` (a loaded State) carries, as a directory listing of them (metroAssetFiles' host).
+export function bundleDirHost(state) {
+  const dirs = Map.groupBy([...state.sources.keys(), ...state.resources.keys()], (rel) => posix.dirname(rel))
+  return {
+    readdir: (dir) => (dirs.get(toPosix(state.relative(dir)) || '.') ?? []).map((rel) => ({ name: posix.basename(rel) })),
+    stat: () => FILE_STAT,
+  }
+}
+
+// Metro's record of the asset module `path` (getAbsoluteAssetRecord): the files it reads to build it and copies out
+// with it, each of the asset's name and type at any scale in its directory, those for `platform` where it has any,
+// else those for none. -> absolute paths
+export function metroAssetFiles(path, platform = null, host = diskHost) {
+  const own = parseMetroAssetName(basename(path), platform)
+  if (own === null) return []
+  const dir = dirname(path)
+  const byPlatform = new Map()
+  for (const { name } of host.readdir(dir)) {
+    const parsed = parseMetroAssetName(name, platform)
+    if (parsed?.asset !== own.asset || host.stat(join(dir, name))?.isFile() !== true) continue
+    upsert(byPlatform, parsed.platform, () => []).push(join(dir, name))
+  }
+  return (platform === null ? undefined : byPlatform.get(platform)) ?? byPlatform.get(null) ?? []
+}
+
+// Whether the package at `root` ships native code: an ios/ or android/ dir, or a podspec under it.
+export function looksNative(root, fs) {
+  return fs.stat(join(root, 'ios')) !== null || fs.stat(join(root, 'android')) !== null || hasPodspec(root, fs)
+}
+
+function hasPodspec(dir, fs) {
+  return listing(dir, fs).some((ent) => !ent.isSymbolicLink() &&
+    (ent.isFile() ? isPodspec(ent.name) : ent.isDirectory() && !isSkippedNativeWalkDir(ent.name) && hasPodspec(join(dir, ent.name), fs)))
+}
+
+// A spec React Native's codegen parses (its filterJSFile): a JS or TS module named `Native…` or `…NativeComponent`,
+// but a type declaration.
+const isCodegenSpec = (name) => /^(?:Native.+|.+NativeComponent)/u.test(name) && /\.(?:jsx?|tsx?)$/u.test(name) && !name.endsWith('.d.ts')
+
+// Each native build input of the package at `root`, walked whole (a podspec may compile `cpp/` beside `ios/`) but
+// build output, prebuilt binaries, an off-platform top-level dir, symlinks and what the shared classifier skips; and
+// its codegen specs, as code with no format. -> yields { file, source, action: 'code' | 'resource', format }
+export function* nativeInputs(root, fs, { resources = new Set(), skip = () => false } = {}) {
+  yield* walk(root, true)
+  for (const dir of codegenSources(root, fs)) yield* sources(dir)
+  function* sources(dir) {
+    for (const ent of listing(dir, fs)) {
+      // As codegen skips a dot, node_modules or tests directory.
+      if (ent.isSymbolicLink() || ent.name.startsWith('.') || ent.name === 'node_modules' || ent.name.startsWith('__tests')) continue
+      const file = join(dir, ent.name)
+      if (ent.isDirectory()) yield* sources(file)
+      else if (ent.isFile() && isCodegenSpec(ent.name) && !skip(file)) yield { file, source: fs.readFile(file), action: 'code', format: undefined }
+    }
+  }
+  function* walk(dir, atRoot) {
+    for (const ent of listing(dir, fs)) {
+      if (ent.isSymbolicLink()) continue
+      const file = join(dir, ent.name)
+      if (ent.isDirectory()) {
+        if (!isSkippedNativeWalkDir(ent.name) && !(atRoot && isExcludedNativeDir(ent.name))) yield* walk(file, false)
+        continue
+      }
+      if (!ent.isFile() || isNativeArtifact(ent.name) || skip(file)) continue
+      const byName = classifyNativeCapture(ent.name)
+      if (byName.action === 'skip') continue
+      const source = fs.readFile(file)
+      const { action, format } = refineNativeCapture(byName, ent.name, source, resources)
+      if (action !== 'skip') yield { file, source, action, format }
+    }
+  }
+}
+
+// The dirs the package's `codegenConfig` names as its JS sources (`jsSrcsDir`, or each of its `libraries`'), each
+// reached through no symlink, as the walk follows none: codegen parses the specs there for the native code a build
+// compiles, and `react-native config` reads them for the components they declare. -> the absolute dirs
+function codegenSources(root, fs) {
+  const config = readJson(join(root, 'package.json'), fs)?.codegenConfig
+  const named = [config?.jsSrcsDir, ...(Array.isArray(config?.libraries) ? config.libraries.map((library) => library?.jsSrcsDir) : [])]
+  return named.filter((dir) => typeof dir === 'string' && dir !== '').map((dir) => packageDir(root, dir, fs)).filter((dir) => dir !== null)
+}
+
+// The directory `rel` names in the package at `root`, each step a directory entry (not a link, not `..`), else null.
+function packageDir(root, rel, fs) {
+  let dir = root
+  for (const name of relative(root, join(root, rel)).split(sep)) {
+    if (name === '') continue
+    if (!listing(dir, fs).some((ent) => ent.name === name && ent.isDirectory())) return null
+    dir = join(dir, name)
+  }
+  return dir
+}
+
+// A package name as npm takes one (`name` or `@scope/name`), which a path can't climb out with.
+const PACKAGE_NAME = /^(?:@[^/\\@.][^/\\]*\/)?[^/\\@.][^/\\]*$/u
+
+// The app's packages `react-native config` reads, as data rather than by running the CLI: each name its package.json
+// lists under dependencies, peerDependencies or devDependencies, in the nearest node_modules up from `projectDir` that
+// holds it. What a react-native.config.js changes, being code, isn't seen. -> the package dirs
+export function linkedPackages(projectDir, fs) {
+  const manifest = readJson(join(projectDir, 'package.json'), fs)
+  const listed = (field) => (manifest?.[field] && typeof manifest[field] === 'object' ? Object.keys(manifest[field]) : [])
+  const names = new Set([...listed('dependencies'), ...listed('peerDependencies'), ...listed('devDependencies')])
+  const dirs = []
+  for (const name of names) {
+    if (!PACKAGE_NAME.test(name)) continue
+    for (let dir = projectDir; ; dir = dirname(dir)) {
+      const found = join(dir, 'node_modules', name)
+      if (fs.stat(found)?.isDirectory()) {
+        dirs.push(found)
+        break
+      }
+      if (dirname(dir) === dir) break
+    }
+  }
+  return dirs
+}
+
+// Those of linkedPackages a native build links: those that ship native code.
+export function linkedNativePackages(projectDir, fs) {
+  return linkedPackages(projectDir, fs).filter((dir) => looksNative(dir, fs))
+}

package/src/metro-transformer.js

@@ -1,85 +1,98 @@
 import { Buffer } from 'node:buffer'
 import { createRequire } from 'node:module'
-import { resolve as resolvePath } from 'node:path'
-import { pathToFileURL } from 'node:url'
+import { dirname, isAbsolute, resolve as resolvePath } from 'node:path'
+import { fileURLToPath, pathToFileURL } from 'node:url'
 
 import { State } from '@exodus/stasis-core/state'
+import { servedByBundle } from '@exodus/stasis-core/state-util'
+import { transformContext } from './metro-babel-transformer.js'
 
-// Companion to the StasisMetro serializer: the LOAD half of stasis for Metro. Load only READS an
-// immutable bundle and every worker reads the same bytes (no IPC), so a per-worker transformer is
-// the natural seam. Wire it permanently in metro.config.js (transformerPath); outside load mode it
-// is a transparent pass-through, and under load the serializer is too -- the mode picks the active
-// half. In load mode, wrap the user's real transformer and replace Metro's on-disk `data` with the
-// bundle's hash-verified bytes (fail-closed; out-of-scope files pass their disk bytes through).
-// KNOWN LIMITATION: does NOT build with sources absent from disk -- Metro reads + hashes each file
-// before this runs, so the guarantee is "build attested bytes, fail closed on disk drift."
+// The worker half of StasisMetro, wired by withStasis() as `transformerPath` around the project's transformer. It
+// keeps each import as written for the serializer (./metro-babel-transformer.js), and is the LOAD half: a per-worker
+// seam, as load only reads an immutable bundle, replacing Metro's `data` with the bundle's hash-verified bytes. Metro
+// reads and hashes each file before this runs, so sources must stay on disk: attested bytes, fail closed on drift.
 
 const require = createRequire(import.meta.url)
 
-// The real transformer we wrap (default matches Metro's; override for a custom TS/Babel preset).
-// Required lazily per worker so a non-load build pays nothing.
-const BASE_TRANSFORMER = process.env.EXODUS_STASIS_METRO_BASE_TRANSFORMER || 'metro-transform-worker'
-
-let base
-function getBase() {
-  base ??= require(BASE_TRANSFORMER)
-  return base
+// The project's transformer: the `transformerPath` withStasis() replaced, else EXODUS_STASIS_METRO_BASE_TRANSFORMER,
+// else Metro's default, a name resolved from the project root first, as Metro requires it from its own dir. This
+// transformer itself (wired by hand as well) means Metro's default.
+const bases = new Map()
+function baseOf(config, projectRoot = process.cwd()) {
+  const path = config?.stasisBaseTransformerPath ?? (process.env.EXODUS_STASIS_METRO_BASE_TRANSFORMER || 'metro-transform-worker')
+  const key = `${projectRoot}\0${path}`
+  if (!bases.has(key)) {
+    const resolve = (name) => require.resolve(name, { paths: [projectRoot, dirname(fileURLToPath(import.meta.url))] })
+    let file = resolve(path)
+    if (require(file).transform === transform) file = resolve('metro-transform-worker')
+    bases.set(key, { file, transformer: require(file) })
+  }
+  return bases.get(key)
 }
 
-// Per-worker load State, built once from the usual env + stasis.config.json. Non-load mode -> null
-// -> transparent pass-through, so wiring it permanently is safe. A construction failure is cached
-// and rethrown on every call -- never throw-once-then-silently-pass-through.
-let stateInited = false
-let loadState = null
-let stateError = null
+const baseConfig = ({ stasisBaseTransformerPath: _, ...config } = {}) => config
+
+// The worker's load State, null outside load (a pass-through, so wiring this permanently is safe). A failure is
+// cached and rethrown on every call, never passed through after the first. It never writes, so claims no write path.
+let loaded
 function getLoadState() {
-  if (!stateInited) {
+  if (loaded === undefined) {
     try {
-      const state = new State(process.cwd())
-      loadState = state.config.loadBundle ? state : null
-    } catch (err) {
-      stateError = err
-      throw err
-    } finally {
-      stateInited = true
+      const state = new State(process.cwd(), { claim: false })
+      loaded = { state: state.config.loadBundle ? state : null }
+    } catch (error) {
+      loaded = { error }
     }
   }
-  if (stateError) throw stateError
-  return loadState
+  if ('error' in loaded) throw loaded.error
+  return loaded.state
 }
 
-// True iff `absolute` is in scope for load mode. state.relative throws outside the project root
-// (out-of-scope: system files, scratch) -> defer to disk. Mirrors the webpack plugin's inScope.
-function inScope(state, absolute) {
-  try {
-    state.relative(absolute)
-  } catch {
-    return false
-  }
-  if (state.config.full) return true
-  return state.inNodeModules(pathToFileURL(absolute).toString())
+const BABEL_TRANSFORMER = fileURLToPath(new URL('./metro-babel-transformer.js', import.meta.url))
+
+// Metro resolves a configured Babel transformer; its default stays a name, which the base requires from its own dir.
+const resolveBabelTransformer = (base, path) => (isAbsolute(path) ? path : createRequire(base.file).resolve(path))
+
+// Each dependency as the source wrote it: `stasisAsWritten` where Babel rewrote its specifier (several written apart
+// may be rewritten alike), `stasisInserted` where no specifier in the source names it (a Babel helper, Metro's
+// asyncRequire). The imports a Babel plugin consumed ride on the module's first output's data, as `stasisConsumed`.
+function withAsWritten(result, { asWritten, consumed }) {
+  if (!Array.isArray(result?.dependencies)) return result
+  const dependencies = result.dependencies.map((dep) => {
+    const written = asWritten.get(dep.name)
+    if (written === undefined) return { ...dep, stasisInserted: true }
+    if (written.size === 1 && written.has(dep.name)) return dep
+    return { ...dep, stasisAsWritten: [...written] }
+  })
+  if (consumed.size === 0 || !Array.isArray(result.output) || result.output.length === 0) return { ...result, dependencies }
+  const [first, ...rest] = result.output
+  return { ...result, dependencies, output: [{ ...first, data: { ...first.data, stasisConsumed: [...consumed] } }, ...rest] }
 }
 
-export function transform(config, projectRoot, filename, data, options) {
+export async function transform(config, projectRoot, filename, data, options) {
+  const base = baseOf(config, projectRoot)
+  config = baseConfig(config)
   const state = getLoadState()
   if (state) {
-    // `filename` is project-relative in Metro's contract; resolve against projectRoot.
+    // `filename` is project-relative in Metro's contract.
     const absolute = resolvePath(projectRoot, filename)
-    if (inScope(state, absolute)) {
-      // getFile re-verifies the hash and throws on a file the bundle doesn't carry (fail-closed,
-      // no disk fallback). Code sources come back as strings; resources as Buffers.
+    // getFile re-verifies the hash and throws on a file the bundle doesn't carry: no disk fallback.
+    if (servedByBundle(state, absolute)) {
       const { source } = state.getFile(pathToFileURL(absolute).toString())
       data = Buffer.isBuffer(source) ? source : Buffer.from(source)
     }
   }
-  return getBase().transform(config, projectRoot, filename, data, options)
+  if (typeof config.babelTransformerPath !== 'string') return base.transformer.transform(config, projectRoot, filename, data, options)
+  const context = { babelTransformerPath: resolveBabelTransformer(base, config.babelTransformerPath), asWritten: new Map(), consumed: new Set() }
+  const result = await transformContext.run(context, () =>
+    base.transformer.transform({ ...config, babelTransformerPath: BABEL_TRANSFORMER }, projectRoot, filename, data, options))
+  return withAsWritten(result, context)
 }
 
-export function getCacheKey(config) {
-  // Fold in the base transformer's cache key, then add a stasis-load discriminator so load results
-  // (which can differ from a disk build even when disk == bundle) don't cross-pollinate the cache.
-  // Derive the marker from the resolved load State (not the env) so config.json activation keys too.
-  const baseKey = typeof getBase().getCacheKey === 'function' ? getBase().getCacheKey(config) : ''
+// The base's key, marked by the load State transform() goes by, so a load result never shares a disk build's entry.
+export function getCacheKey(config, opts) {
+  const { transformer } = baseOf(config, opts?.projectRoot)
+  const baseKey = typeof transformer.getCacheKey === 'function' ? transformer.getCacheKey(baseConfig(config), opts) : ''
   const state = getLoadState()
   const marker = state ? `load:${state.config.bundleFile || 'default'}` : 'off'
   return `${baseKey}$stasis-metro-transformer:${marker}`