@exodus/stasis 1.0.0-beta.3 → 1.0.0-beta.5

package/src/cmd/github-bundle.js

@@ -0,0 +1,39 @@
+import { createHash } from 'node:crypto'
+import { resolve } from 'node:path'
+
+import { createClient } from '@preventive/upstream/github.js'
+import { buildGitHubBundle } from '../vfs-bundle/github.js'
+import { bundledSummary, writeBundle, writeFile } from './output.js'
+
+// The longest file name ext4, APFS and NTFS take: 255 bytes, or characters, ours being ASCII.
+const NAME_MAX = 255
+// Each character outside the portable filename set [A-Za-z0-9._-] made `_`.
+const portable = (s) => s.replaceAll(/[^\w.-]/gu, '_')
+
+// The default output of `stasis github-bundle`: `owner-name.<commit's first 7>.stasis.code.br` of
+// the repo, or `owner-name.<directory, its / made ->.<commit's first 7>.stasis.code.br` of a
+// directory in it, portable. A directory too deep for that to fit in NAME_MAX keeps what fits of
+// its start, and `_` and the first 8 of its sha256 after it; a valid `github` (at most 140
+// characters) leaves at least 91 for it.
+export function githubBundleFile({ github, directory, commit }) {
+  const repo = portable(github.replace('/', '-'))
+  const tail = `.${portable(commit.slice(0, 7))}.stasis.code.br`
+  if (!directory) return `${repo}${tail}`
+  const room = NAME_MAX - `${repo}.${tail}`.length
+  let dir = portable(directory.replaceAll('/', '-'))
+  if (dir.length > room) dir = `${dir.slice(0, room - 9)}_${createHash('sha256').update(directory).digest('hex').slice(0, 8)}`
+  return `${repo}.${dir}${tail}`
+}
+
+// Run `stasis github-bundle`: the bundle of a GitHub repo at a commit, or the one a tag names (the
+// default branch's head without either), as buildGitHubBundle builds it from `options`, written
+// brotli-compressed to `output` (githubBundleFile's name for it by default, `-` for stdout), and a
+// JS bundle's lockfile to `lockfile` where given.
+// The tree is fetched with GITHUB_TOKEN from `env` where it is set; nothing else of `env` is read.
+export async function githubBundleCommand({ cwd = process.cwd(), env = process.env, output, lockfile, brotliQuality, client, ...options } = {}) {
+  const built = await buildGitHubBundle({ ...options, lockfile, client: client ?? createClient({ token: env.GITHUB_TOKEN || null }) })
+  const dest = writeBundle(cwd, output ?? githubBundleFile({ github: options.github, directory: options.directory, commit: built.bundle.repo.commit }), built.bundle.serialize(), brotliQuality)
+  if (lockfile !== undefined) writeFile(resolve(cwd, lockfile), built.lockfile.serialize())
+  const from = `${options.github}@${built.bundle.repo.commit}${options.directory ? `/${options.directory}` : ''}`
+  console.warn(bundledSummary(built.bundle.sources.size, built.bundle.modules, from, dest))
+}

package/src/cmd/output.js

@@ -0,0 +1,36 @@
+import { mkdirSync, writeFileSync } from 'node:fs'
+import { dirname, resolve } from 'node:path'
+import { brotliCompressSync } from 'node:zlib'
+
+import { brotliOptions } from '@exodus/stasis-core/brotli'
+
+// Default bundle output: stasis.code.br, the same name `stasis run --bundle=load` discovers, so the two round-trip with no flags.
+export const DEFAULT_BUNDLE_FILE = 'stasis.code.br'
+
+// Write `data` to the absolute path `abs`, creating its directory.
+export function writeFile(abs, data) {
+  mkdirSync(dirname(abs), { recursive: true })
+  writeFileSync(abs, data)
+}
+
+// Write `data` to `target` (resolved against `cwd`), or to stdout for `-`. -> where it went, for the summary
+export function writeOutput(cwd, target, data) {
+  if (target === '-') {
+    process.stdout.write(data)
+    return '<stdout>'
+  }
+  writeFile(resolve(cwd, target), data)
+  return target
+}
+
+// Write a bundle's `serialized` text brotli-compressed (at `brotliQuality`) as writeOutput does. -> where it went
+export const writeBundle = (cwd, target, serialized, brotliQuality) => writeOutput(cwd, target, brotliCompressSync(serialized, brotliOptions(brotliQuality)))
+
+// `N package(s)`: every non-empty bucket of a bundle's `modules` (the "." workspace bucket counts as one).
+export function packagesLabel(modules) {
+  const packages = [...modules.values()].filter((m) => Object.keys(m.files).length > 0).length
+  return `${packages} package${packages === 1 ? '' : 's'}`
+}
+
+// The summary of a bundle of `count` files in `modules` written from `from` to `dest`.
+export const bundledSummary = (count, modules, from, dest) => `[stasis] Bundled ${count} files in ${packagesLabel(modules)} from ${from} to ${dest}`

package/src/cmd/sbom.js

@@ -1,8 +1,8 @@
-import { mkdirSync, writeFileSync } from 'node:fs'
-import { dirname, resolve } from 'node:path'
+import { resolve } from 'node:path'
 
 import { parseFile } from '../parse.js'
 import { collectComponents, generateSbom } from '../sbom.js'
+import { writeOutput } from './output.js'
 
 // The one-line summary goes to stderr so it never corrupts an SBOM streamed to stdout.
 export function sbomCommand({ cwd = process.cwd(), files, format, output, now, uuid } = {}) {
@@ -14,15 +14,7 @@ export function sbomCommand({ cwd = process.cwd(), files, format, output, now, u
   const doc = generateSbom(format, components, { now, uuid })
   const text = JSON.stringify(doc, undefined, 2) + '\n'
 
-  const target = output ?? '-'
-  if (target === '-') {
-    process.stdout.write(text)
-  } else {
-    const outAbs = resolve(cwd, target)
-    mkdirSync(dirname(outAbs), { recursive: true })
-    writeFileSync(outAbs, text)
-  }
-  const dest = target === '-' ? '<stdout>' : target
+  const dest = writeOutput(cwd, output ?? '-', text)
   const n = components.length
   console.warn(`[stasis] Wrote ${format} SBOM with ${n} component${n === 1 ? '' : 's'} to ${dest}`)
   return { components, doc }

package/src/diff.js

@@ -19,6 +19,8 @@ function digestOf(kind, format, value, hash) {
   return hash(value)
 }
 
+const projectPath = (dir, rel) => (dir === '.' ? rel : `${dir}/${rel}`)
+
 // Project an artifact onto a uniform shape: scope + Map<dir, { name, version, ecosystem,
 // files: Map<rel, digest> }>. `input` is { artifact, kind }; `hash` re-hashes bundle bytes only.
 export function normalizeArtifact(input, { hash } = {}) {
@@ -31,8 +33,7 @@ export function normalizeArtifact(input, { hash } = {}) {
   for (const [dir, { name, version, ecosystem, files }] of artifact.modules) {
     const digests = new Map()
     for (const [rel, value] of Object.entries(files)) {
-      const full = dir === '.' ? rel : `${dir}/${rel}`
-      digests.set(rel, digestOf(kind, formats.get(full), value, hash))
+      digests.set(rel, digestOf(kind, formats.get(projectPath(dir, rel)), value, hash))
     }
     // v0 bundles record no name/version; normalize undefined -> null for a single "unknown" sentinel.
     modules.set(dir, { name: name ?? null, version: version ?? null, ecosystem: ecosystem ?? null, files: digests })
@@ -42,8 +43,6 @@ export function normalizeArtifact(input, { hash } = {}) {
   return { scope: artifact.config?.scope ?? 'full', modules, executable: artifact.executable ?? new Set() }
 }
 
-const projectPath = (dir, rel) => (dir === '.' ? rel : `${dir}/${rel}`)
-
 // Diff two `{ artifact, kind }` operands (`left` = baseline/"from", `right` = "to"). Modules are
 // compared whole-package; files only within packages present on BOTH sides (a one-sided package is
 // reported once as an added/removed module, not per-file). A version/name change is reported only
@@ -60,16 +59,17 @@ export function diffArtifacts(left, right, { imports = false, hash } = {}) {
   const filesRemoved = []
   const filesDiffering = []
 
+  const summary = (dir, m) => ({ dir, name: m.name, version: m.version, ecosystem: m.ecosystem, files: m.files.size })
   const dirs = new Set([...L.modules.keys(), ...R.modules.keys()])
   for (const dir of dirs) {
     const l = L.modules.get(dir)
     const r = R.modules.get(dir)
     if (l && !r) {
-      modulesRemoved.push({ dir, name: l.name, version: l.version, ecosystem: l.ecosystem, files: l.files.size })
+      modulesRemoved.push(summary(dir, l))
       continue
     }
     if (!l && r) {
-      modulesAdded.push({ dir, name: r.name, version: r.version, ecosystem: r.ecosystem, files: r.files.size })
+      modulesAdded.push(summary(dir, r))
       continue
     }
 

package/src/json-stream.js

@@ -0,0 +1,283 @@
+// Incremental JSON parser over UTF-8 bytes, holding only the value being built and one token in flight.
+// Exactly JSON.parse(Buffer.concat(chunks).toString('utf8')) for any chunking (json-stream.test.js checks it),
+// so a bundle read in a stream can't parse differently from one read whole. Pure: no Node builtins.
+// Not mirrored: Node 24+'s JSON.parse can return a stale escaped key cached by an earlier call (a V8 bug).
+
+const QUOTE = 0x22
+const BACKSLASH = 0x5c
+
+// What the next significant byte may be.
+const VALUE = 0 // any value: the top level, after ':' or after an array's ','
+const ARRAY_FIRST = 1 // after '[': a value or ']'
+const OBJECT_FIRST = 2 // after '{': a key or '}'
+const KEY = 3 // after an object's ',': a key
+const COLON = 4
+const AFTER = 5 // after a member or element: ',' or the container's closer
+const DONE = 6 // the top-level value is complete: whitespace only
+// Inside a token that can straddle chunks.
+const STRING = 7
+const NUMBER = 8
+const LITERAL = 9
+
+const NUMBER_RE = /^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?$/u
+const LITERALS = new Map([['true', true], ['false', false], ['null', null]])
+
+const isWhitespace = (b) => b === 0x20 || b === 0x0a || b === 0x0d || b === 0x09
+// The bytes a number can span; NUMBER_RE checks its grammar once it ends.
+const isNumberByte = (b) => (b >= 0x30 && b <= 0x39) || b === 0x2d || b === 0x2b || b === 0x2e || b === 0x65 || b === 0x45
+const isLetter = (b) => b >= 0x61 && b <= 0x7a
+
+const hasControlByte = (bytes) => {
+  for (let i = 0; i < bytes.length; i++) if (bytes[i] < 0x20) return true
+  return false
+}
+
+const indexOrEnd = (bytes, byte, from) => {
+  const at = bytes.indexOf(byte, from)
+  return at === -1 ? bytes.length : at
+}
+
+const concat = (parts) => {
+  let length = 0
+  for (const part of parts) length += part.length
+  const out = new Uint8Array(length)
+  let offset = 0
+  for (const part of parts) {
+    out.set(part, offset)
+    offset += part.length
+  }
+  return out
+}
+
+export class JsonStreamParser {
+  // ignoreBOM keeps a leading U+FEFF, as Buffer#toString does; no UTF-8 sequence spans a quote, so per-string decoding is exact.
+  #decoder = new TextDecoder('utf-8', { ignoreBOM: true })
+  #onString
+  #state = VALUE
+  #stack = [] // open containers, innermost last
+  #path = [] // #path[d]: the key or index #stack[d]'s in-flight member lands at
+  #result
+  #offset = 0 // bytes consumed by earlier writes, for error positions
+  #closed = false
+  // Cached indexOf results in the current chunk (-1: not searched yet; its length: none left).
+  #nextQuote = -1
+  #nextBackslash = -1
+  #tokenStart = 0
+  #isKey = false
+  #parts = [] // copied bytes of a string straddling chunks
+  #escaped = false
+  #pendingEscape = false // a chunk ended on a string's backslash
+  #word = '' // a number or literal straddling chunks
+
+  // onString(value, path) sees each string value (never a key) and its return is stored instead.
+  // `path` is the live key/index path: read it, never keep or mutate it.
+  constructor({ onString } = {}) {
+    this.#onString = onString
+  }
+
+  // A chunk may be reused once write() returns: the bytes of a token left open are copied.
+  write(bytes) {
+    if (this.#closed) throw new Error('JsonStreamParser: write after end or failure')
+    if (!ArrayBuffer.isView(bytes)) throw new TypeError('JsonStreamParser: write() takes a Uint8Array')
+    // A plain Uint8Array view: a Buffer's slice() doesn't copy, and its indexOf/subarray are slower.
+    const view = bytes.constructor === Uint8Array ? bytes : new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength)
+    try {
+      this.#consume(view)
+    } catch (error) {
+      this.#closed = true
+      throw error
+    }
+    this.#offset += view.length
+  }
+
+  end() {
+    if (this.#closed) throw new Error('JsonStreamParser: end after end or failure')
+    this.#closed = true
+    if (this.#state === NUMBER || this.#state === LITERAL) this.#finishWord()
+    if (this.#state !== DONE) throw new SyntaxError(`Unexpected end of JSON input at byte ${this.#offset}`)
+    const result = this.#result
+    this.#result = undefined
+    return result
+  }
+
+  #fail(message, i) {
+    throw new SyntaxError(`${message} at byte ${this.#offset + i}`)
+  }
+
+  #consume(bytes) {
+    const n = bytes.length
+    this.#nextQuote = -1
+    this.#nextBackslash = -1
+    let i = 0
+    while (i < n) {
+      const state = this.#state
+      if (state === STRING) {
+        i = this.#string(bytes, i)
+      } else if (state === NUMBER || state === LITERAL) {
+        i = this.#scanWord(bytes, i)
+      } else if (isWhitespace(bytes[i])) {
+        i += 1
+      } else {
+        i = this.#structural(bytes[i], i)
+      }
+    }
+  }
+
+  #structural(b, i) {
+    const state = this.#state
+    const isArray = Array.isArray(this.#stack.at(-1))
+    if ((state === AFTER || state === ARRAY_FIRST || state === OBJECT_FIRST) && b === (isArray ? 0x5d : 0x7d)) {
+      this.#close()
+      return i + 1
+    }
+    if (state === AFTER) {
+      if (b !== 0x2c) this.#fail(`Unexpected byte 0x${b.toString(16)} after a value`, i)
+      this.#state = isArray ? VALUE : KEY
+      return i + 1
+    }
+    if (state === COLON) {
+      if (b !== 0x3a) this.#fail(`Expected ':' after a key, got byte 0x${b.toString(16)}`, i)
+      this.#state = VALUE
+      return i + 1
+    }
+    if (state === OBJECT_FIRST || state === KEY) {
+      if (b !== QUOTE) this.#fail(`Expected a string key, got byte 0x${b.toString(16)}`, i)
+      this.#startToken(STRING, i)
+      this.#isKey = true
+      return i + 1
+    }
+    if (state === DONE) this.#fail(`Unexpected byte 0x${b.toString(16)} after the JSON value`, i)
+    // VALUE or ARRAY_FIRST
+    if (isArray) this.#path[this.#stack.length - 1] = this.#stack.at(-1).length
+    if (b === QUOTE) {
+      this.#startToken(STRING, i)
+      this.#isKey = false
+      return i + 1
+    }
+    if (b === 0x7b || b === 0x5b) {
+      this.#stack.push(b === 0x7b ? {} : [])
+      this.#path.push(undefined)
+      this.#state = b === 0x7b ? OBJECT_FIRST : ARRAY_FIRST
+      return i + 1
+    }
+    if (b === 0x2d || (b >= 0x30 && b <= 0x39)) {
+      this.#startToken(NUMBER, i)
+      return i
+    }
+    if (b === 0x74 || b === 0x66 || b === 0x6e) {
+      this.#startToken(LITERAL, i)
+      return i
+    }
+    return this.#fail(`Unexpected byte 0x${b.toString(16)} where a value was expected`, i)
+  }
+
+  #startToken(state, i) {
+    this.#state = state
+    this.#tokenStart = this.#offset + i
+  }
+
+  #string(bytes, start) {
+    const n = bytes.length
+    let i = start
+    if (this.#pendingEscape) {
+      this.#pendingEscape = false
+      i += 1
+    }
+    for (;;) {
+      if (this.#nextQuote < i) this.#nextQuote = indexOrEnd(bytes, QUOTE, i)
+      if (this.#nextBackslash < i) this.#nextBackslash = indexOrEnd(bytes, BACKSLASH, i)
+      const quote = this.#nextQuote
+      const backslash = this.#nextBackslash
+      if (backslash < quote) {
+        this.#escaped = true
+        if (backslash + 1 === n) {
+          this.#pendingEscape = true
+          break
+        }
+        i = backslash + 2
+      } else if (quote === n) {
+        break
+      } else {
+        this.#finishString(bytes.subarray(start, quote))
+        return quote + 1
+      }
+    }
+    this.#parts.push(bytes.slice(start))
+    return n
+  }
+
+  #finishString(tail) {
+    let bytes = tail
+    if (this.#parts.length > 0) {
+      this.#parts.push(tail)
+      bytes = concat(this.#parts)
+      this.#parts = []
+    }
+    let value = this.#decoder.decode(bytes)
+    if (this.#escaped) {
+      // JSON.parse handles the escapes, and rejects raw control bytes.
+      this.#escaped = false
+      try {
+        value = JSON.parse(`"${value}"`)
+      } catch (cause) {
+        throw new SyntaxError(`Bad string at byte ${this.#tokenStart}`, { cause })
+      }
+    } else if (hasControlByte(bytes)) {
+      throw new SyntaxError(`Bad control character in string at byte ${this.#tokenStart}`)
+    }
+    if (this.#isKey) {
+      this.#path[this.#path.length - 1] = value
+      this.#state = COLON
+    } else {
+      this.#value(this.#onString === undefined ? value : this.#onString(value, this.#path))
+    }
+  }
+
+  #scanWord(bytes, i) {
+    const n = bytes.length
+    const isNumber = this.#state === NUMBER
+    let j = i
+    while (j < n && (isNumber ? isNumberByte(bytes[j]) : isLetter(bytes[j]))) j += 1
+    this.#word += this.#decoder.decode(bytes.subarray(i, j))
+    if (j === n) return n
+    this.#finishWord()
+    return j
+  }
+
+  #finishWord() {
+    const word = this.#word
+    this.#word = ''
+    if (this.#state === NUMBER) {
+      if (!NUMBER_RE.test(word)) throw new SyntaxError(`Bad number at byte ${this.#tokenStart}`)
+      this.#value(Number(word))
+    } else {
+      if (!LITERALS.has(word)) throw new SyntaxError(`Bad literal at byte ${this.#tokenStart}`)
+      this.#value(LITERALS.get(word))
+    }
+  }
+
+  #close() {
+    this.#path.pop()
+    this.#value(this.#stack.pop())
+  }
+
+  #value(value) {
+    const depth = this.#stack.length
+    if (depth === 0) {
+      this.#result = value
+      this.#state = DONE
+      return
+    }
+    const container = this.#stack[depth - 1]
+    const key = this.#path[depth - 1]
+    if (Array.isArray(container)) {
+      container.push(value)
+    } else if (key === '__proto__') {
+      // JSON.parse defines an own data property; assignment would hit Object.prototype's setter.
+      Object.defineProperty(container, key, { value, writable: true, enumerable: true, configurable: true })
+    } else {
+      container[key] = value
+    }
+    this.#state = AFTER
+  }
+}

package/src/loaders/bash.js

@@ -4,11 +4,14 @@
 // paths, absolute/escaping paths) is best-effort and dropped. A dynamically
 // sourced path can be pinned by a `# shellcheck source=...` directive.
 
-import { realpathSync, statSync } from 'node:fs'
+import { isUtf8 } from 'node:buffer'
+import { realpathSync } from 'node:fs'
 import { readFile } from 'node:fs/promises'
-import { dirname, extname, isAbsolute, join, relative, resolve } from 'node:path'
+import { dirname, extname, join, resolve } from 'node:path'
 
 import { assertRealPathWithinBase } from '@exodus/stasis-core/util'
+import { isFile } from '../resolve-typescript.js'
+import { applyToDir, assertWithinBase } from './paths.js'
 
 // The capture class excludes backtick/`(`/`)` so a ref inside a command
 // substitution (`` `source x.sh` ``, `$(bash x.sh)`) doesn't swallow the closing delimiter.
@@ -51,31 +54,12 @@ export function resolveBashCall(ref, fromFile, { knownFiles, baseDir } = {}) {
   const present = (p) => {
     if (!p) return false
     if (knownFiles) return knownFiles.has(p)
-    if (baseDir) {
-      try {
-        return statSync(join(baseDir, p)).isFile()
-      } catch {
-        return false
-      }
-    }
+    if (baseDir) return isFile(join(baseDir, p))
     return false
   }
 
-  const fromDir = fromFile.includes('/') ? fromFile.slice(0, fromFile.lastIndexOf('/')) : ''
-  const resolveRel = (r) => {
-    const parts = [...(fromDir ? fromDir.split('/') : []), ...r.split('/')]
-    const out = []
-    for (const part of parts) {
-      if (part === '.' || part === '') continue
-      if (part === '..') {
-        if (out.length === 0) return null // escapes baseDir
-        out.pop()
-      } else {
-        out.push(part)
-      }
-    }
-    return out.join('/') || null
-  }
+  // Null when it escapes baseDir or names baseDir itself.
+  const resolveRel = (r) => applyToDir(fromFile, r.split('/')) || null
 
   // Absolute → project-relative, accepted only if it exists (so `/etc/...` is dropped).
   if (ref.startsWith('/')) {
@@ -128,6 +112,13 @@ export function buildBashTree(sources) {
   return { sources, resolutions, missing }
 }
 
+// A script's text. One that isn't UTF-8 is refused rather than carried with U+FFFD in it.
+async function readScript(baseDir, relPath) {
+  const bytes = await readFile(join(baseDir, relPath))
+  if (!isUtf8(bytes)) throw new Error(`Shell script is not valid UTF-8: ${relPath}`)
+  return bytes.toString('utf8')
+}
+
 // Walk from `entries` following resolved refs, reading each script once; a ref
 // missing on disk warns and is skipped without aborting the walk.
 export async function collectBashFilesFromDisk(baseDir, entries) {
@@ -141,7 +132,7 @@ export async function collectBashFilesFromDisk(baseDir, entries) {
       toLoad.map(async (relPath) => {
         try {
           assertRealPathWithinBase(realBase, baseDir, relPath)
-          return [relPath, await readFile(join(baseDir, relPath), 'utf8')]
+          return [relPath, await readScript(baseDir, relPath)]
         } catch (err) {
           if (err.code === 'ENOENT') {
             console.warn(`[loader.bash] Missing file: ${relPath}`)
@@ -173,16 +164,6 @@ export async function collectBashFilesFromDisk(baseDir, entries) {
   return sources
 }
 
-// Reject absolute and `..`-escaping paths in a `.sh.txt` listing so it can't
-// read files outside the listing's own directory.
-function assertWithinBase(baseDir, candidate, label) {
-  if (isAbsolute(candidate)) throw new Error(`${label} must not be absolute: ${candidate}`)
-  const rel = relative(baseDir, resolve(baseDir, candidate)).split(/[\\/]/u).join('/')
-  if (rel.startsWith('..') || isAbsolute(rel)) {
-    throw new Error(`${label} escapes baseDir: ${candidate}`)
-  }
-}
-
 // High-level entry: reads the scripts named in a `.sh.txt` listing (relative to
 // it). Unlike loadRust, the listing is authoritative — bash sourcing is too
 // dynamic to walk — so exactly the listed files are read, resolutions computed among them.
@@ -202,7 +183,7 @@ export async function loadBash(shTxtFile) {
   const reads = await Promise.all(
     lines.map(async (relPath) => {
       assertRealPathWithinBase(realBase, baseDir, relPath)
-      return [relPath, await readFile(join(baseDir, relPath), 'utf8')]
+      return [relPath, await readScript(baseDir, relPath)]
     }),
   )
   return buildBashTree(new Map(reads))