jz 0.5.1 → 0.6.0

package/index.js

@@ -6,7 +6,7 @@
  *   source (string)
  *     ↓  parse (subscript/jessie) — lexing + expression-oriented AST
  *   raw AST: nested arrays `[op, ...args]`, no ctx mutation
- *     ↓  jzify (opt-in via opts.jzify) — lower full-JS subset (var/function/class/switch) to jz-native
+ *     ↓  jzify (default-on; skipped under opts.strict) — lower full-JS subset (var/function/class/switch) to jz-native
  *   desugared AST: arrow functions + let/const/if only
  *     ↓  prepare — validate (reject disallowed ops), normalize (++/--→+=/-=, scope rename),
  *        extract (functions→ctx.func.list with sig), resolve (imports→ctx.module.imports),
@@ -14,12 +14,12 @@
  *   prepared AST: normalized, with `ctx.func.list` / `ctx.module.imports` / `ctx.schema.list`
  *     populated. Arrow bodies carry no type info yet.
  *     ↓  compile — drives per-function emit, interleaves analysis (locals/valTypes/captures/
- *        narrowing fixpoint) with IR generation via the emitter table (src/emit.js).
+ *        narrowing fixpoint) with IR generation via the emitter table (src/compile/emit.js).
  *        Writes: `ctx.func.valTypes`/`.locals`, `ctx.types.*`, `ctx.runtime.*`, `ctx.core.includes`.
- *        Also calls optimizeFunc (src/optimize.js): `hoistPtrType` + fused peephole/inline/memarg walk.
+ *        Also calls optimizeFunc (src/optimize/index.js): `hoistPtrType` + fused peephole/inline/memarg walk.
  *   WAT IR: watr S-expression `['module', ...sections]`, every instruction node carries `.type`.
- *     ↓  watrOptimize (opt-out via opts.optimize=false) — CSE, DCE, const folding at WAT level
- *     ↓  optimizeFunc 2nd pass — re-folds rebox/unbox roundtrips that watrOptimize's inliner
+ *     ↓  watOptimize (opt-out via opts.optimize=false) — CSE, DCE, const folding at WAT level
+ *     ↓  optimizeFunc 2nd pass — re-folds rebox/unbox roundtrips that watOptimize's inliner
  *        re-introduces at inline boundaries (caller's boxPtrIR meets callee's
  *        i32.wrap_i64(i64.reinterpret_f64 __env)). watr's peephole doesn't cover this.
  *     ↓  watrPrint (opts.wat=true) → WAT text, or watrCompile → Uint8Array binary
@@ -40,14 +40,19 @@
  * @module jz
  */
 
-import { parse } from 'subscript/feature/jessie'
-import { compile as watrCompile, print as watrPrint, optimize as watrOptimize } from "watr";
-import { ctx, reset, err } from './src/ctx.js'
-import prepare, { GLOBALS } from './src/prepare.js'
-import compile from './src/compile.js'
-import { emitter } from './src/emit.js'
-import { optimizeFunc, collectVolatileGlobals, resolveOptimize } from './src/optimize.js'
-import jzify from './src/jzify.js'
+import { parse } from './src/parse.js'
+import watrCompile from "watr/compile";
+import watrPrint from "watr/print";
+import watOptimize from "./src/wat/optimize.js";
+import { ctx, reset, err, initWarnings, assertCtxInvariants } from './src/ctx.js'
+import prepare, { GLOBALS } from './src/prepare/index.js'
+import compile from './src/compile/index.js'
+import { resetProgramFactsCache } from './src/compile/program-facts.js'
+import { emit, emitter, emitVoid as flat, emitBlockBody as body, emitBoolStr as bool, emitIndex as idx, buildArrayWithSpreads as spread } from './src/compile/emit.js'
+import { optimizeFunc, collectVolatileGlobals, collectReachableGlobalWrites, hoistGlobalPtrOffset, stablePtrGlobalNames, resolveOptimize } from './src/optimize/index.js'
+import { VAL } from './src/reps.js'
+import jzify from './jzify/index.js'
+import { T } from './src/ast.js'
 import {
   memory as enhanceMemory, instantiate as instantiateRuntime,
 } from './interop.js'
@@ -58,6 +63,11 @@ const importsMayReturnExternal = (imports) =>
   !!imports && Object.values(imports).some(mod =>
     Object.values(mod || {}).some(spec => typeof spec === 'function'))
 
+// WHATWG URL resolution for compile-time import.meta lowering. Injected into
+// ctx.transform (like parse/jzify) so prepare never references the `URL` global
+// directly — keeps the self-host kernel free of host-only built-ins.
+const resolveUrl = (spec, base) => new URL(spec, base).href
+
 const nowMs = () => globalThis.performance?.now ? globalThis.performance.now() : Date.now()
 
 const compileProfiler = (profile) => {
@@ -154,9 +164,10 @@ jz.memory = enhanceMemory
  * @param {string} code - jz source
  * @param {object} [opts]
  * @param {boolean} [opts.wat] - Return WAT text instead of binary
- * @param {boolean} [opts.strict] - Reject dynamic features (obj[k], for-in, unknown
- *   receiver method calls) at compile time. Avoids pulling dynamic-dispatch stdlib
- *   into output; large size win for static programs.
+ * @param {boolean} [opts.strict] - Enforce the pure canonical subset: skip jzify
+ *   (so full-JS syntax like var/function/class is rejected, not lowered) and reject
+ *   dynamic features (obj[k], for-in, unknown receiver method calls) at compile time.
+ *   Avoids pulling dynamic-dispatch stdlib into output; large size win for static programs.
  * @param {WebAssembly.Memory|number} [opts.memory] - Shared memory import, or
  *   initial page count. Prefer `memory: N` for owned memory.
  * @param {boolean} [opts.alloc=true] - Export raw allocator helpers
@@ -172,13 +183,28 @@ jz.memory = enhanceMemory
  *     mutable globals); trades size for speed.
  *   - `{ level?: 0|1|2|3, watr?: bool, hoistAddrBase?: bool, ... }`: per-pass
  *     overrides on top of the chosen level. See PASS_NAMES in src/optimize.js.
+ * @param {object} [opts.warnings] - Optional mutable warning sink populated with
+ *   `entries: [{ code, message, fn?, line?, column? }]`. Heap-growth advisories
+ *   fire when a module uses the bump allocator and an export or loop retains
+ *   allocations without a host-side memory.reset().
  * @param {object} [opts.profile] - Optional mutable profile sink populated with
  *   `entries` and `totals` for parse / jzify / prepare / compile / plan / watr phases.
  *   Set `profile.names = true` to also emit a standard wasm `name` custom section
  *   for profiler/debugger symbolication.
- * @param {boolean} [opts.profileNames] - Legacy alias for `profile.names`.
+ * @param {Object<string,string>} [opts.modules] - Map of module specifier → source
+ *   for compile-time `import`/`export` bundling: jz resolves the module graph
+ *   in-process from this map instead of reading from disk.
+ * @param {boolean} [opts.noTailCall] - Disable proper-tail-call emission (self/mutual
+ *   recursion uses ordinary call frames). For engines/tools without the tail-call proposal.
+ * @param {boolean} [opts.nativeTimers] - Emit a blocking `__timer_loop` in `_start` so
+ *   setTimeout/setInterval fire under a standalone runtime (e.g. the wasmtime CLI) that
+ *   has no host event loop. Default: timers defer to the JS host.
  * @param {string} [opts.importMetaUrl] - Module URL used to lower `import.meta.url`
  *   and static `import.meta.resolve("...")` expressions.
+ * @param {number|boolean} [opts.randomSeed] - Seed for `Math.random`. Default: seeded
+ *   once from host entropy on first use (crypto under `host:'js'`, `random_get` under
+ *   WASI) — non-reproducible. Pass a number for a fixed, reproducible seed; `true` forces
+ *   entropy explicitly. The randomness syscall is emitted only when `Math.random` is used.
  * @param {boolean} [opts.inspect] - When true, return `{ wasm, inspect }`
  *   (or `{ wat, inspect }` with `opts.wat`) instead of the bare output.
  *   `inspect` carries per-function inferred shapes (params, locals, JSON shapes,
@@ -186,7 +212,14 @@ jz.memory = enhanceMemory
  *   without re-running the analyzer. Pays a small serialization cost; off by default.
  * @returns {Uint8Array|string|{wasm: Uint8Array, inspect: object}|{wat: string, inspect: object}}
  */
+// Test-only compile target. When set (by test/index.js under JZ_TEST_TARGET=jz.wasm)
+// every jz.compile / compile / jz() call routes through it instead of the in-process
+// compiler — used to run the whole suite against dist/jz.wasm (the jz compiler
+// compiled to wasm by jz). null in production: one boolean check on a cold path.
+let compileTarget = null
+export const _setCompileTarget = (fn) => { compileTarget = fn }
 jz.compile = (code, opts = {}) => {
+  if (compileTarget) return compileTarget(code, opts)
   try {
     return jzCompileInner(code, opts)
   } catch (e) {
@@ -194,7 +227,8 @@ jz.compile = (code, opts = {}) => {
     // codegen leak — surface it as an internal compile error with the source
     // location we were standing on. err()-thrown errors already pass through.
     if (e?.name === 'TypeError' || e?.name === 'ReferenceError' || e?.name === 'RangeError') {
-      err(`internal: ${e.message} (jz hit an unsupported case while compiling${ctx.error.node ? '; the AST node above shows the trigger' : ''}). This is a jz bug — please report.`)
+      // Pass `e` as the cause so the original stack (the real codegen site) survives.
+      err(`internal: ${e.message} (jz hit an unsupported case while compiling${ctx.error.node ? '; the AST node above shows the trigger' : ''}). This is a jz bug — please report.`, e)
     }
     throw e
   }
@@ -274,7 +308,7 @@ const detectOptimizeConfig = (ast, code) => {
   // jz's already-optimized IR and inflates output. Disable it automatically.
   const isLarge = s.sourceChars > 4000 || s.funcCount > 40 || s.maxFuncBodySize > 300
   const isMachineLike = s.callSites > 300 && s.stringLiteralCount < 10
-  if (isLarge || isMachineLike) cfg.watr = false
+  if (isLarge || isMachineLike) { cfg.watr = false; cfg.splitCharScan = false }
   // Typed-array heavy: tighten scalarization thresholds when we see large
   // fixed-size arrays; keep defaults for small/dynamic ones.
   if (s.typedArrayCount > 0 && s.maxTypedArrayLen > 0) {
@@ -292,13 +326,40 @@ const detectOptimizeConfig = (ast, code) => {
   return cfg
 }
 
-const jzCompileInner = (code, opts = {}) => {
-  const profiler = compileProfiler(opts.profile)
-  const time = (name, fn) => profiler ? profiler.time(name, fn) : fn()
-
-  reset(emitter, GLOBALS)
+// Test-matrix bridge: when JZ_TEST_* env vars are set, inject them as default
+// opts so the npm test suite can be re-run under varying configurations (opt
+// levels, host, jzify, …) without source changes. User-supplied opts always win
+// — env defaults fill only what the caller left unset. Resolved once at module
+// load; no-op (single boolean check) when the env is empty (production path).
+const TEST_ENV_DEFAULTS = (() => {
+  // Guard bare `process` — the compiler bundle must load in browsers/Workers too
+  // (this IIFE runs at module import; an unguarded ref throws ReferenceError there).
+  const e = (typeof process !== 'undefined' && process.env) || {}
+  const out = {}
+  if (e.JZ_TEST_OPTIMIZE != null) {
+    const v = e.JZ_TEST_OPTIMIZE
+    out.optimize = /^-?\d+$/.test(v) ? Number(v) : v === 'false' ? false : v
+  }
+  if (e.JZ_TEST_HOST) out.host = e.JZ_TEST_HOST
+  if (e.JZ_TEST_STRICT) out.strict = e.JZ_TEST_STRICT === '1'
+  return out
+})()
+const HAS_TEST_ENV = Object.keys(TEST_ENV_DEFAULTS).length > 0
+
+// Shared front-half: reset ctx, wire opts → ctx.transform/memory/module/features,
+// and inject parse/resolveUrl. Called by `jzCompileInner` (the only entry point
+// today). The self-host entry (scripts/self.js) drives reset itself rather than
+// going through this path, since it needs only a minimal, interop-free setup.
+const setupCtx = (code, opts) => {
+  if (HAS_TEST_ENV) {
+    const merged = { ...opts }
+    for (const k of Object.keys(TEST_ENV_DEFAULTS)) if (merged[k] == null) merged[k] = TEST_ENV_DEFAULTS[k]
+    opts = merged
+  }
+  reset(emitter, GLOBALS, { emit, flat, body, bool, idx, spread })
+  resetProgramFactsCache()
   ctx.error.src = code
-
+  initWarnings(opts.warnings)
   if (typeof opts.memory === 'number') ctx.memory.pages = opts.memory
   else if (opts.memory) ctx.memory.shared = true
   if (opts.modules) ctx.module.importSources = opts.modules
@@ -306,9 +367,14 @@ const jzCompileInner = (code, opts = {}) => {
     ctx.module.hostImports = opts.imports
     if (importsMayReturnExternal(opts.imports)) ctx.features.external = true
   }
-  // jzify: true → accept full JS subset (function/var/switch lowered to arrows/let/if).
-  // Default: strict jz (prepare rejects disallowed JS features). subscript handles ASI natively.
-  if (opts.jzify) ctx.transform.jzify = jzify
+  // Parser for compile-time import bundling (prepareModule). Injected, not
+  // imported by prepare — see ctx.transform.parse note in prepare/index.js.
+  ctx.transform.parse = parse
+  ctx.transform.resolveUrl = resolveUrl
+  // jzify runs by default — accept the full JS subset (function/var/switch lowered to
+  // arrows/let/if). `strict: true` skips it, so prepare rejects disallowed JS features
+  // and the pure canonical subset is enforced. subscript handles ASI natively.
+  if (!opts.strict) ctx.transform.jzify = jzify
   if (opts.noTailCall) ctx.transform.noTailCall = true
   if (opts.strict) ctx.transform.strict = true
   if (opts.host) {
@@ -319,31 +385,64 @@ const jzCompileInner = (code, opts = {}) => {
   if (opts.alloc === false) ctx.transform.alloc = false
   if (opts.inspect) ctx.transform.inspect = true
   if (opts.importMetaUrl) ctx.transform.importMetaUrl = String(opts.importMetaUrl)
+  if (opts.randomSeed !== undefined) {
+    if (opts.randomSeed !== true && !Number.isFinite(opts.randomSeed))
+      err(`opts.randomSeed must be a finite number (fixed seed — reproducible) or true (seed Math.random from host entropy on first use); got ${typeof opts.randomSeed}`)
+    ctx.transform.randomSeed = opts.randomSeed
+  }
   if (opts.nativeTimers) ctx.features.blockingTimers = true  // wasmtime CLI: include __timer_loop in _start
   ctx.transform.optimize = resolveOptimize(opts.optimize)
-
   if (opts._interp) {
     for (const [name, fn] of Object.entries(opts._interp)) {
-      if (name.startsWith('__ext_')) continue;
+      if (name.startsWith('__ext_')) continue
       if (ctx.transform.host === 'wasi') throw new Error(`host:'wasi' does not support _interp['${name}']: env imports are unavailable in WASI. Implement it natively.`)
       ctx.features.external = true
       const params = Array(fn.length).fill(['param', 'f64'])
       ctx.module.imports.push(['import', '"env"', `"${name}"`, ['func', `$${name}`, ...params, ['result', 'f64']]])
     }
   }
+}
+
+// U+E000 (T) prefixes every jz-generated local. The JS spec forbids it in
+// identifiers, but subscript's parser is lenient and accepts it — so a user name
+// carrying it could silently alias a compiler temp. Reject it in identifier
+// position on the RAW parse (before jzify, which legitimately mints T-prefixed
+// temps of its own). String-literal nodes are `[null, …]` and skipped, so
+// `"……"` data is fine; only walked when the char is present in source.
+const rejectReservedPrefix = (node) => {
+  if (!Array.isArray(node)) return
+  if (node.length === 2 && node[0] == null) return   // [null, X] — value literal, not an identifier
+  for (let i = 1; i < node.length; i++) {
+    const v = node[i]
+    if (typeof v === 'string') {
+      if (v.includes(T)) err(`identifier '${v.split(T).join('\\uE000')}' contains the reserved compiler prefix (U+E000) — jz uses it for generated locals; rename it`)
+    } else rejectReservedPrefix(v)
+  }
+}
+
+const jzCompileInner = (code, opts = {}) => {
+  const profiler = compileProfiler(opts.profile)
+  const time = (name, fn) => profiler ? profiler.time(name, fn) : fn()
+
+  setupCtx(code, opts)
+  assertCtxInvariants('post-reset')
 
   let parsed = time('parse', () => parse(code))
-  if (opts.jzify) parsed = time('jzify', () => jzify(parsed))
+  if (typeof code === 'string' && code.includes(T)) rejectReservedPrefix(parsed)
+  if (!opts.strict) parsed = time('jzify', () => jzify(parsed))
   const ast = time('prepare', () => prepare(parsed))
+  assertCtxInvariants('post-prepare')
 
   // Auto-detect optimization tuning from source characteristics when the user
-  // hasn't provided any optimize option. At the default level its only *live*
-  // output is the typed-array scalarization thresholds (every other override —
-  // watr off, sortStrPool/hoistPtr on — already matches the level-2 preset), so
-  // skip the AST scan entirely unless the program touches typed arrays.
-  // NOTE: if the level-2 default ever flips watr on, drop this guard so the
-  // machine-generated-code heuristic in detectOptimizeConfig can take effect.
-  if (opts.optimize == null && ctx.module.modules.typedarray) {
+  // hasn't provided any optimize option. detectOptimizeConfig has two *live*
+  // overrides over the level-2 preset: the typed-array scalarization thresholds,
+  // and `watr: false` for large/machine-generated code (whose WAT-level CSE/DCE
+  // fights jz's already-optimized IR and inflates output). watr is ON at level 2,
+  // so that switch is a real override — run the scan when the program either
+  // touches typed arrays or is large enough for the machine-code heuristic to
+  // bite. `code.length` is the one signal free without scanning; gating on it
+  // keeps small programs (the common case) on the no-scan fast path.
+  if (opts.optimize == null && (ctx.module.modules.typedarray || code.length > 4000)) {
     const autoCfg = detectOptimizeConfig(ast, code)
     if (Object.keys(autoCfg).length) {
       ctx.transform.optimize = resolveOptimize(autoCfg)
@@ -351,6 +450,7 @@ const jzCompileInner = (code, opts = {}) => {
   }
 
   const module = time('compile', () => compile(ast, profiler))
+  assertCtxInvariants('post-compile')
 
   // host: 'wasi' — error if the wasm would import any env.__ext_* helper. Those exist
   // only to defer to a JS host's value-aware semantics; in a wasmtime/wasmer/deno
@@ -368,24 +468,45 @@ const jzCompileInner = (code, opts = {}) => {
   }
 
   const cfg = ctx.transform.optimize
-  // watr's `loopify` collapses the `(block $brk (loop … (br_if $brk !cond) … (br $loop)))`
-  // idiom into `(loop … (if cond …))` — sound, but destroys the exact shape jz's
-  // post-watr vectorizer scans for. Disable loopify when vectorize is going to run;
-  // otherwise hand watr its full default pass set (inlineOnce + coalesce on, inline off).
-  const watrOpts = cfg.vectorizeLaneLocal ? { loopify: false } : true
-  const optimized = cfg.watr ? time('watrOptimize', () => watrOptimize(module, watrOpts)) : module
-  // Final peephole pass: watrOptimize's inliner can re-introduce rebox/unbox at boundaries
+  let watrOpts = typeof cfg.watr === 'object' ? { ...cfg.watr } : true
+  if (cfg.vectorizeLaneLocal) {
+    if (watrOpts === true) watrOpts = { loopify: false }
+    else if (typeof watrOpts === 'object' && watrOpts.loopify === undefined) watrOpts.loopify = false
+  }
+  if (cfg.devirtIndirect) {
+    if (watrOpts === true) watrOpts = { devirt: true }
+    else if (typeof watrOpts === 'object' && watrOpts.devirt === undefined) watrOpts.devirt = true
+  }
+  const optimized = cfg.watr ? time('watOptimize', () => watOptimize(module, watrOpts)) : module
+  // Stable-pointee module globals: resolve the __ptr_offset once per function.
+  // Never-forwarding kinds — every PTR tag outside __ptr_offset's forwarding
+  // set {ARRAY, HASH, SET, MAP} — give the same offset for the same bits, so
+  // the snapshot only needs the global's VALUE stable through the function:
+  // the reachable-writes call graph proves that precisely. Independent of watr
+  // (the auto-config turns watr off for large sources — exactly the
+  // module-global DSP-state programs this pass exists for: rfft, diffusion);
+  // when watr DID run, it goes before the post leaf passes so the snapped
+  // local participates in hoistAddrBase/cseScalarLoad.
+  if (cfg.hoistGlobalPtrOffset !== false) {
+    const stableGlobals = stablePtrGlobalNames()
+    if (stableGlobals.size) {
+      const funcs = optimized.filter(node => Array.isArray(node) && node[0] === 'func')
+      const reach = collectReachableGlobalWrites(funcs)
+      for (const node of funcs) hoistGlobalPtrOffset(node, stableGlobals, reach)
+    }
+  }
+  // Final peephole pass: watOptimize's inliner can re-introduce rebox/unbox at boundaries
   // (e.g. inlined closure body's `i32.wrap_i64 (i64.reinterpret_f64 __env)` next to caller's
   // `boxPtrIR(g)` rebox). Our fusedRewrite folds these, watr's peephole doesn't.
   // Only valuable to re-run when watr ran (watr is what re-introduces the boundaries).
   if (cfg.watr) {
-    const postCfg = { ...cfg, __phase: 'post' }
     // Build global name→type map from ctx.scope.globalTypes for promoteGlobals
     const globalTypesMap = ctx.scope.globalTypes ? new Map([...ctx.scope.globalTypes].map(([k, v]) => [`$${k}`, v])) : null
     time('watrReopt', () => {
       const funcs = optimized.filter(node => Array.isArray(node) && node[0] === 'func')
       const volatileGlobals = collectVolatileGlobals(funcs)
-      for (const node of funcs) optimizeFunc(node, postCfg, globalTypesMap, volatileGlobals)
+      const reach = collectReachableGlobalWrites(funcs)
+      for (const node of funcs) optimizeFunc(node, cfg, globalTypesMap, volatileGlobals, 'post', reach)
     })
   }
   try {
@@ -395,7 +516,7 @@ const jzCompileInner = (code, opts = {}) => {
     }
     const wasm = time('watrCompile', () => watrCompile(optimized))
     let bytes = wasm
-    if (opts.profileNames || opts.profile?.names) bytes = appendFunctionNames(bytes, optimized)
+    if (opts.profile?.names) bytes = appendFunctionNames(bytes, optimized)
     return opts.inspect ? { wasm: bytes, inspect: ctx.inspect } : bytes
   } catch (e) {
     // watr surfaces dangling identifiers as "Unknown local|func|global|table|memory $X".
@@ -427,6 +548,8 @@ export default function jz(code, ...args) {
 
     // Serialize JS value to jz source literal. Returns null if not serializable.
     const serialize = (v) => {
+      if (v === undefined) return 'undefined'  // else falls through → treated as a
+      // host object → memory.Object(undefined) → Object.keys(undefined) crash
       if (typeof v === 'number' || typeof v === 'boolean') return String(v)
       if (v === null) return 'null'
       if (typeof v === 'string') return JSON.stringify(v)
@@ -484,13 +607,12 @@ export default function jz(code, ...args) {
   // String call: jz('code', opts?) — compile + instantiate + wrap
   const callOpts = args[0] || {}
   const out = jz.compile(code, callOpts)
-  // inspect:true returns { wasm, inspect } from jz.compile — unwrap the bytes for
-  // instantiation and surface inspect on the runtime result alongside exports/memory.
-  if (callOpts.inspect && out && typeof out === 'object' && 'wasm' in out) {
-    const result = instantiateRuntime(out.wasm, callOpts)
-    return Object.assign(result, { inspect: out.inspect })
-  }
-  return instantiateRuntime(out, callOpts)
+  const wasm = out && typeof out === 'object' && 'wasm' in out ? out.wasm : out
+  const result = instantiateRuntime(wasm, callOpts)
+  const extra = {}
+  if (callOpts.inspect && out && typeof out === 'object' && 'inspect' in out) extra.inspect = out.inspect
+  if (callOpts.warnings) extra.warnings = callOpts.warnings.entries
+  return Object.keys(extra).length ? Object.assign(result, extra) : result
 }
 
 export { jz }