jz 0.3.1 → 0.5.0

package/cli.js

@@ -4,13 +4,14 @@
  * JZ CLI - Command-line interface for JZ compiler
  */
 
-import { readFileSync, writeFileSync, existsSync } from 'fs'
-import { dirname, resolve, join } from 'path'
+import { readFileSync, writeFileSync } from 'fs'
+import { resolve } from 'path'
 import { pathToFileURL } from 'url'
-import { execFileSync } from 'child_process'
 import { parse } from 'subscript/feature/jessie'
 import jz, { compile } from './index.js'
-import jzifyFn, { codegen } from './src/jzify.js'
+import jzifyFn from './src/jzify.js'
+import { codegen } from './src/codegen.js'
+import { resolveModuleGraph } from './src/resolve.js'
 import { createRequire } from 'module'
 
 const jzRequire = createRequire(import.meta.url)
@@ -148,53 +149,10 @@ async function handleCompile(args) {
   if (!outputFile) outputFile = inputFile.replace(/\.(js|jz)$/, wat ? '.wat' : '.wasm')
   if (outputFile.endsWith('.wat')) wat = true
 
-  const code = readFileSync(inputFile, 'utf8')
-
-  // Resolve imports
-  const dir = dirname(resolve(inputFile))
-  const modules = {}
-
-  const pkgFile = join(dir, 'package.json')
-  if (existsSync(pkgFile)) {
-    try {
-      const pkg = JSON.parse(readFileSync(pkgFile, 'utf8'))
-      if (pkg.imports) for (const [spec, path] of Object.entries(pkg.imports)) {
-        const full = resolve(dir, path)
-        try { modules[spec] = readFileSync(full, 'utf8') } catch {}
-      }
-    } catch {}
-  }
-
-  // Recursively resolve relative imports from entry file and all discovered modules
-  const importRe = /import\s+.*?\s+from\s+['"]([^'"]+)['"]/g
-  const resolveBareModule = (specifier, fromDir) => execFileSync(
-    process.execPath,
-    ['--input-type=module', '-e', 'process.stdout.write(import.meta.resolve(process.argv[1]))', specifier],
-    { cwd: fromDir, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }
-  ).trim()
-  const resolveModule = (specifier, fromDir) => {
-    if (modules[specifier]) return
-    // Relative imports: resolve from filesystem
-    if (specifier.startsWith('./') || specifier.startsWith('../')) {
-      const full = resolve(fromDir, specifier)
-      let src
-      try { src = readFileSync(full, 'utf8') }
-      catch { try { src = readFileSync(full + '.js', 'utf8') } catch { return } }
-      modules[specifier] = src
-      let m; importRe.lastIndex = 0
-      while ((m = importRe.exec(src)) !== null) resolveModule(m[1], dirname(full))
-      return
-    }
-    // Bare specifiers: opt-in Node.js resolution
-    if (resolveNode) {
-      try {
-        const resolved = resolveBareModule(specifier, fromDir)
-        if (resolved.startsWith('file:')) modules[specifier] = readFileSync(new URL(resolved), 'utf8')
-      } catch {}
-    }
-  }
-  let m; importRe.lastIndex = 0
-  while ((m = importRe.exec(code)) !== null) resolveModule(m[1], dir)
+  // Resolve imports — canonicalize every specifier to an absolute path so the
+  // same physical file always produces one module instance (see src/resolve.js).
+  const { code: codeRewritten, modules } = resolveModuleGraph(inputFile, { resolveNode })
+  if (process.env.JZ_DEBUG_MODULES === '1') console.error('modules:', Object.keys(modules))
 
   // .jz = strict (no auto-transform), .js = auto-jzify
   // --strict forces strict for any extension
@@ -215,7 +173,7 @@ async function handleCompile(args) {
     opts.imports = JSON.parse(readFileSync(importsPath, 'utf8'))
   }
 
-  const result = compile(code, opts)
+  const result = compile(codeRewritten, opts)
 
   if (outputFile === '-') {
     process.stdout.write(result)

package/index.js

@@ -33,7 +33,9 @@
  * Feature flags (ctx.features.*) gate conditional stdlib branches for dead-code elimination.
  * Capability hooks (ctx.schema.register, ctx.closure.make) are installed by capability modules.
  *
- * Interop host layer (memory marshaling, wrap, instantiate) lives in src/host.js.
+ * Interop host layer (memory marshaling, wrap, instantiate) lives in
+ * interop.js — also exported as the standalone `jz/interop` subpath for
+ * hosts that want to run prebuilt jz wasm without pulling the compiler.
  *
  * @module jz
  */
@@ -44,25 +46,17 @@ import { ctx, reset, err } from './src/ctx.js'
 import prepare, { GLOBALS } from './src/prepare.js'
 import compile from './src/compile.js'
 import { emitter } from './src/emit.js'
-import { optimizeFunc, resolveOptimize } from './src/optimize.js'
-import { detectOptimizeConfig } from './src/auto-config.js'
+import { optimizeFunc, collectVolatileGlobals, resolveOptimize } from './src/optimize.js'
 import jzify from './src/jzify.js'
 import {
   memory as enhanceMemory, instantiate as instantiateRuntime,
-} from './src/host.js'
+} from './interop.js'
 
-const importSpecMayReturnExternal = (spec) => {
-  if (typeof spec === 'function') return true
-  return false
-}
-
-const importsMayReturnExternal = (imports) => {
-  if (!imports) return false
-  for (const mod of Object.values(imports))
-    for (const spec of Object.values(mod || {}))
-      if (importSpecMayReturnExternal(spec)) return true
-  return false
-}
+// A host import that's a JS function may hand back any value, including a host
+// object — which arrives in wasm as a PTR.EXTERNAL ref. Constants/typed specs can't.
+const importsMayReturnExternal = (imports) =>
+  !!imports && Object.values(imports).some(mod =>
+    Object.values(mod || {}).some(spec => typeof spec === 'function'))
 
 const nowMs = () => globalThis.performance?.now ? globalThis.performance.now() : Date.now()
 
@@ -171,8 +165,11 @@ jz.memory = enhanceMemory
  * @param {boolean|number|object} [opts.optimize] - Optimization level/config.
  *   - `false` / `0`: nothing. Fastest compile, largest output (live coding).
  *   - `1`: encoding-compactness only (treeshake + sortLocalsByUse + fusedRewrite-inline).
- *   - `true` / `2` (default): all current passes (watr CSE/DCE/inline + every jz pass).
- *   - `3`: reserved for future aggressive passes (currently == 2).
+ *   - `true` / `2` (default): every stable jz pass + full watr (inlineOnce +
+ *     coalesce on; `inline` stays off per watr's own default).
+ *   - `3` / `'speed'`: level 2 + larger array/hash initial caps (`arrayMinCap`,
+ *     `hashSmallInitCap`) + `hoistConstantPool` off (inline `f64.const` over
+ *     mutable globals); trades size for speed.
  *   - `{ level?: 0|1|2|3, watr?: bool, hoistAddrBase?: bool, ... }`: per-pass
  *     overrides on top of the chosen level. See PASS_NAMES in src/optimize.js.
  * @param {object} [opts.profile] - Optional mutable profile sink populated with
@@ -182,9 +179,120 @@ jz.memory = enhanceMemory
  * @param {boolean} [opts.profileNames] - Legacy alias for `profile.names`.
  * @param {string} [opts.importMetaUrl] - Module URL used to lower `import.meta.url`
  *   and static `import.meta.resolve("...")` expressions.
- * @returns {Uint8Array|string}
+ * @param {boolean} [opts.inspect] - When true, return `{ wasm, inspect }`
+ *   (or `{ wat, inspect }` with `opts.wat`) instead of the bare output.
+ *   `inspect` carries per-function inferred shapes (params, locals, JSON shapes,
+ *   cross-call paramReps) for editor hosts to drive inlay hints / hover types
+ *   without re-running the analyzer. Pays a small serialization cost; off by default.
+ * @returns {Uint8Array|string|{wasm: Uint8Array, inspect: object}|{wat: string, inspect: object}}
  */
 jz.compile = (code, opts = {}) => {
+  try {
+    return jzCompileInner(code, opts)
+  } catch (e) {
+    // Any uncaught native exception (TypeError, ReferenceError, etc.) is a jz
+    // codegen leak — surface it as an internal compile error with the source
+    // location we were standing on. err()-thrown errors already pass through.
+    if (e?.name === 'TypeError' || e?.name === 'ReferenceError' || e?.name === 'RangeError') {
+      err(`internal: ${e.message} (jz hit an unsupported case while compiling${ctx.error.node ? '; the AST node above shows the trigger' : ''}). This is a jz bug — please report.`)
+    }
+    throw e
+  }
+}
+
+// =============================================================================
+// Optimization auto-tuning: scan prepared AST + ctx.func.list to infer program
+// properties, then emit per-pass overrides. When the user does not explicitly
+// configure individual passes, the result is merged in before resolveOptimize()
+// so the compiler self-tunes.
+// =============================================================================
+
+const AUTO_CFG_LOOP_OPS = new Set(['for', 'while', 'do', 'do-while'])
+const AUTO_CFG_TYPED_CTORS = new Set([
+  'new.Float32Array', 'new.Float64Array', 'new.Int8Array', 'new.Int16Array',
+  'new.Int32Array', 'new.Uint8Array', 'new.Uint16Array', 'new.Uint32Array',
+  'new.Uint8ClampedArray',
+])
+
+const autoCfgNodeSize = (node) => {
+  if (!Array.isArray(node)) return 1
+  let n = 1
+  for (let i = 1; i < node.length; i++) n += autoCfgNodeSize(node[i])
+  return n
+}
+
+const autoCfgScanNode = (node, stats, loopDepth) => {
+  if (!Array.isArray(node)) return
+  const op = node[0]
+  if (AUTO_CFG_LOOP_OPS.has(op)) {
+    stats.loopCount++
+    const d = loopDepth + 1
+    if (d > stats.maxLoopDepth) stats.maxLoopDepth = d
+    for (let i = 1; i < node.length; i++) autoCfgScanNode(node[i], stats, d)
+    return
+  }
+  if (op === '()') {
+    stats.callSites++
+    const callee = node[1]
+    if (typeof callee === 'string' && AUTO_CFG_TYPED_CTORS.has(callee)) {
+      stats.typedArrayCount++
+      const args = node[2]
+      const argList = args == null ? [] : (Array.isArray(args) && args[0] === ',') ? args.slice(1) : [args]
+      const lenLit = typeof argList[0] === 'number' ? argList[0] : null
+      if (lenLit != null && lenLit > stats.maxTypedArrayLen) stats.maxTypedArrayLen = lenLit
+    }
+  }
+  if (op === 'str') stats.stringLiteralCount++
+  if (op === '=>') stats.closureCount++
+  for (let i = 1; i < node.length; i++) autoCfgScanNode(node[i], stats, loopDepth)
+}
+
+/** Detect optimization config from source characteristics.
+ *  Returns an object of pass overrides; empty object means "use defaults". */
+const detectOptimizeConfig = (ast, code) => {
+  const s = {
+    sourceChars: code?.length || 0,
+    funcCount: 0, maxFuncBodySize: 0,
+    loopCount: 0, maxLoopDepth: 0,
+    typedArrayCount: 0, maxTypedArrayLen: 0,
+    stringLiteralCount: 0, closureCount: 0, callSites: 0,
+  }
+  if (ctx.func?.list) {
+    s.funcCount = ctx.func.list.length
+    for (const f of ctx.func.list) {
+      if (f.body) {
+        const sz = autoCfgNodeSize(f.body)
+        if (sz > s.maxFuncBodySize) s.maxFuncBodySize = sz
+        autoCfgScanNode(f.body, s, 0)
+      }
+    }
+  }
+  if (ast) autoCfgScanNode(ast, s, 0)
+
+  const cfg = {}
+  // Machine-generated or large code: watr's WAT-level CSE/DCE/inline fights
+  // jz's already-optimized IR and inflates output. Disable it automatically.
+  const isLarge = s.sourceChars > 4000 || s.funcCount > 40 || s.maxFuncBodySize > 300
+  const isMachineLike = s.callSites > 300 && s.stringLiteralCount < 10
+  if (isLarge || isMachineLike) cfg.watr = false
+  // Typed-array heavy: tighten scalarization thresholds when we see large
+  // fixed-size arrays; keep defaults for small/dynamic ones.
+  if (s.typedArrayCount > 0 && s.maxTypedArrayLen > 0) {
+    cfg.scalarTypedArrayLen = Math.min(32, Math.max(8, s.maxTypedArrayLen + 4))
+    cfg.scalarTypedLoopUnroll = s.maxLoopDepth > 1 ? 8 : 16
+    cfg.scalarTypedNestedUnroll = s.maxLoopDepth > 1 ? 32 : 128
+  }
+  // String-heavy: ensure pool sorting is on (already default, but explicit).
+  if (s.stringLiteralCount > 30) cfg.sortStrPoolByFreq = true
+  // Closure-heavy: ptr hoists pay off.
+  if (s.closureCount > 4) {
+    cfg.hoistPtrType = true
+    cfg.hoistInvariantPtrOffset = true
+  }
+  return cfg
+}
+
+const jzCompileInner = (code, opts = {}) => {
   const profiler = compileProfiler(opts.profile)
   const time = (name, fn) => profiler ? profiler.time(name, fn) : fn()
 
@@ -204,10 +312,12 @@ jz.compile = (code, opts = {}) => {
   if (opts.noTailCall) ctx.transform.noTailCall = true
   if (opts.strict) ctx.transform.strict = true
   if (opts.host) {
-    if (opts.host !== 'js' && opts.host !== 'wasi') err(`Invalid host '${opts.host}'. Expected 'js' or 'wasi'.`)
+    if (opts.host === 'gc') err(`host:'gc' is reserved for a planned wasm-gc backend, not yet implemented. Use 'js' (default — JS host with externref/js-string interop) or 'wasi' (standalone runtimes — no env imports).`)
+    if (opts.host !== 'js' && opts.host !== 'wasi') err(`Invalid host '${opts.host}'. Expected 'js' (default) or 'wasi'.`)
     ctx.transform.host = opts.host
   }
   if (opts.alloc === false) ctx.transform.alloc = false
+  if (opts.inspect) ctx.transform.inspect = true
   if (opts.importMetaUrl) ctx.transform.importMetaUrl = String(opts.importMetaUrl)
   if (opts.nativeTimers) ctx.features.blockingTimers = true  // wasmtime CLI: include __timer_loop in _start
   ctx.transform.optimize = resolveOptimize(opts.optimize)
@@ -227,8 +337,13 @@ jz.compile = (code, opts = {}) => {
   const ast = time('prepare', () => prepare(parsed))
 
   // Auto-detect optimization tuning from source characteristics when the user
-  // hasn't provided any optimize option.
-  if (opts.optimize == null) {
+  // hasn't provided any optimize option. At the default level its only *live*
+  // output is the typed-array scalarization thresholds (every other override —
+  // watr off, sortStrPool/hoistPtr on — already matches the level-2 preset), so
+  // skip the AST scan entirely unless the program touches typed arrays.
+  // NOTE: if the level-2 default ever flips watr on, drop this guard so the
+  // machine-generated-code heuristic in detectOptimizeConfig can take effect.
+  if (opts.optimize == null && ctx.module.modules.typedarray) {
     const autoCfg = detectOptimizeConfig(ast, code)
     if (Object.keys(autoCfg).length) {
       ctx.transform.optimize = resolveOptimize(autoCfg)
@@ -253,7 +368,12 @@ jz.compile = (code, opts = {}) => {
   }
 
   const cfg = ctx.transform.optimize
-  const optimized = cfg.watr ? time('watrOptimize', () => watrOptimize(module)) : module
+  // watr's `loopify` collapses the `(block $brk (loop … (br_if $brk !cond) … (br $loop)))`
+  // idiom into `(loop … (if cond …))` — sound, but destroys the exact shape jz's
+  // post-watr vectorizer scans for. Disable loopify when vectorize is going to run;
+  // otherwise hand watr its full default pass set (inlineOnce + coalesce on, inline off).
+  const watrOpts = cfg.vectorizeLaneLocal ? { loopify: false } : true
+  const optimized = cfg.watr ? time('watrOptimize', () => watrOptimize(module, watrOpts)) : module
   // Final peephole pass: watrOptimize's inliner can re-introduce rebox/unbox at boundaries
   // (e.g. inlined closure body's `i32.wrap_i64 (i64.reinterpret_f64 __env)` next to caller's
   // `boxPtrIR(g)` rebox). Our fusedRewrite folds these, watr's peephole doesn't.
@@ -263,12 +383,35 @@ jz.compile = (code, opts = {}) => {
     // Build global name→type map from ctx.scope.globalTypes for promoteGlobals
     const globalTypesMap = ctx.scope.globalTypes ? new Map([...ctx.scope.globalTypes].map(([k, v]) => [`$${k}`, v])) : null
     time('watrReopt', () => {
-      for (const node of optimized) if (Array.isArray(node) && node[0] === 'func') optimizeFunc(node, postCfg, globalTypesMap)
+      const funcs = optimized.filter(node => Array.isArray(node) && node[0] === 'func')
+      const volatileGlobals = collectVolatileGlobals(funcs)
+      for (const node of funcs) optimizeFunc(node, postCfg, globalTypesMap, volatileGlobals)
     })
   }
-  if (opts.wat) return time('watrPrint', () => watrPrint(optimized))
-  const wasm = time('watrCompile', () => watrCompile(optimized))
-  return (opts.profileNames || opts.profile?.names) ? appendFunctionNames(wasm, optimized) : wasm
+  try {
+    if (opts.wat) {
+      const wat = time('watrPrint', () => watrPrint(optimized))
+      return opts.inspect ? { wat, inspect: ctx.inspect } : wat
+    }
+    const wasm = time('watrCompile', () => watrCompile(optimized))
+    let bytes = wasm
+    if (opts.profileNames || opts.profile?.names) bytes = appendFunctionNames(bytes, optimized)
+    return opts.inspect ? { wasm: bytes, inspect: ctx.inspect } : bytes
+  } catch (e) {
+    // watr surfaces dangling identifiers as "Unknown local|func|global|table|memory $X".
+    // That's always a jz codegen leak — we emitted IR that references something never
+    // declared (typically: a built-in / stdlib we don't implement). Rewrite to a clean
+    // user-facing message instead of leaking watr internals.
+    const m = /Unknown (local|func|global|table|memory|type) \$?(\S+)/.exec(e?.message || '')
+    if (m) {
+      const [, kind, name] = m
+      const friendly = kind === 'func' ? `'${name}' is not a known function or built-in`
+        : kind === 'global' ? `'${name}' is not a known global or imported binding`
+        : `'${name}' is not in scope`
+      err(`${friendly} — jz emitted a reference it cannot resolve (likely an unsupported built-in or missing import).`)
+    }
+    throw e
+  }
 }
 
 /**
@@ -327,7 +470,8 @@ export default function jz(code, ...args) {
     }
     if (hoisted.length) src = hoisted.join('; ') + '; ' + src
     const hasInterp = Object.keys(interp).length
-    const result = instantiateRuntime(jz.compile, src, { _interp: hasInterp ? interp : null })
+    const tplOpts = { _interp: hasInterp ? interp : null }
+    const result = instantiateRuntime(jz.compile(src, tplOpts), tplOpts)
     // Patch data getters: allocate values in WASM memory, update closure refs
     for (const [, { val, ref }] of Object.entries(data)) {
       if (typeof val === 'string') ref.ptr = result.memory.String(val)
@@ -338,7 +482,15 @@ export default function jz(code, ...args) {
   }
 
   // String call: jz('code', opts?) — compile + instantiate + wrap
-  return instantiateRuntime(jz.compile, code, args[0] || {})
+  const callOpts = args[0] || {}
+  const out = jz.compile(code, callOpts)
+  // inspect:true returns { wasm, inspect } from jz.compile — unwrap the bytes for
+  // instantiation and surface inspect on the runtime result alongside exports/memory.
+  if (callOpts.inspect && out && typeof out === 'object' && 'wasm' in out) {
+    const result = instantiateRuntime(out.wasm, callOpts)
+    return Object.assign(result, { inspect: out.inspect })
+  }
+  return instantiateRuntime(out, callOpts)
 }
 
 export { jz }