jz 0.8.1 → 0.9.0

package/src/optimize/vectorize.js

@@ -533,7 +533,11 @@ const slpStorePairsIn = (node, fnLocals, freshIdRef, newLocalDecls, getCounts) =
   for (let i = 0; i < node.length; i++) {
     const u0 = slpUnitAt(node, i, getCounts)
     if (!u0) continue
-    const u1 = slpUnitAt(node, u0.hi + 1, getCounts)
+    // u1's MATCH index is its store's index, which for the flat tee'd shape is one PAST
+    // its span's lo (the tee'd `local.set` precedes it) — try both hi+1 (inline/block-wrapped,
+    // where lo===hi) and hi+2 (flat tee'd, where the store sits at lo+1) and keep whichever
+    // yields a unit that actually starts right after u0.
+    const u1 = slpUnitAt(node, u0.hi + 1, getCounts) || slpUnitAt(node, u0.hi + 2, getCounts)
     if (!u1 || u1.lo !== u0.hi + 1) continue
     if (u1.off - u0.off !== 8 || !slpSameBase(u0.addr, u1.addr)) continue
     // RAW hazard: the high store's value must not read the low store's target — the pack
@@ -916,21 +920,61 @@ function firstAccess(node, name) {
  *
  * Returns { strideLog2, teeName?: string } or null.
  */
-function matchLaneOffset(off, ind, offsetTees) {
+// The per-iteration element count P (≥2) of an array-of-structs index `P*ind`, or null. Accepts the
+// `(i32.mul P ind)`/`(i32.mul ind P)` form (non-power-of-2 P — RGB's 3) AND the strength-reduced
+// power-of-2 form `(i32.shl ind S)` = ind·2^S (a `const j = 2*i`/`4*i` folds to a shift — complex/
+// RGBA). P is the ELEMENT stride, not a byte offset.
+function matchConstMulIV(node, ind) {
+  if (!isArr(node) || node.length !== 3) return null
+  if (node[0] === 'i32.mul') {
+    let p = null
+    if (isLocalGet(node[1], ind)) p = constNum(node[2])
+    else if (isLocalGet(node[2], ind)) p = constNum(node[1])
+    return (p != null && p >= 2 && p <= 64) ? p : null
+  }
+  if (node[0] === 'i32.shl' && isLocalGet(node[1], ind)) {
+    const s = constNum(node[2])
+    if (s != null && s >= 1 && s <= 6) return 1 << s   // ind·2^S, stride 2..64
+  }
+  return null
+}
+
+function matchLaneOffset(off, ind, offsetTees, allowAos, aosPix, idxTees) {
   if (isArr(off) && off[0] === 'local.get' && typeof off[1] === 'string' &&
       offsetTees && offsetTees.has(off[1])) {
-    return { strideLog2: offsetTees.get(off[1]), teeName: null }
+    return { strideLog2: offsetTees.get(off[1]), pixelStride: (aosPix && aosPix.get(off[1])) || 1, teeName: null }
   }
   let teeName = null
   let n = off
   if (isArr(n) && n[0] === 'local.tee' && n.length === 3) { teeName = n[1]; n = n[2] }
-  // (i32.shl (local.get ind) (i32.const K))
-  if (isArr(n) && n[0] === 'i32.shl' && n.length === 3 && isLocalGet(n[1], ind)) {
+  // (i32.shl <ind-scaled> (i32.const K))
+  if (isArr(n) && n[0] === 'i32.shl' && n.length === 3) {
     const k = constNum(n[2])
-    if (k != null && k >= 0 && k <= 3) return { strideLog2: k, teeName }
+    if (k != null && k >= 0 && k <= 3) {
+      if (isLocalGet(n[1], ind)) return { strideLog2: k, pixelStride: 1, teeName }
+      // AoS (array-of-structs / interleaved channels), P elements per iteration, element
+      // byte-size 1<<K. Only under allowAos (tryVectorize gathers/scatters the lanes); every
+      // other recognizer returns null here. Two shapes: the folded `(i32.mul P ind)`, and a
+      // pixel-index local `(local.get $J)` with $J = P*ind (the `const j = P*i`, via idxTees).
+      if (allowAos) {
+        const p = matchConstMulIV(n[1], ind)
+        if (p != null) return { strideLog2: k, pixelStride: p, teeName }
+        if (isArr(n[1]) && n[1][0] === 'local.get' && idxTees && idxTees.has(n[1][1])) {
+          const pj = idxTees.get(n[1][1])
+          if (pj > 1) return { strideLog2: k, pixelStride: pj, teeName }
+        }
+      }
+    }
+    // POWER-OF-2 AoS stride: `a[P*i]` (P = 2,4,8 — complex/RGBA/…) folds `(P*i)<<3` to a single
+    // `(i32.shl ind K)` with K = 3 + log2(P) > 3. The excess shift over the f64 element (3) is the
+    // pixel stride. f64 lane only — scanForLoadsStores' `1<<strideLog2 === elemSize` gate keeps
+    // strideLog2=3 (=8 bytes), so a folded narrower-lane stride can't be mis-accepted here.
+    else if (allowAos && k != null && k >= 4 && k <= 9 && isLocalGet(n[1], ind)) {
+      return { strideLog2: 3, pixelStride: 1 << (k - 3), teeName }
+    }
   }
   // (local.get ind) — stride 1
-  if (isLocalGet(n, ind)) return { strideLog2: 0, teeName }
+  if (isLocalGet(n, ind)) return { strideLog2: 0, pixelStride: 1, teeName }
   return null
 }
 
@@ -944,13 +988,13 @@ function matchLaneOffset(off, ind, offsetTees) {
  *   `strideLog2` = K for i32.shl form, 0 for plain add form.
  *   `base` is the loop-invariant base subtree.
  */
-function matchLaneAddr(addr, ind, addrLocals, offsetTees) {
+function matchLaneAddr(addr, ind, addrLocals, offsetTees, allowAos, aosPix, idxTees) {
   let teeName = null
   let n = addr
   // (local.get $A) where $A holds a previously-tee'd FULL lane-address.
   if (isArr(n) && n[0] === 'local.get' && typeof n[1] === 'string' && addrLocals && addrLocals.has(n[1])) {
     const e = addrLocals.get(n[1])
-    return { strideLog2: e.strideLog2, base: e.base, teeName: null, viaLocal: n[1] }
+    return { strideLog2: e.strideLog2, pixelStride: e.pixelStride || 1, base: e.base, teeName: null, viaLocal: n[1] }
   }
   if (isArr(n) && n[0] === 'local.tee' && n.length === 3) {
     teeName = n[1]
@@ -958,9 +1002,9 @@ function matchLaneAddr(addr, ind, addrLocals, offsetTees) {
   }
   if (!isArr(n) || n[0] !== 'i32.add' || n.length !== 3) return null
   const a = n[1], b = n[2]
-  const off = matchLaneOffset(b, ind, offsetTees)
+  const off = matchLaneOffset(b, ind, offsetTees, allowAos, aosPix, idxTees)
   if (!off) return null
-  return { strideLog2: off.strideLog2, base: a, teeName, offsetTeeName: off.teeName }
+  return { strideLog2: off.strideLog2, pixelStride: off.pixelStride || 1, base: a, teeName, offsetTeeName: off.teeName }
 }
 
 /**
@@ -969,17 +1013,30 @@ function matchLaneAddr(addr, ind, addrLocals, offsetTees) {
  * Returns the consistent strideLog2, or null if any write to it diverges.
  * This soundness check backs every `(local.get $T)` resolved via `offsetTees`.
  */
-function _offsetLocalStride(body, name, ind) {
-  let stride = null, found = false, ok = true
+function _offsetLocalStride(body, name, ind, allowAos, idxTees) {
+  let stride = null, found = false, ok = true, pix = null
   function walk(n) {
     if (!isArr(n)) return
     if ((n[0] === 'local.tee' || n[0] === 'local.set') && n[1] === name && n.length === 3) {
       found = true
       const v = n[2]
       let k = null
-      if (isArr(v) && v[0] === 'i32.shl' && v.length === 3 && isLocalGet(v[1], ind)) {
-        k = constNum(v[2])
-        if (k == null || k < 0 || k > 3) ok = false
+      if (isArr(v) && v[0] === 'i32.shl' && v.length === 3) {
+        const kk = constNum(v[2])
+        // Power-of-2 AoS offset local `(i32.shl ind K)` with K>3 — matches matchLaneOffset's
+        // power-of-2 arm: element shift 3, pixel stride 2^(K-3). Verify all writes share one P.
+        if (allowAos && isLocalGet(v[1], ind) && kk != null && kk >= 4 && kk <= 9) {
+          k = 3; const p = 1 << (kk - 3); if (pix == null) pix = p; else if (pix !== p) ok = false
+        }
+        else if (isLocalGet(v[1], ind)) { k = kk; if (k == null || k < 0 || k > 3) ok = false }
+        else if (allowAos && kk != null && kk >= 0 && kk <= 3) {
+          // AoS offset local (i32.shl <P·ind> K) — folded `(i32.mul P ind)` or a pixel-index
+          // local `(local.get $J)` with $J = P·ind (idxTees). Verify every write shares one P.
+          let p = matchConstMulIV(v[1], ind)
+          if (p == null && isArr(v[1]) && v[1][0] === 'local.get' && idxTees && idxTees.get(v[1][1]) > 1) p = idxTees.get(v[1][1])
+          if (p != null) { k = kk; if (pix == null) pix = p; else if (pix !== p) ok = false }
+          else ok = false
+        } else ok = false
       } else if (isLocalGet(v, ind)) {
         k = 0
       } else ok = false
@@ -994,6 +1051,16 @@ function _offsetLocalStride(body, name, ind) {
   return found && ok ? stride : null
 }
 
+// True if the tree contains any branch or return — control flow that a flattened value-block
+// lift can't preserve (an early `br`/`return` out of the block changes which value is produced).
+const hasBranchOrReturn = (node) => {
+  if (!isArr(node)) return false
+  const op = node[0]
+  if (op === 'br' || op === 'br_if' || op === 'br_table' || op === 'return') return true
+  for (let i = 1; i < node.length; i++) if (hasBranchOrReturn(node[i])) return true
+  return false
+}
+
 // True if any node in the tree writes a global. When false for a loop body,
 // every `global.get` inside it is loop-invariant — safe to splat for SIMD.
 const hasGlobalSet = (node) => {
@@ -1044,6 +1111,74 @@ const hasSideEffect = (node) => {
  *   a non-`$__li` preamble, an impure value, or any array content AFTER the loop
  *   bails. When false, ANY non-loop array content in the block bails.
  */
+// A transparent block — no label (first child isn't a `$label` string) and no result — is
+// pure statement grouping: wasm locals are function-scoped, and an unlabeled resultless block
+// is neither a branch target nor a value producer, so it can ONLY appear in statement position
+// (a resultless block in value position is a type error). jz emits one per source statement
+// group; watr's mergeBlocks/vacuum flattens them post-hoc. The vectorizer is jz LOWERING
+// (pre-watr), so it normalizes them itself IN PLACE — splicing each transparent block's
+// children into its parent's statement list — so every recognizer (scaffold-consuming AND
+// raw-node) reads the flat statement lists they were tuned against. Post-order: children are
+// already flat when a block is spliced up. A labeled block (branch target) or result-carrying
+// block (value producer) is kept — including the `(block $brk (loop …))` SIMD scaffold itself.
+function normalizeTransparentBlocks(node) {
+  if (!isArr(node)) return
+  for (let i = 1; i < node.length; i++) normalizeTransparentBlocks(node[i])
+  for (let i = node.length - 1; i >= 1; i--) {
+    const c = node[i]
+    if (isArr(c) && c[0] === 'block' &&
+        !(typeof c[1] === 'string' && c[1].startsWith('$')) &&
+        !(isArr(c[1]) && c[1][0] === 'result'))
+      node.splice(i, 1, ...c.slice(1))
+  }
+}
+
+// Fold the arithmetic identities watr's `identity` pass removes but jz emits raw — most
+// importantly `i<<0` (a byte-stride address for a u8/i8 array: `base + (i << 0)`), which the
+// vectorizer's address matchers, tuned on watr's folded IR, read as bare `i`. Also the trivial
+// `x±0`, `x|0`, `x^0`, `x<<0/>>0`, `x*1`. In place, bottom-up; returns the (possibly folded)
+// node so a parent can rebind. Pure syntactic identities — always sound, watr-equivalent.
+function foldVecIdentities(node) {
+  if (!isArr(node)) return node
+  for (let i = 1; i < node.length; i++) node[i] = foldVecIdentities(node[i])
+  if (node.length !== 3) return node
+  const op = node[0], a = node[1], b = node[2]
+  const ci = (n) => isArr(n) && (n[0] === 'i32.const' || n[0] === 'i64.const') ? Number(n[1]) : NaN
+  const rb = ci(b), ra = ci(a)
+  switch (op) {
+    case 'i32.shl': case 'i32.shr_s': case 'i32.shr_u':
+    case 'i64.shl': case 'i64.shr_s': case 'i64.shr_u':
+      return rb === 0 ? a : node                       // x << 0 = x (right-identity only)
+    case 'i32.add': case 'i32.or': case 'i32.xor':
+    case 'i64.add': case 'i64.or': case 'i64.xor':
+      return rb === 0 ? a : (ra === 0 ? b : node)      // x±0, x|0, x^0 (either side)
+    case 'i32.sub': case 'i64.sub':
+      return rb === 0 ? a : node                       // x - 0 = x
+    case 'i32.mul': case 'i64.mul':
+      return rb === 1 ? a : (ra === 1 ? b : node)      // x*1
+    default: return node
+  }
+}
+
+// Canonicalize jz's `if COND (then (br L))` break-idiom to watr's `br_if L COND` — the shape the
+// loop-scan recognizers (byte-scan, divergent-escape) match. watr's `brif` pass does this
+// post-hoc; the pre-watr vectorizer needs it now. Only the statement-form (no result), no-else,
+// single-`br`-then shape becomes a br_if — anything richer is left untouched for watr. In place,
+// top-down (a converted br_if has no nested `if` to revisit).
+function canonicalizeIfBr(node) {
+  if (!isArr(node)) return
+  for (let i = 1; i < node.length; i++) {
+    const c = node[i]
+    if (isArr(c) && c[0] === 'if' && c.length === 3 &&
+        !(isArr(c[1]) && c[1][0] === 'result') &&
+        isArr(c[2]) && c[2][0] === 'then' && c[2].length === 2 &&
+        isArr(c[2][1]) && c[2][1][0] === 'br' && c[2][1].length === 2 && typeof c[2][1][1] === 'string')
+      // `(br L)` only — a value-carrying `(br L v)` would lose its operand under br_if's 2-arg form.
+      node[i] = ['br_if', c[2][1][1], c[1]]
+    else canonicalizeIfBr(c)
+  }
+}
+
 function matchBlockLoop(blockNode, opts = {}) {
   if (!isArr(blockNode) || blockNode[0] !== 'block') return null
   const allowPreamble = !!opts.allowPreamble
@@ -1104,7 +1239,7 @@ function matchBlockLoop(blockNode, opts = {}) {
  * Try to vectorize the inner loop. Returns the replacement node array
  * (synthetic outer block) or null on no match.
  */
-function tryVectorize(bl, fnLocals, freshIdRef) {
+function tryVectorize(bl, fnLocals, freshIdRef, pureFuncMap, constLocals) {
   // Consumes the shared scaffold descriptor (matchBlockLoop, computed once by the
   // dispatch). The LICM `$__li` preamble is cloned ahead of the SIMD block; each
   // set is pure & loop-invariant, so the kept scalar tail harmlessly re-runs it.
@@ -1125,6 +1260,28 @@ function tryVectorize(bl, fnLocals, freshIdRef) {
   // Offset tees: name → strideLog2. A CSE'd `i << K` shared across base
   // pointers (map loops over distinct arrays). Soundness re-checked post-scan.
   const offsetTees = new Map()
+  // AoS (array-of-structs) de-interleave: this recognizer alone accepts a pixel-stride
+  // access `base[P*i + c]` (interleaved RGB/vec3/complex). allowAos enables it in every
+  // shared matcher; aosPix carries P for a CSE'd offset tee; aosPixelStride is the loop's
+  // single P (1 = plain stride-1, unchanged). Set once, verified equal across all sites.
+  const allowAos = true
+  const aosPix = new Map()
+  let aosPixelStride = 1
+  // Pixel-INDEX locals: `$J = P*i` (the `const j = 3*i` of an AoS loop, kept as its own local
+  // pre-watr). A channel address is then `base + ((local.get $J) << K)`; idxTees lets
+  // matchLaneOffset resolve $J → pixel-stride P. Value -1 marks an inconsistent local (bail).
+  const idxTees = new Map()
+  {
+    const walk = (n) => {
+      if (!isArr(n)) return
+      if ((n[0] === 'local.set' || n[0] === 'local.tee') && typeof n[1] === 'string' && n.length === 3) {
+        const p = matchConstMulIV(n[2], incVar)
+        if (p != null) idxTees.set(n[1], idxTees.has(n[1]) && idxTees.get(n[1]) !== p ? -1 : p)
+      }
+      for (let i = 1; i < n.length; i++) walk(n[i])
+    }
+    for (const s of body) walk(s)
+  }
 
   // The compute/lane type is the WIDEST FLOAT among all loads+stores. A narrower
   // float/int LOAD is then a widening read (INT_WIDEN_F32 / f32→f64), a narrower
@@ -1146,6 +1303,26 @@ function tryVectorize(bl, fnLocals, freshIdRef) {
   for (const s of body) scanFloatWidth(s)
   if (preFloat) { laneType = preFloat; stride = LANE_INFO[preFloat].stride }
 
+  // Record a memory site's pixel stride. An AoS stride (P>1) is f64-lane only (the gather/scatter
+  // lifts 2 f64 lanes). Strides are collected for the post-scan uniformity gate: EVERY site must
+  // share one stride, else the loop mixes stride-1 and stride-P accesses (e.g. AoS-struct loads
+  // feeding stride-1 array stores) and a single gather/scatter delta would corrupt the odd sites.
+  const siteStrides = []
+  const recordAos = (m) => {
+    const ps = m.pixelStride || 1
+    if (ps > 1) {
+      if (laneType !== 'f64') return false
+      if (aosPixelStride === 1) aosPixelStride = ps
+      if (m.offsetTeeName) aosPix.set(m.offsetTeeName, ps)
+    }
+    siteStrides.push(ps)
+    return true
+  }
+  // The real address is node[1], unless a folded `offset=N` memarg precedes it (node[1] is the
+  // string `offset=N`, node[2] the address) — the AoS channels `d[j+1]`,`d[j+2]` and stencil
+  // neighbours arrive this way.
+  const memAddr = (node) => (typeof node[1] === 'string' && node[1].startsWith('offset=')) ? node[2] : node[1]
+
   function scanForLoadsStores(node, parent, pi) {
     if (!isArr(node)) return true
     const op = node[0]
@@ -1162,10 +1339,11 @@ function tryVectorize(bl, fnLocals, freshIdRef) {
       } else if (lt !== laneType && !widenInt) {
         return false
       }
-      const m = matchLaneAddr(node[1], incVar, addrLocals, offsetTees)
+      const m = matchLaneAddr(memAddr(node), incVar, addrLocals, offsetTees, allowAos, aosPix, idxTees)
       if (!m) return false
       if ((1 << m.strideLog2) !== (widenInt ? LANE_INFO[lt].stride : stride)) return false
-      if (m.teeName) addrLocals.set(m.teeName, { strideLog2: m.strideLog2, base: m.base })
+      if (!recordAos(m)) return false
+      if (m.teeName) addrLocals.set(m.teeName, { strideLog2: m.strideLog2, pixelStride: m.pixelStride, base: m.base })
       if (m.offsetTeeName) offsetTees.set(m.offsetTeeName, m.strideLog2)
       loadStoreSites.push({ parent, idx: pi, kind: 'load' })
       return true
@@ -1178,27 +1356,30 @@ function tryVectorize(bl, fnLocals, freshIdRef) {
       const narrowing = laneType != null && sty !== laneType && isNarrowStore(laneType, sty)
       if (laneType != null && sty !== laneType && !narrowing) return false
       if (laneType == null) { laneType = sty; stride = LANE_INFO[laneType].stride }
-      const m = matchLaneAddr(node[1], incVar, addrLocals, offsetTees)
+      const memarg = typeof node[1] === 'string' && node[1].startsWith('offset=')
+      const m = matchLaneAddr(memAddr(node), incVar, addrLocals, offsetTees, allowAos, aosPix, idxTees)
       if (!m) return false
       if ((1 << m.strideLog2) !== (narrowing ? LANE_INFO[sty].stride : stride)) return false
-      if (m.teeName) addrLocals.set(m.teeName, { strideLog2: m.strideLog2, base: m.base })
+      if (!recordAos(m)) return false
+      if (m.teeName) addrLocals.set(m.teeName, { strideLog2: m.strideLog2, pixelStride: m.pixelStride, base: m.base })
       if (m.offsetTeeName) offsetTees.set(m.offsetTeeName, m.strideLog2)
       loadStoreSites.push({ parent, idx: pi, kind: 'store' })
-      // Recurse into VALUE child (idx 2) — it's data, not address.
-      if (!scanForLoadsStores(node[2], node, 2)) return false
+      // Recurse into VALUE child (idx 2, or 3 past an offset= memarg) — it's data, not address.
+      const valIdx = memarg ? 3 : 2
+      if (!scanForLoadsStores(node[valIdx], node, valIdx)) return false
       return true
     }
     // local.set/tee of an address local outside a load/store context (e.g.
     // `(local.set $a (i32.add base (i32.shl i 2)))` as a standalone stmt) —
     // record so a later `(local.get $a)` resolves.
     if ((op === 'local.set' || op === 'local.tee') && typeof node[1] === 'string' && node.length === 3) {
-      const valM = matchLaneAddr(['local.tee', node[1], node[2]], incVar, addrLocals, offsetTees)
+      const valM = matchLaneAddr(['local.tee', node[1], node[2]], incVar, addrLocals, offsetTees, allowAos, aosPix, idxTees)
       if (valM && valM.teeName) {
-        addrLocals.set(valM.teeName, { strideLog2: valM.strideLog2, base: valM.base })
+        addrLocals.set(valM.teeName, { strideLog2: valM.strideLog2, pixelStride: valM.pixelStride, base: valM.base })
       }
-      // Standalone offset compute: `(local.set $t (i32.shl i K))`.
-      const offM = matchLaneOffset(node[2], incVar, offsetTees)
-      if (offM) offsetTees.set(node[1], offM.strideLog2)
+      // Standalone offset compute: `(local.set $t (i32.shl i K))` (or AoS `(i32.shl (mul P i) K)`).
+      const offM = matchLaneOffset(node[2], incVar, offsetTees, allowAos, aosPix, idxTees)
+      if (offM) { offsetTees.set(node[1], offM.strideLog2); if (offM.pixelStride > 1) aosPix.set(node[1], offM.pixelStride) }
     }
     // Recurse into all children
     for (let i = 1; i < node.length; i++) {
@@ -1212,11 +1393,14 @@ function tryVectorize(bl, fnLocals, freshIdRef) {
   if (body.some(hasGlobalSet)) return null  // a global write breaks the "global.get is invariant" splat
   if (!laneType) return null  // no memory ops — vectorizing buys nothing
   if (loadStoreSites.length === 0) return null
+  // Uniform stride gate: an AoS loop must have EVERY load/store at the same pixel stride. A mix of
+  // stride-1 and stride-P sites can't share one lift stride — bail (stays scalar, always correct).
+  if (aosPixelStride > 1 && siteStrides.some(s => s !== aosPixelStride)) return null
 
   // Soundness gate for offset-tee resolution: every `(local.get $T)` we
   // accepted as `i << K` is only valid if EVERY write of $T is that offset.
   for (const [name, k] of offsetTees) {
-    if (_offsetLocalStride(body, name, incVar) !== k) return null
+    if (_offsetLocalStride(body, name, incVar, allowAos, idxTees) !== k) return null
   }
 
   // Classify all locals referenced in body.
@@ -1252,7 +1436,7 @@ function tryVectorize(bl, fnLocals, freshIdRef) {
       // Discriminate lane-data vs address-tee. Address tees hold i32 addresses,
       // not vector data. We classify by checking the local's declared type.
       const decl = fnLocals.get(name)
-      if (decl === 'i32' && (offsetTees.has(name) || _isAddressLocal(body, name, incVar))) {
+      if (decl === 'i32' && (addrLocals.has(name) || offsetTees.has(name) || _isAddressLocal(body, name, incVar) || _isPixelIndexLocal(body, name, incVar))) {
         localKind.set(name, 'addr')
       } else {
         localKind.set(name, 'lane')
@@ -1285,10 +1469,43 @@ function tryVectorize(bl, fnLocals, freshIdRef) {
     if (dropped.size) body2 = inlined.filter(s => !dropped.has(s))
   }
 
+  // A signum ternary (`a<0?-1:1`) is commonly CSE'd into its own lane-local just before its
+  // sole use (`set $s (select (i32.const -1)(i32.const 1) COND); … f64.convert_i32_s($s) …`),
+  // hiding it from liftExprV's `f64.convert_i32_s(select …)` fusion (below), which only matches
+  // the select INLINE. Post-watr this local hop would already be copy-propagated away; pre-watr
+  // it survives. When such a lane-local is read exactly once, and that read is inside a
+  // `f64.convert_i32_s`, inline the select back — same "sink a single-use def into its sole
+  // specialized consumer" trick as the narrowing-store case above.
+  {
+    const getCount2 = new Map()
+    const countGets2 = (n) => { if (!isArr(n)) return; if (n[0] === 'local.get' && typeof n[1] === 'string') getCount2.set(n[1], (getCount2.get(n[1]) || 0) + 1); for (let i = 1; i < n.length; i++) countGets2(n[i]) }
+    for (const s of body2) countGets2(s)
+    const dropDefs = new Set()
+    const inlineSign = (n) => {
+      if (!isArr(n)) return n
+      if (n[0] === 'f64.convert_i32_s' && n.length === 2 && isArr(n[1]) && n[1][0] === 'local.get' &&
+          typeof n[1][1] === 'string' && getCount2.get(n[1][1]) === 1) {
+        const nm = n[1][1]
+        const def = body2.find(x => isArr(x) && x[0] === 'local.set' && x[1] === nm && x.length === 3)
+        if (def && isArr(def[2]) && def[2][0] === 'select' && def[2].length === 4 && isI32Const(def[2][1]) && isI32Const(def[2][2])) {
+          dropDefs.add(def)
+          return ['f64.convert_i32_s', def[2]]
+        }
+      }
+      return n.map((c, i) => i === 0 ? c : inlineSign(c))
+    }
+    const inlined2 = body2.map(inlineSign)
+    // Two-pass: inlineSign allocates a fresh array for every node it visits (even unchanged
+    // ones), so filtering `inlined2` by `dropDefs.has(...)` would fail on reference identity —
+    // `dropDefs` holds references into the PRE-map `body2`, so the drop-filter must run against
+    // `body2` (matching indices into `inlined2`), not against the mapped output.
+    if (dropDefs.size) body2 = body2.map((s, i) => dropDefs.has(s) ? null : inlined2[i]).filter(s => s != null)
+  }
+
   // Build lifted body. If anything fails to lift, bail.
   const newLanedLocals = new Map()  // origName → laneName (bare string; see getOrAllocLanedLocal)
   const extraLocals = []  // canon temps allocated during lift
-  const ctx = { laneType, incVar, rampVar: null, rampTemp: null, widenLoads: false, localKind, fnLocals, newLanedLocals, extraLocals, freshIdRef, fail: false, failReason: null }
+  const ctx = { laneType, incVar, rampVar: null, rampTemp: null, widenLoads: false, localKind, fnLocals, newLanedLocals, extraLocals, freshIdRef, fail: false, failReason: null, aosPixelStride, pureFuncMap, inlineDepth: 0, constLocals }
   const lifted = []
   for (const s of body2) {
     const r = liftStmt(s, ctx)
@@ -2175,6 +2392,24 @@ function _isAddressLocal(body, name, ind) {
   return foundTee && onlyAsAddrTee
 }
 
+// A pixel-INDEX local — every write is `(i32.mul P ind)` — is the `const j = P*i` of an
+// AoS loop (feeds channel addresses `base + ((j+c)<<K)`). Classified as 'addr' so the lift
+// keeps it a recomputed scalar i32, never a v128 lane. (Only tryVectorize consults this.)
+function _isPixelIndexLocal(body, name, ind) {
+  let found = false, ok = true
+  function walk(n) {
+    if (!isArr(n)) return
+    if ((n[0] === 'local.set' || n[0] === 'local.tee') && n[1] === name && n.length === 3) {
+      found = true
+      if (matchConstMulIV(n[2], ind) == null) ok = false
+      return
+    }
+    for (let i = 1; i < n.length; i++) walk(n[i])
+  }
+  for (const s of body) walk(s)
+  return found && ok
+}
+
 // ---- Lifter ----------------------------------------------------------------
 
 // Returns the v128 lane-local NAME (a string) for `name`, allocating once. We store the bare
@@ -2190,6 +2425,217 @@ function getOrAllocLanedLocal(name, newLanedLocals) {
   return laneName
 }
 
+// AoS de-interleave gather/scatter (ctx.aosPixelStride P > 1). The SIMD block steps the IV
+// by `lanes`, so a scalar address `A` points at pixel i, channel c; pixel i+1's same channel
+// is P elements = P*elemSize bytes further — reachable as a static load/store `offset`.
+// aosAddrPair yields two address forms that evaluate `A` exactly ONCE (teeing when needed).
+function aosAddrPair(addr, ctx) {
+  if (isArr(addr) && addr[0] === 'local.get') return { a0: addr, a1: addr }               // live local — read twice, free
+  if (isArr(addr) && addr[0] === 'local.tee' && addr.length === 3) return { a0: addr, a1: ['local.get', addr[1]] }
+  const g = `$__aosa${ctx.freshIdRef.next++}`                                              // bare expr — tee into a scratch
+  ctx.extraLocals.push(['local', g, 'i32'])
+  return { a0: ['local.tee', g, addr], a1: ['local.get', g] }
+}
+const aosLoad = (off, addr) => off ? ['f64.load', `offset=${off}`, addr] : ['f64.load', addr]
+const aosStore = (off, addr, val) => off ? ['f64.store', `offset=${off}`, addr, val] : ['f64.store', addr, val]
+
+// A scalar `(f64.load [offset=X] A)` → the f64x2 [pixel i chan, pixel i+1 chan]. Bit-exact:
+// the two lanes are the exact bytes the two scalar iterations read.
+function aosGather(expr, ctx) {
+  const delta = ctx.aosPixelStride * LANE_INFO.f64.stride
+  let baseOff = 0, addr
+  if (typeof expr[1] === 'string' && expr[1].startsWith('offset=')) { baseOff = parseInt(expr[1].slice(7)) || 0; addr = expr[2] }
+  else addr = expr[1]
+  const { a0, a1 } = aosAddrPair(addr, ctx)
+  return ['f64x2.replace_lane', 1, ['f64x2.splat', aosLoad(baseOff, a0)], aosLoad(baseOff + delta, a1)]
+}
+
+// Inline a PURE user function call `(call $f ARG…)` into a single scalar value-BLOCK, feeding
+// the result back through liftExprV so the callee's ternaries/compares/transcendentals lift via
+// the SAME machinery (no separate restricted inliner). Bails (null) on any non-value statement
+// (store/loop/impure) — only straight-line pure helpers (spow, a signed-power, …) inline.
+//
+// Every argument AND every callee local is bound ONCE to a fresh block-local; param/local reads
+// substitute to `(local.get bind)`. This is critical for NESTED calls (spow whose ratio arg is
+// used 3× and itself nests spow): naive expr substitution would duplicate each arg per use and
+// blow up exponentially (there is no CSE pass after the 'post' vectorizer). Binding keeps the
+// SIMD body the same size as the scalar call graph.
+// Infer the wasm type of a value node — from its `.type` expando (jz stamps every instruction) or
+// the op prefix (`f64.add`→f64, `i32.mul`→i32, v128 ops→v128). Used to declare inline temps.
+function nodeWasmType(n) {
+  if (isArr(n)) {
+    if (typeof n.type === 'string') return n.type
+    const op = n[0]
+    if (typeof op === 'string') {
+      if (op.startsWith('f64.') || op === 'f64x2.extract_lane') return 'f64'
+      if (op.startsWith('f32.')) return 'f32'
+      if (op.startsWith('i64.')) return 'i64'
+      if (op.startsWith('i32.')) return 'i32'
+      if (op.startsWith('f64x2.') || op.startsWith('f32x4.') || op.startsWith('i32x4.') || op.startsWith('i8x16.') || op.startsWith('i16x8.') || op.startsWith('i64x2.') || op.startsWith('v128.')) return 'v128'
+    }
+  }
+  return null
+}
+
+// Inline a pure function call into an expression, returning a `(block (result T) …binds… value)`
+// (or the bare value if no binding was needed) — or null if the callee isn't straight-line pure.
+// `resultType` is the callee's result type ('f64' by default, the vectorizer's only use). When a
+// `localSink` array is passed, the fresh `$__ia` binding temps are declared into it (`['local', n, T]`)
+// so a general caller can hoist them into the enclosing function; the vectorizer omits it (its lane
+// lift re-types the block). Params must be read-only (else the substitution model breaks).
+export function inlinePureCallExpr(callNode, pureFuncMap, freshIdRef, localSink = null, resultType = 'f64', tempPrefix = '$__ia') {
+  const callee = pureFuncMap && pureFuncMap.get(callNode[1])
+  if (!callee) return null
+  const bodyStart = findBodyStart(callee)
+  if (bodyStart < 0) return null
+  const params = [], paramType = new Map(), localType = new Map()
+  for (let i = 2; i < bodyStart; i++) {
+    const d = callee[i]
+    if (isArr(d) && d[0] === 'param' && typeof d[1] === 'string') { params.push(d[1]); paramType.set(d[1], d[2]) }
+    else if (isArr(d) && d[0] === 'local' && typeof d[1] === 'string') localType.set(d[1], d[2])
+  }
+  const args = callNode.slice(2)
+  if (args.length !== params.length) return null
+  const body = callee.slice(bodyStart)
+  for (const p of params) if (writesName(body, p)) return null   // params must be read-only
+  const subst = new Map()
+  // Callee-local RENAMING (general-inliner path, localSink passed): a callee local
+  // reached via `local.tee` / control-flow `local.set` isn't captured by bindOnce,
+  // so its NAME would collide with same-named caller locals (the canonical trap:
+  // arrow `(x,k)=>…` inlined into a caller whose variable is also `x`). Rename at
+  // substitution time — sub() returns substituted caller-arg nodes WHOLE without
+  // descending, so a rename can never touch a caller node. A tee/set of a name
+  // bindOnce already substituted means reads and writes diverged — bail (broken).
+  let broken = false
+  // Caller-origin subtrees injected by substitution, tracked by node IDENTITY: the
+  // leak backstop must skip them — a caller local legitimately named like a callee
+  // local (`x`/`k` args into an `(x,k)=>…` arrow) is not a leak.
+  const injected = new Set()
+  const renames = localSink ? new Map() : null
+  const renameOf = (name) => {
+    let r = renames.get(name)
+    if (!r) {
+      r = `${tempPrefix}${freshIdRef.next++}`
+      renames.set(name, r)
+      localSink.push(['local', r, localType.get(name) || 'f64'])
+    }
+    return r
+  }
+  const sub = (n) => {
+    if (!isArr(n)) return n
+    if (n[0] === 'local.get' && typeof n[1] === 'string') {
+      if (subst.has(n[1])) { const v = subst.get(n[1]); injected.add(v); return v }
+      if (renames && localType.has(n[1])) return ['local.get', renameOf(n[1])]
+    }
+    if ((n[0] === 'local.set' || n[0] === 'local.tee') && typeof n[1] === 'string') {
+      if (subst.has(n[1])) { broken = true; return n }
+      if (renames && localType.has(n[1])) return [n[0], renameOf(n[1]), ...n.slice(2).map(sub)]
+    }
+    return n.map((c, i) => i === 0 ? c : sub(c))
+  }
+  // A constant / bare local read is free to duplicate — substitute it directly (no binding),
+  // which also keeps a constant exponent literal at the `pow` node so it can lower to 2-wide exp∘log.
+  // convert-of-local too: one op over a register read, and keeping the convert SYNTACTIC at
+  // every use is what lets the trunc∘convert / guard-vs-impossible-const identities fire
+  // (the devirt arm-inline spills i32 args in that exact shape).
+  const isTrivial = (n) => isArr(n) && (n[0] === 'f64.const' || n[0] === 'i32.const' ||
+    (n[0] === 'local.get' && typeof n[1] === 'string') || (n[0] === 'global.get' && typeof n[1] === 'string') ||
+    (n[0] === 'f64.convert_i32_s' && isArr(n[1]) && n[1][0] === 'local.get'))
+  const pre = []
+  const bindOnce = (name, valueExpr, declType, alreadySubbed) => {
+    const v = alreadySubbed ? valueExpr : sub(valueExpr)
+    if (isTrivial(v)) { subst.set(name, v); return }   // cheap → substitute directly, no temp
+    const bn = `${tempPrefix}${freshIdRef.next++}`
+    const t = declType || nodeWasmType(v) || 'f64'
+    if (localSink) localSink.push(['local', bn, t])
+    pre.push(['local.set', bn, v])
+    subst.set(name, ['local.get', bn])
+  }
+  params.forEach((p, i) => bindOnce(p, args[i], paramType.get(p), true))   // args live in the OUTER scope — do NOT sub
+  // Leak guard: a callee local reached only via `local.tee` (a CSE'd subexpression) or set inside
+  // control flow is NOT captured by the top-level bindOnce, so its name would survive into the caller
+  // where it isn't declared ("$x not in scope"). For the general inliner (localSink passed): RENAME
+  // any surviving TRUE-local name to a fresh caller-scope local declared into the sink — sound,
+  // locals are function-scoped names (a tee'd NaN-guard local `(x,k)=>(x??0)|0` is the canonical
+  // shape). Params are read-only and fully substituted by bindOnce, so a surviving PARAM name means
+  // the model broke — bail (keep the call) as the backstop. The VECTORIZER path (localSink == null)
+  // re-processes the returned expression in its lane context — a tee'd callee local becomes a lane
+  // local there — so it must NOT bail or rename, or pure helpers with a CSE'd tee (spow's `av`)
+  // stop vectorizing.
+  const calleeLocals = new Set([...paramType.keys(), ...localType.keys()])
+  // Backstop: with renaming inlined into sub(), the only way a callee name survives
+  // into a sink-spliced result is a broken substitution model (e.g. a param name in
+  // write position, or a bindOnce'd local later tee'd). Bail — keep the call. The
+  // VECTORIZER path (localSink == null) neither renames nor bails: it re-processes
+  // the expression in its lane context, where a tee'd callee local becomes a lane
+  // local — bailing there would stop pure helpers with a CSE'd tee (spow's `av`)
+  // from vectorizing.
+  const leaks = (n) => isArr(n) && !injected.has(n) &&
+    (((n[0] === 'local.get' || n[0] === 'local.set' || n[0] === 'local.tee') && calleeLocals.has(n[1])) || n.some((c, i) => i > 0 && leaks(c)))
+  const wrap = (val) => {
+    const r = pre.length ? ['block', ['result', resultType], ...pre, val] : val
+    return (localSink && (broken || leaks(r))) ? null : r
+  }
+  for (let k = 0; k < body.length; k++) {
+    const stmt = body[k]
+    if (!isArr(stmt)) return null
+    if (stmt[0] === 'local.set' && typeof stmt[1] === 'string' && stmt.length === 3) { bindOnce(stmt[1], stmt[2], localType.get(stmt[1]), false); continue }
+    if (stmt[0] === 'return' && stmt.length === 2) return wrap(sub(stmt[1]))
+    // Trailing value expression = implicit return (a bare `if`/`block`/… as the function's last
+    // statement, `(v) => cond ? a : b`). Earlier non-set/non-return statements can't be values.
+    if (k === body.length - 1) return wrap(sub(stmt))
+    return null
+  }
+  return null
+}
+
+// Statement-position containers: a call that is a DIRECT child here may be a statement (void /
+// block-fallthrough), where the value-producing `(block (result T) …)` inline form is ill-typed.
+// The general inliner only rewrites calls in operand (value) position — the common `x = f(…)`,
+// `a[i] = f(…)`, `f(…) * k` shapes — and recurses into these so nested-in-operand calls still inline.
+const INLINE_STMT_CTX = new Set(['block', 'loop', 'func', 'then', 'else', 'if'])
+
+// General pre-watr pure-function inlining — jz LOWERING (runs before the vectorizer). Replaces a
+// `(call $g …)` in value position with $g's inlined body when $g is PURE (pureFuncMap) and
+// straight-line. jz decides by PURITY + TYPES — knowledge watr's untyped, size-gated inliner lacks —
+// exposing the callee's arithmetic to the vectorizer / narrower / const-folder. watr keeps only the
+// mechanical residual. Bit-exact: params are read-only, args bind once (or substitute if trivial),
+// the callee's straight-line body becomes a result-typed block. Fresh temps are declared into `fn`.
+export function inlinePureFnsInFn(fn, pureFuncMap, freshIdRef, canInline) {
+  if (!isArr(fn) || fn[0] !== 'func' || !pureFuncMap || !pureFuncMap.size || !canInline || !canInline.size) return
+  const selfName = fn[1]
+  const bodyStart = findBodyStart(fn)
+  if (bodyStart < 0) return
+  const newLocals = []
+  const resultTypeOf = (callee) => {
+    for (let i = 2; i < callee.length; i++) {
+      const d = callee[i]
+      if (!isArr(d)) break
+      if (d[0] === 'result') return d[1]
+      if (d[0] !== 'param' && d[0] !== 'export' && d[0] !== 'local' && d[0] !== 'type') break
+    }
+    return 'f64'
+  }
+  const walk = (node) => {
+    if (!isArr(node)) return node
+    const parentIsStmt = INLINE_STMT_CTX.has(node[0])
+    for (let i = 1; i < node.length; i++) {
+      let child = node[i]
+      if (!isArr(child)) continue
+      child = walk(child)          // recurse first → inline nested calls (e.g. in this call's args)
+      node[i] = child
+      if (!parentIsStmt && child[0] === 'call' && typeof child[1] === 'string' &&
+          child[1] !== selfName && canInline.has(child[1]) && pureFuncMap.has(child[1])) {
+        const inlined = inlinePureCallExpr(child, pureFuncMap, freshIdRef, newLocals, resultTypeOf(pureFuncMap.get(child[1])), '$__gi')
+        if (inlined != null) node[i] = inlined
+      }
+    }
+    return node
+  }
+  for (let i = bodyStart; i < fn.length; i++) fn[i] = walk(fn[i])
+  if (newLocals.length) fn.splice(bodyStart, 0, ...newLocals)
+}
+
 // Wrap an already-lifted v128 value `coreV` in per-lane NaN canonicalization:
 //   v128.bitselect(splat(C), coreV, laneNe(coreV, coreV))
 // coreV is referenced three times. When it's a bare local.get (the common
@@ -2229,6 +2675,10 @@ let _whyNotReason = null
 // on unconditionally. Armed for the duration of a vectorizeLaneLocal call.
 let _relaxF32 = false
 
+// optimize.crPow, armed the same way — the const-exponent pow arm picks its lowering
+// from it (lift ctx objects don't carry the optimize config; module flag is the pattern).
+let _crPow = false
+
 // Mark a lift bail and record its reason. First-write-wins: the innermost failing op
 // sets ctx.failReason; outer frames see ctx.fail already set and return without
 // overwriting, so the reason names the actual blocking op, not a wrapper.
@@ -2320,7 +2770,10 @@ function liftStmt(stmt, ctx) {
       // Address-only local: lift the value as-is (it's i32 arithmetic on ind).
       return ['local.set', name, stmt[2]]
     }
-    if (kind === 'lane') {
+    // 'lane', or an UNCLASSIFIED local — which can only be one introduced by an inlined pure
+    // callee (classification covers every original body local; a pure helper's temps are fresh
+    // per-iteration lane values, never loop-carried). Both lift as lane data.
+    if (kind === 'lane' || kind === undefined) {
       const laneName = getOrAllocLanedLocal(name, ctx.newLanedLocals)
       const v = liftExprV(stmt[2], ctx)
       if (ctx.fail) return null
@@ -2330,11 +2783,30 @@ function liftStmt(stmt, ctx) {
   }
 
   if (STORE_OPS[op]) {
+    const sty = STORE_OPS[op]
+    // AoS de-interleave scatter: `(f64.store [offset=X] A V)` → tee the f64x2 V once, then
+    // write lane 0 at X (pixel i) and lane 1 at X + P*elemSize (pixel i+1) — the exact two
+    // scalar stores. Handles the folded `offset=` memarg form (channels d[j+1], d[j+2]).
+    if (ctx.aosPixelStride > 1) {
+      if (sty !== ctx.laneType) return liftFail(ctx, 'AoS narrowing store unsupported')
+      let baseOff = 0, addr, val
+      if (typeof stmt[1] === 'string' && stmt[1].startsWith('offset=')) { baseOff = parseInt(stmt[1].slice(7)) || 0; addr = stmt[2]; val = stmt[3] }
+      else { addr = stmt[1]; val = stmt[2] }
+      const v = liftExprV(val, ctx)
+      if (ctx.fail) return null
+      const delta = ctx.aosPixelStride * LANE_INFO.f64.stride
+      const { a0, a1 } = aosAddrPair(addr, ctx)
+      const vt = `$__aosv${ctx.freshIdRef.next++}`
+      ctx.extraLocals.push(['local', vt, 'v128'])
+      return ['__seq__',
+        ['local.set', vt, v],
+        aosStore(baseOff, a0, ['f64x2.extract_lane', 0, ['local.get', vt]]),
+        aosStore(baseOff + delta, a1, ['f64x2.extract_lane', 1, ['local.get', vt]])]
+    }
     const addr = stmt[1]  // we leave addresses as-is (scalar i32 expressions)
     // Handle memarg if present (last positional after addr/val): unlikely in
     // pre-watr IR for this shape; bail if more than 3 children.
     if (stmt.length !== 3) return liftFail(ctx, `${op} with memarg`)
-    const sty = STORE_OPS[op]
     // Narrowing store: a narrower element written from a wider float lane (`o[i] =
     // narrow(f(x))` — codec encode / downsample). The scalar store value carries a
     // conversion (f32.demote_f64, or the float→int ToInt32 idiom); peel it, lift the
@@ -2494,6 +2966,9 @@ function liftExprV(expr, ctx) {
   // Loads → v128.load (preserving address, including any local.tee).
   if (LOAD_OPS[op]) {
     if (LOAD_OPS[op] !== ctx.laneType) return liftFail(ctx, `${op}: load type ≠ lane type ${ctx.laneType}`)
+    // AoS de-interleave: consecutive elements are DIFFERENT channels, so a plain v128.load
+    // would mix channels — gather the same channel of pixels i, i+1 into the f64x2 instead.
+    if (ctx.aosPixelStride > 1) return aosGather(expr, ctx)
     // memarg form `(T.load offset=N addr)` — the stencil neighbour `a[i+1]` jz folds
     // onto `a[i]`'s address tee. `v128.load offset=N` reads the N-byte-shifted vector,
     // i.e. the (a[i+1], a[i+2]) pair — exactly the δ-shifted lane data. Preserve it.
@@ -2536,7 +3011,22 @@ function liftExprV(expr, ctx) {
     if (kind === 'addr' || name === ctx.incVar) {
       return liftFail(ctx, `${name}: address/induction var used as lane data`)
     }
-    return liftFail(ctx, `${name}: unclassified local in value position`)
+    // Unclassified (undefined) & not the IV/addr: a local introduced by an inlined pure callee —
+    // read its lane shadow (the matching lane-set default above allocated it).
+    return ['local.get', getOrAllocLanedLocal(name, ctx.newLanedLocals)]
+  }
+
+  // `(local.tee $x V)` in value position — a CSE temp inside a value expression (e.g. the base
+  // teed for reuse in an inlined `x**(k/5)` fifthroot / a repeated subexpression). Lift V into the
+  // lane shadow of $x and tee it: later `(local.get $x)` reads resolve to the same shadow.
+  if (op === 'local.tee' && typeof expr[1] === 'string' && expr.length === 3) {
+    const name = expr[1]
+    const kind = ctx.localKind.get(name)
+    if (kind === 'lane' || kind === undefined) {
+      const v = liftExprV(expr[2], ctx); if (ctx.fail) return null
+      return ['local.tee', getOrAllocLanedLocal(name, ctx.newLanedLocals), v]
+    }
+    return liftFail(ctx, `local.tee ${name}: non-lane local in value position`)
   }
 
   // Loop-invariant global (e.g. a hoistConstantPool'd const, or any global the
@@ -2553,6 +3043,24 @@ function liftExprV(expr, ctx) {
     const inner = expr[1]
     const inv = isArr(inner) && (inner[0] === 'global.get' || (inner[0] === 'local.get' && ctx.localKind.get(inner[1]) === 'invariant'))
     if (inv) return [info.splat, expr]
+    // Sign / small-int ternary `cond ? A : B` (A,B integer literals) lowered to
+    // `(f64.convert_i32_s (select (i32.const A) (i32.const B) COND))` — e.g. `a<0 ? -1 : 1`.
+    // Convert the literals to f64 and bitselect by COND's lane mask (the f64-lane ternary).
+    if (ctx.laneType === 'f64' && isArr(inner) && inner[0] === 'select' && inner.length === 4 &&
+        isI32Const(inner[1]) && isI32Const(inner[2])) {
+      let cond = inner[3]
+      if (isArr(cond) && cond[0] === 'i32.ne' && isI32Const(cond[2]) && cond[2][1] === 0) cond = cond[1]
+      const cmpS = isArr(cond) && cond.length === 3 ? LANE_COMPARE.f64?.[cond[0]] : null
+      if (cmpS) {
+        const ca = liftExprV(cond[1], ctx); if (ctx.fail) return null
+        const cb = liftExprV(cond[2], ctx); if (ctx.fail) return null
+        const mtmp = `$__mask${ctx.freshIdRef.next++}`
+        ctx.extraLocals.push(['local', mtmp, 'v128'])
+        return ['block', ['result', 'v128'],
+          ['local.set', mtmp, [cmpS, ca, cb]],
+          ['v128.bitselect', ['f64x2.splat', ['f64.const', inner[1][1]]], ['f64x2.splat', ['f64.const', inner[2][1]]], ['local.get', mtmp]]]
+      }
+    }
   }
 
   // NaN-canonicalization wrapper (float lanes only; integer lanes never carry
@@ -2594,9 +3102,24 @@ function liftExprV(expr, ctx) {
   }
   if ((ctx.laneType === 'f64' || ctx.laneType === 'f32') && op === 'block') {
     const m = matchCanonBlock(expr, ctx.laneType)
-    if (!m) return liftFail(ctx, 'non-canonical value-block')
-    const coreV = liftExprV(m.core, ctx)
-    return ctx.fail ? null : liftCanon(coreV, m.C, ctx, info)
+    if (m) { const coreV = liftExprV(m.core, ctx); return ctx.fail ? null : liftCanon(coreV, m.C, ctx, info) }
+    // General value-block (a let-binding): `(block [label] (result T) …laneSets… TAILVALUE)`.
+    // jz emits these for an inlined value function (e.g. `av ** e` → an exp∘log block). Lift the
+    // intermediate lane-local sets, then the tail value. Sound ONLY when the block is straight-line
+    // (no br/br_if/br_table/return targeting it — an early-exit can't be flattened) — bail otherwise.
+    let bi = 1
+    if (typeof expr[bi] === 'string') bi++
+    if (isArr(expr[bi]) && expr[bi][0] === 'result') bi++
+    const parts = expr.slice(bi)
+    if (parts.length === 0 || parts.some(hasBranchOrReturn)) return liftFail(ctx, 'non-canonical value-block')
+    const out = ['block', ['result', 'v128']]
+    for (let k = 0; k < parts.length - 1; k++) {
+      const l = liftStmt(parts[k], ctx); if (ctx.fail) return null
+      if (l != null) { if (Array.isArray(l) && l[0] === '__seq__') out.push(...l.slice(1)); else out.push(l) }
+    }
+    const tail = liftExprV(parts[parts.length - 1], ctx); if (ctx.fail) return null
+    out.push(tail)
+    return out
   }
 
   // Conditional select — jz lowers `cond ? X : Y` to (if (result LT) COND (then X)
@@ -2615,8 +3138,13 @@ function liftExprV(expr, ctx) {
     const resTy = isArr(expr[1]) && expr[1][0] === 'result' ? expr[1][1] : null
     if (resTy !== ctx.laneType && !(ctx.laneType === 'f32' && resTy === 'f64')) return liftFail(ctx, 'conditional without lane-typed result')
     const thenN = expr[3], elseN = expr[4]
-    if (!isArr(thenN) || thenN[0] !== 'then' || thenN.length !== 2) return liftFail(ctx, 'malformed conditional then-branch')
-    if (!isArr(elseN) || elseN[0] !== 'else' || elseN.length !== 2) return liftFail(ctx, 'malformed conditional else-branch')
+    // A branch is `(then …preludeSets… TAILVALUE)` — usually just the tail (length 2), but jz's
+    // NaN-canonicalization of a negation tees the value first (`(then (set $t (neg a)) (canon $t))`),
+    // so accept intermediate lane-local sets before the tail. Both branches evaluate speculatively
+    // (lane-pure ⇒ trap-free); each tail is snapshotted into its own temp BEFORE the other branch's
+    // prelude runs, so a shared prelude local can't clobber the already-computed value.
+    if (!isArr(thenN) || thenN[0] !== 'then' || thenN.length < 2) return liftFail(ctx, 'malformed conditional then-branch')
+    if (!isArr(elseN) || elseN[0] !== 'else' || elseN.length < 2) return liftFail(ctx, 'malformed conditional else-branch')
     let cond = expr[2]
     if (isArr(cond) && cond[0] === 'i32.ne' && isI32Const(cond[2]) && cond[2][1] === 0) cond = cond[1]  // strip `!= 0`
     // f32 lane: operands were promoted, so the compare is `f64.*` — use its f32x4 form
@@ -2626,13 +3154,28 @@ function liftExprV(expr, ctx) {
     if (!cmpSimd) return liftFail(ctx, `${isArr(cond) ? cond[0] : 'condition'}: not a lane-vectorizable comparison`)
     const ca = liftExprV(cond[1], ctx); if (ctx.fail) return null
     const cb = liftExprV(cond[2], ctx); if (ctx.fail) return null
-    const x = liftExprV(thenN[1], ctx); if (ctx.fail) return null
-    const y = liftExprV(elseN[1], ctx); if (ctx.fail) return null
-    const mtmp = `$__mask${ctx.freshIdRef.next++}`
-    ctx.extraLocals.push(['local', mtmp, 'v128'])
+    // Lift a branch: its prelude sets, then its tail value snapshotted into `outTmp`.
+    const liftArm = (arm, outTmp) => {
+      const out = []
+      for (let i = 1; i < arm.length - 1; i++) {
+        const l = liftStmt(arm[i], ctx); if (ctx.fail) return null
+        if (l != null) { if (Array.isArray(l) && l[0] === '__seq__') out.push(...l.slice(1)); else out.push(l) }
+      }
+      const v = liftExprV(arm[arm.length - 1], ctx); if (ctx.fail) return null
+      out.push(['local.set', outTmp, v])
+      return out
+    }
+    const id = ctx.freshIdRef.next++
+    const tv = `$__then${id}`, ev = `$__else${id}`, mtmp = `$__mask${id}`
+    ctx.extraLocals.push(['local', tv, 'v128'], ['local', ev, 'v128'], ['local', mtmp, 'v128'])
+    // Mask FIRST: COND may carry an address `local.tee` the branch values read, so it must run
+    // before them (matching scalar order — COND evaluates before the taken branch).
+    const maskSet = ['local.set', mtmp, [cmpSimd, ca, cb]]
+    const thenSeq = liftArm(thenN, tv); if (ctx.fail) return null
+    const elseSeq = liftArm(elseN, ev); if (ctx.fail) return null
     return ['block', ['result', 'v128'],
-      ['local.set', mtmp, [cmpSimd, ca, cb]],
-      ['v128.bitselect', x, y, ['local.get', mtmp]]]
+      maskSet, ...thenSeq, ...elseSeq,
+      ['v128.bitselect', ['local.get', tv], ['local.get', ev], ['local.get', mtmp]]]
   }
 
   // Lane-pure op?
@@ -2657,6 +3200,64 @@ function liftExprV(expr, ctx) {
     return [entry.simd, a, b]
   }
 
+  // Transcendental call → its bit-exact f64x2 mirror (pow/exp/log/exp2/sin/cos/atan2/hypot).
+  // f64 lane only (the *2/_v helpers are f64x2). SIMD_PINNED keeps the scalar target alive
+  // through watr's single-caller inlining so the `call` node still exists at lift time.
+  // `$__to_num` is a numeric coercion jz wraps around a helper param it couldn't prove is f64
+  // (e.g. `decode(src[j])`), boxing it via `i64.reinterpret_f64` first. In the lane every value
+  // is already a genuine finite f64, so `__to_num(reinterpret_i64(x)) == x` — lift straight
+  // through, peeling the box round-trip.
+  if (op === 'call' && expr[1] === '$__to_num' && expr.length === 3) {
+    let arg = expr[2]
+    if (isArr(arg) && arg[0] === 'i64.reinterpret_f64' && arg.length === 2) arg = arg[1]
+    return liftExprV(arg, ctx)
+  }
+
+  // `$math.pow(x, c)` with a CONSTANT non-integer exponent, found only during vectorization
+  // (`ctx.constLocals`) — e.g. spow's `av ** nv` after pure-function inlining substitutes the
+  // literal (module/math.js's own `emitPow` const-exponent fold never reaches here: its constant
+  // exponent is known at EMIT time, so it already lowers straight to the scalar const-exponent
+  // path, picked up by the generic PPC_CALL2 lift below). `optimize.crPow` picks the lowering,
+  // mirroring emitPow's own default/crPow split (see the authoritative comment above emitPow):
+  //   OFF (DEFAULT): truly-2-wide `exp_v(c · log_v(x))` — bit-identical to the scalar `$math.pow`
+  //     for EVERY x when c is non-integer (verified: negative base → NaN and x=0 → 0/∞ both carry
+  //     through log/exp identically; only the integer fast path differs, and it is excluded).
+  //   ON: the truly-2-wide correctly-rounded `$math.pow_fold_v` (module/math.js) — the SIMD twin
+  //     of the scalar `$math.pow_fold` (c needs no pre-split; the shared kernel twoProd-splits
+  //     both multiply operands internally — see its own comment). Bit-identical to the scalar
+  //     `$math.pow_fold` for every x — same function, called on both lanes.
+  if (op === 'call' && ctx.laneType === 'f64' && expr[1] === '$math.pow' && expr.length === 4) {
+    const ex = expr[3]
+    let c = null
+    if (isArr(ex) && ex[0] === 'f64.const') c = +ex[1]
+    else if (isArr(ex) && ex[0] === 'local.get' && ctx.constLocals && ctx.constLocals.has(ex[1])) c = ctx.constLocals.get(ex[1])
+    if (c != null && Number.isFinite(c) && !Number.isInteger(c)) {
+      const base = liftExprV(expr[2], ctx); if (ctx.fail) return null
+      if (_crPow) {
+        return ['call', '$math.pow_fold_v', base, ['f64x2.splat', ['f64.const', c]]]
+      }
+      return ['call', '$math.exp_v', ['f64x2.mul', ['f64x2.splat', ex], ['call', '$math.log_v', base]]]
+    }
+  }
+
+  if (op === 'call' && ctx.laneType === 'f64' && PPC_CALL2[expr[1]]) {
+    const args = []
+    for (let i = 2; i < expr.length; i++) { const a = liftExprV(expr[i], ctx); if (ctx.fail) return null; args.push(a) }
+    return ['call', PPC_CALL2[expr[1]], ...args]
+  }
+
+  // Pure user-function call → inline its body as a value-expr and lift that (handles the callee's
+  // ternaries/compares/pow via the arms above). Depth-guarded against pure→pure recursion.
+  if (op === 'call' && ctx.laneType === 'f64' && ctx.pureFuncMap && ctx.pureFuncMap.has(expr[1]) && ctx.inlineDepth < 8) {
+    const inlined = inlinePureCallExpr(expr, ctx.pureFuncMap, ctx.freshIdRef)
+    if (inlined != null) {
+      ctx.inlineDepth++
+      const v = liftExprV(inlined, ctx)
+      ctx.inlineDepth--
+      return v
+    }
+  }
+
   return liftFail(ctx, `${op}: no lane-pure SIMD mapping for ${ctx.laneType}`)
 }
 
@@ -3598,7 +4199,26 @@ function tryBlurMultiPixel(blockNode, fnLocals, freshIdRef) {
   const rBound = ic[2][1][2]
   if (!(isLocalGet(rBound) || isI32Const(rBound))) return null
 
-  // the RGBA store: 4 consecutive i32.store8 at `ab1 + c`, ab1 tee'd in the first.
+  // the RGBA store: 4 i32.store8 at `ab1 + c`, ab1 tee'd in the first. jz's raw pre-watr
+  // emission of `dst[o]=(sr/win)|0` materializes the divide into its own single-use temp
+  // (`tw = sr/win; store(tw)`) — pre-watr propagateSingleUse runs AFTER the vectorizer
+  // (ordered there so it doesn't scramble the dot-pair matcher), so this indirection is
+  // never folded before this recognizer sees it, unlike the old post-watr pipeline where
+  // watr's own copy-prop had already inlined it into the store operand. Resolve that one-hop
+  // indirection here: if a store's value is a bare local.get and the statement immediately
+  // before it is that local's SOLE def in the loop, substitute the def's RHS.
+  const resolvedTemps = new Set()
+  const resolveStoreVal = (idx, val) => {
+    if (!(isArr(val) && val[0] === 'local.get')) return val
+    const t = val[1], prev = loopNode[idx - 1]
+    if (!(isArr(prev) && prev[0] === 'local.set' && prev.length === 3 && prev[1] === t)) return val
+    let uses = 0
+    const count = (n) => { if (!isArr(n)) return; if (n[0] === 'local.get' && n[1] === t) uses++; n.forEach(count) }
+    count(loopNode)
+    if (uses !== 1) return val
+    resolvedTemps.add(t)
+    return prev[2]
+  }
   let storeIdx = -1
   for (let i = innerIdx + 1; i + 3 <= bodyEnd; i++) {
     const s0 = loopNode[i]
@@ -3607,11 +4227,14 @@ function tryBlurMultiPixel(blockNode, fnLocals, freshIdRef) {
   if (storeIdx < 0) return null
   const s0 = loopNode[storeIdx]
   const ab1 = s0[1][1], dstExpr = s0[1][2]      // ab1 local, its address expr
-  const storeVals = [s0[2]]
+  const storeVals = [resolveStoreVal(storeIdx, s0[2])]
+  let scanIdx = storeIdx + 1
   for (let c = 1; c < 4; c++) {
-    const s = loopNode[storeIdx + c]
+    if (isArr(loopNode[scanIdx]) && loopNode[scanIdx][0] === 'local.set') scanIdx++   // skip the next store's div-into-temp
+    const s = loopNode[scanIdx]
     if (!(isArr(s) && s[0] === 'i32.store8' && s[1] === `offset=${c}` && isLocalGet(s[2], ab1))) return null
-    storeVals.push(s[3])
+    storeVals.push(resolveStoreVal(scanIdx, s[3]))
+    scanIdx++
   }
   // each store value must read its accumulator (the divided sum)
   for (let c = 0; c < 4; c++) if (!readsVar(storeVals[c], accInits[c])) return null
@@ -3666,7 +4289,10 @@ function tryBlurMultiPixel(blockNode, fnLocals, freshIdRef) {
   const newInnerBlock = innerBlock.map(c => c === innerLoop ? newInnerLoop : c)
   // store epilogue: the o=(row+x)<<2 setup, then ab1, then per pixel j set acc_c to
   // its lane and reuse the scalar store template at byte offset j*4+c.
-  const preStore = loopNode.slice(innerIdx + 1, storeIdx)   // `o = (row+x)<<2` etc.
+  // `o = (row+x)<<2` etc. — drop resolved-away div-into-temp defs (unused now; storeVals
+  // carries their RHS directly), else a dead `tw15 = sr/win` reading a stale `sr` gets
+  // carried into the 4-pixel loop (harmless — unused — but unnecessary bytes/work).
+  const preStore = loopNode.slice(innerIdx + 1, storeIdx).filter(s => !(isArr(s) && s[0] === 'local.set' && resolvedTemps.has(s[1])))
   const epilogue = [...preStore, ['local.set', ab1, dstExpr]]
   for (let j = 0; j < 4; j++) {
     const vec = j < 2 ? accLo : accHi, base = (j % 2) * 4
@@ -4127,22 +4753,49 @@ function tryDivergentEscapeVectorize(blockNode, fnLocals, freshIdRef) {
   const outcomeVarSetEarly = new Set(midBreaks.flatMap(mb => mb.assigns.map(a => a.name)))
   const carriedInit = new Map()   // carried var → its f64.const seed (before the loop)
   const perPxInit = new Map()     // c-var → its per-pixel init expr
+  // Pre-watr, jz's raw lowering hasn't run DCE yet — obody[0..innerIdx) may hold pre-loop f64
+  // locals the OLD post-watr recognizer never saw (watr had already deleted them). Two live
+  // shapes beyond direct escape-loop reads (cVars, already populated by the liftable() walk
+  // above): (a) truly dead — a per-pixel grid coord the update ignores (Julia fixed-c: cx/cy
+  // computed but the orbit never reads them — mandelbrot's dual DOES read them); safe to drop.
+  // (b) a carried var's z0 seed reads it INDIRECTLY through one extra local (Julia/Newton
+  // per-pixel z0: x0 = <ramp>; zx = x0) — cVars only sees reads INSIDE the escape loop, so x0
+  // is invisible to it even though it IS the real seed expression. Close that gap with a
+  // needed-set fixpoint before classifying, so x0 promotes to a c-var exactly like a directly
+  // read grid coord — liftCLane already resolves a cVar reference at emit.
+  const epilogue = obody.slice(innerIdx + 1)
+  const preStmts = []
   for (let i = 0; i < innerIdx; i++) {
     const s = obody[i]
     if (!isArr(s) || s[0] !== 'local.set' || s.length !== 3) return null
-    const tgt = s[1]
+    preStmts.push({ tgt: s[1], expr: s[2] })
+  }
+  const preTgt = new Set(preStmts.map(p => p.tgt))
+  // A pre-loop local promotes to a c-var only if some OTHER classified site actually reads it —
+  // never one of the reserved roles (carried/temp/itVar/outcome), which already have their own
+  // per-lane home and must keep taking their existing branch below.
+  const promotable = (v) => !carried.has(v) && !temp.has(v) && v !== itVar && !outcomeVarSetEarly.has(v)
+  const needed = new Set(cVars)
+  for (const { tgt, expr } of preStmts) if (carried.has(tgt)) for (const v of preTgt) if (promotable(v) && readsVar(expr, v)) needed.add(v)
+  for (const e of epilogue) for (const v of preTgt) if (promotable(v) && readsVar(e, v)) needed.add(v)
+  for (let changed = true; changed;) {
+    changed = false
+    for (const { tgt, expr } of preStmts) if (needed.has(tgt))
+      for (const v of preTgt) if (v !== tgt && promotable(v) && !needed.has(v) && readsVar(expr, v)) { needed.add(v); changed = true }
+  }
+  for (const { tgt, expr } of preStmts) {
     if (carried.has(tgt)) {
       // z₀ seed: a constant (mandelbrot/burning-ship z₀=0) OR a per-pixel expr (Julia set, where
       // z₀ = the pixel and c is constant — the dual of mandelbrot). liftCLane handles both at emit.
-      carriedInit.set(tgt, s[2])
+      carriedInit.set(tgt, expr)
     } else if (temp.has(tgt)) { /* recomputed each iteration — init ignored */ }
-    else if (tgt === itVar) { if (constNum(s[2]) !== 0) return null }
-    else if (cVars.has(tgt)) perPxInit.set(tgt, s[2])
+    else if (tgt === itVar) { if (constNum(expr) !== 0) return null }
+    else if (needed.has(tgt)) { cVars.add(tgt); perPxInit.set(tgt, expr) }
     else if (outcomeVarSetEarly.has(tgt)) { /* i32 outcome var default init — ignored, handled per-lane */ }
+    else if (!hasSideEffect(expr)) { /* dead per-pixel local — the escape update never reads it, drop */ }
     else return null
   }
   for (const c of carried) if (!carriedInit.has(c)) return null
-  const epilogue = obody.slice(innerIdx + 1)
   // The epilogue runs scalar per lane; it may only read carried/it/pixel-IV/invariant
   // values (each statement's reads, before that statement's writes). A read of an
   // inner-loop temp or a per-pixel c-var has no post-loop per-lane value → bail.
@@ -4468,15 +5121,20 @@ const PPC_CALL2 = {
   '$math.sin': '$math.sin2', '$math.cos': '$math.cos2',
   '$math.pow': '$math.pow2',   // 2-arg; bit-exact per-lane scalar (cancellation-sensitive — see module/math.js)
   '$math.atan2': '$math.atan2_2', '$math.hypot': '$math.hypot_2',   // 2-arg; bit-exact extract/repack
+  '$math.cbrt': '$math.cbrt_v', '$math.fifthroot': '$math.fifthroot_v',   // 1-arg; per-lane scalar repack
+  '$math.pow_fold': '$math.pow_fold_v',   // 2-arg (x, c); only reachable under optimize.crPow — see module/math.js
   // log/exp/exp2: TRUE f64x2 polys — both lanes one evaluation (≈2×, beats V8 native log). Bit-exact
   // via hot-path-vectorized + scalar-edge-fallback ($math.log_v/exp_v/exp2_v, module/math.js).
   '$math.log': '$math.log_v', '$math.exp': '$math.exp_v', '$math.exp2': '$math.exp2_v',
 }
 
-// Scalar transcendentals the auto-vectorizer rewrites to f64x2 mirrors above. watr's inline
-// passes must NOT dissolve their (single-caller) call nodes before this lift runs — jz passes
-// these to watOptimize's `pin` option (the protection policy lives here, not hardcoded in watr).
-export const SIMD_PINNED = Object.keys(PPC_CALL2)
+// Transcendentals the auto-vectorizer bridges to f64x2 mirrors — BOTH the scalar sources (kept
+// intact in the vectorized loop's scalar tail) AND the f64x2 mirrors themselves (the calls the
+// SIMD path emits). jz passes this to watOptimize's `pin` option so watr's inliner dissolves
+// NEITHER: the scalar tail keeps calling `$math.cbrt` and the SIMD body keeps calling
+// `$math.cbrt_v` (inlining the small per-lane repack mirror would erase the vectorized call the
+// lift produced). The protection policy lives here in jz, not hardcoded in watr.
+export const SIMD_PINNED = [...new Set([...Object.keys(PPC_CALL2), ...Object.values(PPC_CALL2)])]
 
 // Per-pixel-color vectorizer. The dual of tryDivergentEscapeVectorize for kernels with NO inner
 // escape loop: an outer pixel loop whose body computes an f64 value from the pixel index (via
@@ -5208,16 +5866,23 @@ function tryConvColumn(blockNode, fnLocals, freshIdRef, enabled) {
 
   // A byte tap operand: convert_i32_{s,u}(i32.load8_{s,u}(addr)). Returns { load, addr, signed }.
   const matchByteLoad = (n) => {
-    if (!isArr(n) || (n[0] !== 'f64.convert_i32_s' && n[0] !== 'f64.convert_i32_u') || !isArr(n[1])) return null
-    const ld = n[1]
-    if (ld[0] !== 'i32.load8_s' && ld[0] !== 'i32.load8_u') return null
+    // Accept the f64 form (convert_i32_{s,u}(load8)) AND the bare i32 load —
+    // the emit-level convert-peel narrows int8·int8 to i32.mul(load8, load8),
+    // so the taps arrive unconverted (the better shape: no f64 detour to undo).
+    let ld = null
+    if (isArr(n) && (n[0] === 'f64.convert_i32_s' || n[0] === 'f64.convert_i32_u') && isArr(n[1])) ld = n[1]
+    else if (isArr(n) && (n[0] === 'i32.load8_s' || n[0] === 'i32.load8_u')) ld = n
+    if (!ld || (ld[0] !== 'i32.load8_s' && ld[0] !== 'i32.load8_u')) return null
     const addr = (typeof ld[1] === 'string' && ld[1].startsWith('offset=')) ? ld[2] : ld[1]
     return { load: ld, addr, signed: ld[0] === 'i32.load8_s' }
   }
   const load64 = (ld) => (typeof ld[1] === 'string' && ld[1].startsWith('offset=')) ? ['v128.load64_zero', ld[1], ld[2]] : ['v128.load64_zero', ld[1]]
   // Lift a single product addend `inp·wt` (exactly one side gathers on ox) to an i16x8 of 8 products.
   const liftProduct = (prod) => {
-    if (!isArr(prod) || prod[0] !== 'f64.mul') return null
+    // f64.mul(cvt(load), cvt(load)) — pre-peel — or f64.convert_i32_s(i32.mul(
+    // load, load)) / bare i32.mul(load, load) — the peeled faithful product.
+    if (isArr(prod) && prod[0] === 'f64.convert_i32_s' && isArr(prod[1]) && prod[1][0] === 'i32.mul') prod = prod[1]
+    if (!isArr(prod) || (prod[0] !== 'f64.mul' && prod[0] !== 'i32.mul')) return null
     const a = matchByteLoad(prod[1]), b = matchByteLoad(prod[2])
     if (!a || !b) return null
     const ag = isGatherAddr(a.addr), bg = isGatherAddr(b.addr)
@@ -5702,6 +6367,183 @@ function tryToneMap(bl, fnLocals, freshIdRef, enabled) {
   return { wrapper, newLocalDecls }
 }
 
+// ---- Radix-2 butterfly 2-wide lift (tryButterfly) ----
+//
+// The Cooley-Tukey inner loop — the one shape the generic lane lift can never take:
+// a DUAL-IV scaffold (`j++` carries the exit test, `k += STEP` walks the twiddle
+// table) with an in-place complex-rotation update over four disjoint streams
+// (re/im × a/b, b = a + HALF, twiddles wre/wim read-only). Lanes j and j+1:
+//   - every a/b access is an ADJACENT pair → one v128.load / v128.store,
+//   - the twiddle pair is strided by STEP → two scalar f64.loads + lane combine
+//     (same load count as two scalar iterations),
+//   - the +/−/× rotation lanes with NO reassociation and NO fusion — each lane
+//     computes the exact scalar sequence, so the result is bit-identical and the
+//     cross-engine checksum contract holds.
+// LEGALITY. The strip body runs only while j+1 < half, so b = a + half ≥ a + 2:
+// {a,a+1} and {b,b+1} never overlap — the b-pair stores cannot clobber lane 1's
+// a-pair loads, and the single im[a] pair load legitimately serves both the
+// im[b] store and the im[a] writeback (im[a] ∉ {im[b−1], im[b]}). re/im/wre/wim
+// are distinct base locals under the vectorizer's standing distinct-base
+// non-aliasing model. HALF/STEP and the four bases must not be written in the
+// loop (checked); j/k are exactly reproduced for the scalar tail (each strip
+// iteration consumes two scalar iterations), and the tail IS the original loop,
+// so an odd half (or half < 2) falls through untouched.
+function tryButterfly(blockNode, fnLocals, freshIdRef) {
+  if (!isArr(blockNode) || blockNode[0] !== 'block' || typeof blockNode[1] !== 'string') return null
+  const brk = blockNode[1]
+  if (blockNode.length !== 3 || !isArr(blockNode[2]) || blockNode[2][0] !== 'loop') return null
+  const loop = blockNode[2]
+  const lbl = loop[1]
+  if (typeof lbl !== 'string') return null
+  // scaffold: (loop $L (br_if $brk (i32.eqz (i32.lt_s J HALF))) BODY×17 INC 'drop' (br $L))
+  const exit = loop[2]
+  if (!isArr(exit) || exit[0] !== 'br_if' || exit[1] !== brk) return null
+  const ez = exit[2]
+  if (!isArr(ez) || ez[0] !== 'i32.eqz' || !isArr(ez[1]) || ez[1][0] !== 'i32.lt_s') return null
+  const [, jGet, halfGet] = ez[1]
+  if (!isLocalGet(jGet) || !isLocalGet(halfGet)) return null
+  const J = jGet[1], HALF = halfGet[1]
+  const end = loop.length - 1
+  if (!isArr(loop[end]) || loop[end][0] !== 'br' || loop[end][1] !== lbl) return null
+  if (loop[end - 1] !== 'drop') return null
+  // inc: (block (result i32) (drop (i32.sub (local.tee J (i32.add J 1)) 1)) (local.tee K (i32.add K STEP)))
+  const inc = loop[end - 2]
+  if (!isArr(inc) || inc[0] !== 'block' || !isArr(inc[1]) || inc[1][0] !== 'result' || inc.length !== 4) return null
+  const jInc = inc[2], kInc = inc[3]
+  if (!isArr(jInc) || jInc[0] !== 'drop' || !isArr(jInc[1]) || jInc[1][0] !== 'i32.sub') return null
+  const jTee = jInc[1][1]
+  if (!isArr(jTee) || jTee[0] !== 'local.tee' || jTee[1] !== J || !isArr(jTee[2]) || jTee[2][0] !== 'i32.add'
+      || !isLocalGet(jTee[2][1], J) || constNum(jTee[2][2]) !== 1) return null
+  if (!isArr(kInc) || kInc[0] !== 'local.tee' || !isArr(kInc[2]) || kInc[2][0] !== 'i32.add') return null
+  const K = kInc[1]
+  if (typeof K !== 'string' || !isLocalGet(kInc[2][1], K) || !isLocalGet(kInc[2][2])) return null
+  const STEP = kInc[2][2][1]
+  const body = loop.slice(3, end - 2)
+  if (body.length !== 17) return null
+
+  // unification environment over the exact emit shapes
+  const U = {}
+  const bind = (name, v) => U[name] === undefined ? (U[name] = v, true) : U[name] === v
+  const idx8 = (n, base, iv) => isArr(n) && n[0] === 'i32.add'
+    && isLocalGet(n[1]) && bind(base, n[1][1])
+    && isArr(n[2]) && n[2][0] === 'i32.shl' && isLocalGet(n[2][1]) && bind(iv, n[2][1][1])
+    && constNum(n[2][2]) === 3
+  const setF64Load = (st, name, base, iv, ab) => {
+    if (!isArr(st) || st[0] !== 'local.set' || st.length !== 3 || !isArr(st[2]) || st[2][0] !== 'f64.load') return false
+    let addr = st[2][1]
+    if (ab != null) {
+      if (!isArr(addr) || addr[0] !== 'local.tee') return false
+      if (!bind(ab, addr[1])) return false
+      addr = addr[2]
+    }
+    if (!idx8(addr, base, iv)) return false
+    return bind(name, st[1])
+  }
+  const g = (n, name) => isLocalGet(n) && U[name] !== undefined && n[1] === U[name]
+  const mulPair = (n, x, y) => isArr(n) && n[0] === 'f64.mul' && g(n[1], x) && g(n[2], y)
+  const setArith = (st, name, op, mk) => {
+    if (!isArr(st) || st[0] !== 'local.set' || st.length !== 3 || !isArr(st[2]) || st[2][0] !== op) return false
+    if (!mk(st[2])) return false
+    return bind(name, st[1])
+  }
+  // flat pair: (local.set T (op LHS VAL)) ; (f64.store (local.get AB) (local.get T))
+  const storePair = (setSt, stoSt, op, lhs, val2, ab) => {
+    if (!isArr(setSt) || setSt[0] !== 'local.set' || !isArr(setSt[2]) || setSt[2][0] !== op) return false
+    const e = setSt[2]
+    if (!lhs(e[1]) || !g(e[2], val2)) return false
+    if (!isArr(stoSt) || stoSt[0] !== 'f64.store' || !g(stoSt[1], ab) || !isLocalGet(stoSt[2], setSt[1])) return false
+    return true
+  }
+
+  if (!setF64Load(body[0], 'WR', 'WRE', 'K0', null) || U.K0 !== K) return null
+  if (!setF64Load(body[1], 'WI', 'WIM', 'K1', null) || U.K1 !== K) return null
+  {  // a = I + j (either order), I ≠ J
+    const st = body[2]
+    if (!isArr(st) || st[0] !== 'local.set' || !isArr(st[2]) || st[2][0] !== 'i32.add') return null
+    const [, l, r] = st[2]
+    if (isLocalGet(l) && isLocalGet(r, J) && l[1] !== J) U.I = l[1]
+    else if (isLocalGet(r) && isLocalGet(l, J) && r[1] !== J) U.I = r[1]
+    else return null
+    U.A = st[1]
+  }
+  {  // b = a + half (either order)
+    const st = body[3]
+    if (!isArr(st) || st[0] !== 'local.set' || !isArr(st[2]) || st[2][0] !== 'i32.add') return null
+    const [, l, r] = st[2]
+    if (!((isLocalGet(l, U.A) && isLocalGet(r, HALF)) || (isLocalGet(r, U.A) && isLocalGet(l, HALF)))) return null
+    U.B = st[1]
+  }
+  if (!setF64Load(body[4], 'XR', 'RE', 'B0', 'AB4') || U.B0 !== U.B) return null
+  if (!setF64Load(body[5], 'XI', 'IM', 'B1', 'AB5') || U.B1 !== U.B) return null
+  if (!setArith(body[6], 'TR', 'f64.sub', e => mulPair(e[1], 'WR', 'XR') && mulPair(e[2], 'WI', 'XI'))) return null
+  if (!setArith(body[7], 'TI', 'f64.add', e => mulPair(e[1], 'WR', 'XI') && mulPair(e[2], 'WI', 'XR'))) return null
+  if (!setF64Load(body[8], 'C0', 'RE', 'A0', 'AB6') || U.A0 !== U.A) return null
+  const c0lhs = (n) => g(n, 'C0')
+  const ab7teeLhs = (n) => {  // (f64.load (local.tee AB7 (im + a<<3)))
+    if (!isArr(n) || n[0] !== 'f64.load' || !isArr(n[1]) || n[1][0] !== 'local.tee') return false
+    if (!bind('AB7', n[1][1])) return false
+    return idx8(n[1][2], 'IM', 'A2') && U.A2 === U.A
+  }
+  const ab7getLhs = (n) => isArr(n) && n[0] === 'f64.load' && g(n[1], 'AB7')
+  if (!storePair(body[9], body[10], 'f64.sub', c0lhs, 'TR', 'AB4')) return null
+  if (!storePair(body[11], body[12], 'f64.sub', ab7teeLhs, 'TI', 'AB5')) return null
+  if (!storePair(body[13], body[14], 'f64.add', c0lhs, 'TR', 'AB6')) return null
+  if (!storePair(body[15], body[16], 'f64.add', ab7getLhs, 'TI', 'AB7')) return null
+  // loop-invariance: the four bases, HALF/STEP and the outer offset I are never written in the body
+  const invariants = new Set([U.RE, U.IM, U.WRE, U.WIM, HALF, STEP, U.I].filter(x => typeof x === 'string'))
+  if (invariants.size !== 7) return null
+  let clobbered = false
+  const wscan = (n) => { if (clobbered || !isArr(n)) return
+    if ((n[0] === 'local.set' || n[0] === 'local.tee') && invariants.has(n[1])) { clobbered = true; return }
+    for (let i = 1; i < n.length; i++) wscan(n[i]) }
+  for (const st of body) wscan(st)
+  if (clobbered) return null
+  if (new Set([U.RE, U.IM, U.WRE, U.WIM]).size !== 4) return null
+
+  const id = freshIdRef.next++
+  const nm = (t) => `$__bf${id}_${t}`
+  const L = (x) => ['local.get', x]
+  const addr = (base, iv) => ['i32.add', L(base), ['i32.shl', L(iv), ['i32.const', 3]]]
+  const twiddle = (base) => ['f64x2.replace_lane', 1,
+    ['f64x2.splat', ['f64.load', addr(base, K)]],
+    ['f64.load', ['i32.add', L(base), ['i32.shl', ['i32.add', L(K), L(STEP)], ['i32.const', 3]]]]]
+  const wrv = nm('wrv'), wiv = nm('wiv'), xrv = nm('xrv'), xiv = nm('xiv')
+  const trv = nm('trv'), tiv = nm('tiv'), c0v = nm('c0v'), iav = nm('iav')
+  const av = nm('a'), bv = nm('b'), vl = nm('L'), strip = nm('go')
+  const newLocalDecls = [
+    ['local', wrv, 'v128'], ['local', wiv, 'v128'], ['local', xrv, 'v128'], ['local', xiv, 'v128'],
+    ['local', trv, 'v128'], ['local', tiv, 'v128'], ['local', c0v, 'v128'], ['local', iav, 'v128'],
+    ['local', av, 'i32'], ['local', bv, 'i32'],
+  ]
+  const stripGuard = () => ['i32.lt_s', ['i32.add', L(J), ['i32.const', 1]], L(HALF)]
+  const vbody = [
+    ['local.set', wrv, twiddle(U.WRE)],
+    ['local.set', wiv, twiddle(U.WIM)],
+    ['local.set', av, ['i32.add', L(U.I), L(J)]],
+    ['local.set', bv, ['i32.add', L(av), L(HALF)]],
+    ['local.set', xrv, ['v128.load', addr(U.RE, bv)]],
+    ['local.set', xiv, ['v128.load', addr(U.IM, bv)]],
+    ['local.set', trv, ['f64x2.sub', ['f64x2.mul', L(wrv), L(xrv)], ['f64x2.mul', L(wiv), L(xiv)]]],
+    ['local.set', tiv, ['f64x2.add', ['f64x2.mul', L(wrv), L(xiv)], ['f64x2.mul', L(wiv), L(xrv)]]],
+    ['local.set', c0v, ['v128.load', addr(U.RE, av)]],
+    ['v128.store', addr(U.RE, bv), ['f64x2.sub', L(c0v), L(trv)]],
+    ['local.set', iav, ['v128.load', addr(U.IM, av)]],
+    ['v128.store', addr(U.IM, bv), ['f64x2.sub', L(iav), L(tiv)]],
+    ['v128.store', addr(U.RE, av), ['f64x2.add', L(c0v), L(trv)]],
+    ['v128.store', addr(U.IM, av), ['f64x2.add', L(iav), L(tiv)]],
+    ['local.set', J, ['i32.add', L(J), ['i32.const', 2]]],
+    ['local.set', K, ['i32.add', L(K), ['i32.add', L(STEP), L(STEP)]]],
+    ['br_if', vl, stripGuard()],
+  ]
+  const wrapper = ['block',
+    ['block', strip,
+      ['br_if', strip, ['i32.eqz', stripGuard()]],
+      ['loop', vl, ...vbody]],
+    blockNode]  // the ORIGINAL loop is the scalar tail: 0..1 leftover iterations, or everything when half < 2
+  return { wrapper, newLocalDecls }
+}
+
+
 // ---- Pass entry ------------------------------------------------------------
 
 /**
@@ -5713,13 +6555,27 @@ function tryToneMap(bl, fnLocals, freshIdRef, enabled) {
 //   multiAcc, relaxedFma, blurMP, whyNot, stencil, outerStrip, pureFuncMap, toneMap.
 export function vectorizeLaneLocal(fn, opts = {}) {
   const { multiAcc = false, relaxedFma = false, blurMP = true, whyNot = false,
-    stencil = false, outerStrip = false, pureFuncMap = null, toneMap = false, slp = false } = opts
+    stencil = false, outerStrip = false, pureFuncMap = null, toneMap = false, slp = false, crPow = false } = opts
   if (!isArr(fn) || fn[0] !== 'func') return
   const bodyStart = findBodyStart(fn)
   if (bodyStart < 0) return
   const fnName = typeof fn[1] === 'string' ? fn[1] : '(anon)'
   let whyNotN = 0
 
+  // Normalize jz's per-statement `block` grouping into flat statement lists ONCE, up front —
+  // the recognizers below (both the scaffold consumers and the raw-node matchers like ramp-map,
+  // stencil, per-pixel) were tuned on watr's flattened shape. Pre-watr, jz wraps each source
+  // statement group in a transparent block; without this every loop body would arrive as a
+  // single opaque `block` node and no lift would fire. Walking `fn` itself also flattens a
+  // top-level body block (decls never match — they aren't blocks).
+  normalizeTransparentBlocks(fn)
+  // Canonicalize the raw arithmetic identities watr would fold (chiefly `i<<0` byte addresses),
+  // so the address/value matchers read dataflow, not jz's un-folded emission.
+  for (let i = bodyStart; i < fn.length; i++) fn[i] = foldVecIdentities(fn[i])
+  // Canonicalize the `if COND (then (br L))` break idiom to `br_if L COND` (watr's brif shape),
+  // so the loop-scan recognizers see the branch form they were tuned against.
+  canonicalizeIfBr(fn)
+
   // Build local-name → wasm-type map.
   const fnLocals = new Map()
   for (let i = 2; i < bodyStart; i++) {
@@ -5731,8 +6587,31 @@ export function vectorizeLaneLocal(fn, opts = {}) {
     }
   }
 
+  // Loop-invariant constant locals — hoistConstantPool's `$…_pg` pool (`set $L (f64.const C)`,
+  // written exactly once). name → numeric value, used to resolve a constant `pow` exponent that
+  // reached the loop as a pooled local rather than a literal.
+  const constLocals = new Map()
+  {
+    const setCount = new Map()
+    const walkC = (n) => {
+      if (!isArr(n)) return
+      if ((n[0] === 'local.set' || n[0] === 'local.tee') && typeof n[1] === 'string' && n.length === 3) {
+        setCount.set(n[1], (setCount.get(n[1]) || 0) + 1)
+        if (isArr(n[2]) && n[2][0] === 'f64.const') constLocals.set(n[1], +n[2][1])
+      }
+      for (let i = 1; i < n.length; i++) walkC(n[i])
+    }
+    for (let i = bodyStart; i < fn.length; i++) walkC(fn[i])
+    for (const [k, c] of setCount) if (c !== 1) constLocals.delete(k)   // multiply-written → not invariant
+  }
+
   const freshIdRef = { next: 0 }
   const newLocalDeclsAll = []
+  // Whether a REAL SIMD lift happened (as opposed to the scalar tryStrengthReduceIV
+  // fallback below, which also populates newLocalDeclsAll with plain i32 locals) —
+  // the caller pins the function's $name/$name$exp boundary wrapper on this, so a
+  // false positive here would needlessly block watr's inlineOnce on a non-SIMD fn.
+  let simdFired = false
 
   // Hoist loop-invariant partial products out of unrolled dot reductions (rust/LLVM's
   // mat4 prologue trick). Reassociates the float sum, so tied to the relaxedFma tier;
@@ -5743,6 +6622,7 @@ export function vectorizeLaneLocal(fn, opts = {}) {
   // SLP within-iteration store pairs. Sound only with no aliasing typed-array view
   // in the module (else a shifted view could reorder-hazard the packed read/write).
   if (slp && !ctx.features.typedView) slpStorePairsIn(fn, fnLocals, freshIdRef, newLocalDeclsAll, slpGetCounts(fn))
+  if (newLocalDeclsAll.length) simdFired = true
 
   // Walk body recursively. Process inner-most matches first (post-order)
   // so we don't try to vectorize an outer loop whose inner is the lane-local one.
@@ -5769,7 +6649,7 @@ export function vectorizeLaneLocal(fn, opts = {}) {
       const bl = matchBlockLoop(node, { allowPreamble: true, allowInlinedLi: true })
       let r = tryDivergentEscapeVectorize(node, fnLocals, freshIdRef)
         ?? tryMemCopyFill(bl, fnLocals, freshIdRef)
-        ?? tryVectorize(bl, fnLocals, freshIdRef)
+        ?? tryVectorize(bl, fnLocals, freshIdRef, pureFuncMap, constLocals)
         ?? tryReduceVectorize(bl, fnLocals, freshIdRef, multiAcc)
         ?? tryMapReduceVectorize(bl, fnLocals, freshIdRef)
         ?? tryStencil(node, fnLocals, freshIdRef, stencil)
@@ -5782,6 +6662,7 @@ export function vectorizeLaneLocal(fn, opts = {}) {
         ?? tryIteratedReduce(node, fnLocals, freshIdRef, outerStrip)
         ?? tryConvColumn(node, fnLocals, freshIdRef, outerStrip)
         ?? tryToneMap(bl, fnLocals, freshIdRef, toneMap)
+        ?? tryButterfly(node, fnLocals, freshIdRef)
       // --why-not-simd: a canonical loop-shaped candidate that no SIMD pass took.
       // Reported BEFORE the scalar strength-reduce fallback (which fires on most
       // affine loops and would otherwise mask "didn't vectorize"). Diagnostic only.
@@ -5791,19 +6672,42 @@ export function vectorizeLaneLocal(fn, opts = {}) {
           `${fnName}: loop #${whyNotN} not vectorized — ${_whyNotReason || 'no SIMD-liftable shape (loop-carried dependency, non-affine address, or unsupported control flow)'}`,
           { fn: `${fnName}#${whyNotN}` })
       }
-      // Scalar IV strength-reduction is a non-SIMD fallback; it may still fire.
-      r = r ?? tryStrengthReduceIV(bl, fnLocals, freshIdRef)
+      if (r) simdFired = true   // one of the real SIMD recognizers above matched
       if (r) {
+        // Mark the consumed subtree: the wrapper REUSES these nodes (scalar tail, and the
+        // lane splats alias the original load nodes), so a deferred strength-reduce must
+        // never rewrite inside them — it would mutate the lifted lanes through the alias.
+        const mark = (n) => { if (isArr(n)) { srConsumed.add(n); for (let i = 0; i < n.length; i++) mark(n[i]) } }
+        mark(node)
         parent[idx] = r.wrapper
         newLocalDeclsAll.push(...r.newLocalDecls)
+      } else if (bl) {
+        // Scalar IV strength-reduction is a non-SIMD fallback — DEFERRED to after the
+        // whole walk. Applied eagerly here (post-order = innermost first) it rewrites an
+        // inner reduction loop into its wrapper before the ENCLOSING loop's outer
+        // recognizers (outer-strip / iterated-reduce / conv-column / tone-map) ever run,
+        // and they bail on the non-canonical inner shape — metaballs' pixel loop lost its
+        // whole f64x2 outer-strip to an eager strength-reduce of the blob loop.
+        deferredSR.push([node, parent, idx, bl])
       }
     }
   }
+  const deferredSR = [], srConsumed = new Set()
   _whyNotActive = whyNot
   _relaxF32 = relaxedFma
+  _crPow = crPow
   for (let i = bodyStart; i < fn.length; i++) walk(fn, i)
   _whyNotActive = false
   _relaxF32 = false
+  _crPow = false
+  // Apply the deferred scalar fallback innermost-first (push order), skipping candidates
+  // inside any SIMD-consumed subtree (see the mark above), plus a same-slot check for
+  // wrappers that replaced the candidate node itself.
+  for (const [node, parent, idx, bl] of deferredSR) {
+    if (srConsumed.has(node) || parent[idx] !== node) continue
+    const r = tryStrengthReduceIV(bl, fnLocals, freshIdRef)
+    if (r) { parent[idx] = r.wrapper; newLocalDeclsAll.push(...r.newLocalDecls) }
+  }
 
   if (newLocalDeclsAll.length) {
     // Sibling loops (and the straight-line dot pass) can each lift the SAME source
@@ -5815,4 +6719,5 @@ export function vectorizeLaneLocal(fn, opts = {}) {
     // so keep one decl per name (all dups are the identical `['local', name, 'v128']`).
     fn.splice(bodyStart, 0, ...new Map(newLocalDeclsAll.map(d => [d[1], d])).values())
   }
+  return simdFired
 }