jz 0.7.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/README.md +37 -33
  2. package/bench/README.md +176 -73
  3. package/bench/bench.svg +58 -71
  4. package/cli.js +12 -5
  5. package/dist/interop.js +1 -1
  6. package/dist/jz.js +1156 -984
  7. package/index.js +40 -9
  8. package/interop.js +193 -138
  9. package/layout.js +29 -18
  10. package/module/array.js +29 -73
  11. package/module/collection.js +83 -25
  12. package/module/console.js +1 -1
  13. package/module/core.js +161 -15
  14. package/module/json.js +3 -3
  15. package/module/math.js +26 -3
  16. package/module/number.js +247 -13
  17. package/module/object.js +11 -5
  18. package/module/regex.js +8 -7
  19. package/module/string.js +162 -156
  20. package/module/typedarray.js +169 -105
  21. package/package.json +7 -3
  22. package/src/abi/string.js +29 -29
  23. package/src/ast.js +19 -2
  24. package/src/compile/analyze.js +64 -2
  25. package/src/compile/cse-load.js +200 -0
  26. package/src/compile/emit-assign.js +73 -14
  27. package/src/compile/emit.js +253 -23
  28. package/src/compile/index.js +198 -61
  29. package/src/compile/loop-divmod.js +12 -58
  30. package/src/compile/loop-model.js +91 -0
  31. package/src/compile/loop-square.js +102 -0
  32. package/src/compile/narrow.js +169 -32
  33. package/src/compile/peel-stencil.js +18 -64
  34. package/src/compile/plan/common.js +29 -0
  35. package/src/compile/plan/index.js +4 -1
  36. package/src/compile/plan/inline.js +176 -21
  37. package/src/compile/plan/literals.js +93 -19
  38. package/src/ctx.js +51 -12
  39. package/src/helper-counters.js +137 -0
  40. package/src/ir.js +102 -13
  41. package/src/kind-traits.js +7 -3
  42. package/src/kind.js +14 -1
  43. package/src/op-policy.js +5 -2
  44. package/src/ops.js +119 -0
  45. package/src/optimize/index.js +960 -136
  46. package/src/optimize/recurse.js +182 -0
  47. package/src/optimize/vectorize.js +1292 -144
  48. package/src/prepare/index.js +11 -7
  49. package/src/reps.js +4 -1
  50. package/src/type.js +53 -45
  51. package/src/wat/assemble.js +92 -9
  52. package/src/widen.js +21 -0
  53. package/src/wat/optimize.js +0 -3938
@@ -1,5 +1,6 @@
1
1
  import { findBodyStart } from '../ir.js'
2
- import { warn } from '../ctx.js'
2
+ import { warn, ctx } from '../ctx.js'
3
+ import { nodeEqual as exprEq } from '../ast.js'
3
4
 
4
5
  /**
5
6
  * Lane-local SIMD-128 vectorizer.
@@ -48,7 +49,9 @@ import { warn } from '../ctx.js'
48
49
 
49
50
  const isArr = n => Array.isArray(n)
50
51
 
51
- const exprEq = (a, b) => JSON.stringify(a) === JSON.stringify(b)
52
+ // Structural node equality must be non-finite- AND bigint-safe: plain
53
+ // JSON.stringify maps Infinity/-Infinity/NaN→null and -0→0, so it would equate a
54
+ // `[Inf,-Inf]` lane pair and splat it (dropping -Inf). nodeEqual tags those.
52
55
  const localGetName = n => isArr(n) && n[0] === 'local.get' && typeof n[1] === 'string' ? n[1] : null
53
56
  const f64Zero = n => isArr(n) && n[0] === 'f64.const' && Number(n[1]) === 0
54
57
 
@@ -67,9 +70,16 @@ const isSplatConst = (n, constOp) =>
67
70
  function matchCanonSelect(sel, laneType) {
68
71
  if (!isArr(sel) || sel[0] !== 'select') return null
69
72
  const C = sel[1], val = sel[2], cond = sel[3]
70
- const neOp = laneType === 'f32' ? 'f32.ne' : 'f64.ne'
71
- if (!isSplatConst(C, LANE_INFO[laneType].constOp)) return null
72
- if (!(isArr(cond) && cond[0] === neOp && exprEq(cond[1], val) && exprEq(cond[2], val))) return null
73
+ // f32 lane: jz computes the value in f64, so `Math.min/max` (and any NaN-canon'd
74
+ // f32 result) emit the canon with `f64.ne` + an f64 NaN const. Accept that
75
+ // alongside the native f32 form; liftCanon splats the const as f32.
76
+ const f64Canon = laneType === 'f32' && isArr(C) && C[0] === 'f64.const' && isArr(cond) && cond[0] === 'f64.ne'
77
+ if (!f64Canon) {
78
+ const neOp = laneType === 'f32' ? 'f32.ne' : 'f64.ne'
79
+ if (!isSplatConst(C, LANE_INFO[laneType].constOp)) return null
80
+ if (!(isArr(cond) && cond[0] === neOp)) return null
81
+ }
82
+ if (!(exprEq(cond[1], val) && exprEq(cond[2], val))) return null
73
83
  return { val, C }
74
84
  }
75
85
 
@@ -93,10 +103,11 @@ function normTee(n) {
93
103
  // RESULT — equal operands tie to the same value — so only the direction axis matters.
94
104
  function matchIntMinMaxReduce(rhs, accName) {
95
105
  if (!isArr(rhs)) return null
96
- let cond, T, E
106
+ let cond, T, E, resTy = null
97
107
  if (rhs[0] === 'if') {
98
108
  let i = 1
99
- if (!(isArr(rhs[i]) && rhs[i][0] === 'result' && rhs[i][1] === 'i32')) return null
109
+ if (!(isArr(rhs[i]) && rhs[i][0] === 'result' && (rhs[i][1] === 'i32' || rhs[i][1] === 'f64'))) return null
110
+ resTy = rhs[i][1]
100
111
  i++
101
112
  if (rhs.length !== i + 3) return null
102
113
  cond = rhs[i]
@@ -116,8 +127,12 @@ function matchIntMinMaxReduce(rhs, accName) {
116
127
  let cmp = cond
117
128
  if (isArr(cmp) && cmp[0] === 'i32.ne' && isI32Const(cmp[2]) && cmp[2][1] === 0) cmp = cmp[1]
118
129
  if (!isArr(cmp) || cmp.length !== 3) return null
119
- const dir = { 'i32.gt_s': 'gt', 'i32.ge_s': 'gt', 'i32.lt_s': 'lt', 'i32.le_s': 'lt' }[cmp[0]]
130
+ // Integer (i32x4.max_s, exact) or float (f64x2.pmax, exact per-element incl NaN/±0) compare.
131
+ const dir = { 'i32.gt_s': 'gt', 'i32.ge_s': 'gt', 'i32.lt_s': 'lt', 'i32.le_s': 'lt',
132
+ 'f64.gt': 'gt', 'f64.ge': 'gt', 'f64.lt': 'lt', 'f64.le': 'lt' }[cmp[0]]
120
133
  if (!dir) return null
134
+ const laneType = cmp[0].startsWith('f64.') ? 'f64' : 'i32'
135
+ if (resTy != null && resTy !== laneType) return null // if-form result type must agree with the compare
121
136
  // Comparison operands must be {acc, EXPR}; take the non-acc side as the canonical lane
122
137
  // expr (it carries the address tee). exprIsLeftOfCmp records its position.
123
138
  let condExpr, exprIsLeftOfCmp
@@ -128,7 +143,7 @@ function matchIntMinMaxReduce(rhs, accName) {
128
143
  if (!exprEq(normTee(condExpr), normTee(exprBr))) return null
129
144
  // cond true ⟺ EXPR > acc ⇒ picking EXPR-when-true is a max; picking-when-false a min.
130
145
  const predExprGreater = dir === 'gt' ? exprIsLeftOfCmp : !exprIsLeftOfCmp
131
- return { exprNode: condExpr, isMax: takeExprWhenTrue === predExprGreater }
146
+ return { exprNode: condExpr, isMax: takeExprWhenTrue === predExprGreater, laneType }
132
147
  }
133
148
 
134
149
  // Match the un-flattened canon, emitted when a Math.* result feeds another op
@@ -175,19 +190,24 @@ const matchDotStore = (n, acc) => {
175
190
  return null
176
191
  }
177
192
 
193
+ // Unroll width this dot-product recognizer expects: a `acc=0` reset, exactly this
194
+ // many `acc += L[k]*R[k]` steps, then the store. Tied to the emitter's 4-wide dot
195
+ // unroll — matchDotStore / f64x2Pair / dotPairExpr below are hardwired to it.
196
+ const DOT_UNROLL = 4
197
+
178
198
  const matchF64DotSeq = (stmts, i) => {
179
199
  const reset = stmts[i]
180
200
  if (!isArr(reset) || reset[0] !== 'local.set' || typeof reset[1] !== 'string' || !f64Zero(reset[2])) return null
181
201
  const acc = reset[1]
182
202
  const left = [], right = []
183
- for (let k = 0; k < 4; k++) {
203
+ for (let k = 0; k < DOT_UNROLL; k++) {
184
204
  const pair = matchAccumStep(stmts[i + 1 + k], acc)
185
205
  if (!pair) return null
186
206
  left.push(pair[0])
187
207
  right.push(pair[1])
188
208
  }
189
- const store = matchDotStore(stmts[i + 5], acc)
190
- return store ? { end: i + 6, acc, left, right, ...store } : null
209
+ const store = matchDotStore(stmts[i + 1 + DOT_UNROLL], acc)
210
+ return store ? { end: i + 2 + DOT_UNROLL, acc, left, right, ...store } : null
191
211
  }
192
212
 
193
213
  const f64x2Pair = (lo, hi) => ['f64x2.replace_lane', 1, ['f64x2.splat', ['local.get', lo]], ['local.get', hi]]
@@ -243,7 +263,7 @@ const vectorizeStraightLineF64DotPairsIn = (node, fnLocals, freshIdRef, newLocal
243
263
  addend = ['local.get', tmp]
244
264
  }
245
265
  const pairs = []
246
- for (let k = 0; k < 4; k++) {
266
+ for (let k = 0; k < DOT_UNROLL; k++) {
247
267
  const key = `${a.right[k]}\0${b.right[k]}`
248
268
  let tmp = pairTemps.get(key)
249
269
  if (!tmp) {
@@ -267,6 +287,270 @@ const vectorizeStraightLineF64DotPairsIn = (node, fnLocals, freshIdRef, newLocal
267
287
  }
268
288
  }
269
289
 
290
+ // =============================================================================
291
+ // Loop-invariant partial-product hoist for unrolled f64 dot reductions.
292
+ //
293
+ // A fully-unrolled inner reduction over scalar-replaced array cells (mat4's
294
+ // `out[r][c] = Σ a[r][k]·b[k][c]`) lives in the body of an OUTER loop that
295
+ // mutates only a few of those cells (mat4: a[0],a[5],b[0],b[5]). Every product
296
+ // whose two operands are both outer-loop-invariant is therefore the SAME every
297
+ // iteration — yet the body recomputes all of them. rust/LLVM precomputes those
298
+ // invariant partials in a loop prologue (mat4.rs → ~294 lines before its loop);
299
+ // V8/wasmtime/JSC cannot, because at the wasm level they can't prove the cells
300
+ // are loop-invariant (no aliasing model). So jz must hoist them itself.
301
+ //
302
+ // Splitting `s = t0+t1+t2+t3` into `INV = Σ(invariant tk)` (hoisted) + `Σ(variant
303
+ // tk)` (kept) REASSOCIATES the float sum — invariant terms are summed first,
304
+ // regardless of original position — so results differ by ULPs from the strict
305
+ // left-to-right order. That is the SAME class of reorder jz already ships for
306
+ // horizontal/multi-accumulator reductions (policy at lines ~620 and ~1584), and
307
+ // rust itself does it at -O3 without fast-math. Gated to the relaxedFma/speed
308
+ // tier exactly like those, so strict opts keep bit-exact order.
309
+ //
310
+ // Surgical by construction: fires only on a dot that MIXES invariant and variant
311
+ // terms inside a loop. A pure-variant dot (a real matmul kernel, every operand
312
+ // streaming) has no invariant term → untouched. Runs BEFORE the dot-pair
313
+ // vectorizer; a hoisted dot has < DOT_UNROLL accumulate steps so matchF64DotSeq
314
+ // no longer matches it — it stays the (faster here) scalar form, like rust.
315
+ const hoistDotInvariant = (loop, parent, idx, fnLocals, freshIdRef, newLocalDecls) => {
316
+ const writeSet = new Set()
317
+ collectWrites(loop, writeSet)
318
+ const isInv = name => typeof name === 'string' && !writeSet.has(name) && fnLocals.get(name) === 'f64'
319
+ const invInits = []
320
+ const processList = (list) => {
321
+ for (let i = 0; i < list.length;) {
322
+ const seq = matchF64DotSeq(list, i)
323
+ if (!seq) { i++; continue }
324
+ const invKs = [], varKs = []
325
+ for (let k = 0; k < DOT_UNROLL; k++) (isInv(seq.left[k]) && isInv(seq.right[k]) ? invKs : varKs).push(k)
326
+ if (invKs.length === 0) { i = seq.end; continue } // nothing loop-invariant — leave for the vectorizer
327
+ // INV = Σ invariant products, in original k-order, computed once before the loop.
328
+ let inv = ['f64.const', '0']
329
+ for (const k of invKs) inv = ['f64.add', inv, ['f64.mul', ['local.get', seq.left[k]], ['local.get', seq.right[k]]]]
330
+ const invName = `$__rinv_${freshIdRef.next++}`
331
+ newLocalDecls.push(['local', invName, 'f64']); fnLocals.set(invName, 'f64')
332
+ invInits.push(['local.set', invName, inv])
333
+ // In-loop: seed acc with INV, add only the variant products, then the unchanged store.
334
+ const repl = [['local.set', seq.acc, ['local.get', invName]]]
335
+ for (const k of varKs) repl.push(['local.set', seq.acc, ['f64.add', ['local.get', seq.acc], ['f64.mul', ['local.get', seq.left[k]], ['local.get', seq.right[k]]]]])
336
+ repl.push(['local.set', seq.out, seq.addend ? ['f64.add', ['local.get', seq.acc], seq.addend] : ['local.get', seq.acc]])
337
+ list.splice(i, seq.end - i, ...repl)
338
+ i += repl.length
339
+ }
340
+ }
341
+ const scan = (n) => { if (!isArr(n)) return; processList(n); for (let j = 0; j < n.length; j++) if (isArr(n[j])) scan(n[j]) }
342
+ scan(loop)
343
+ if (invInits.length) parent.splice(idx, 0, ...invInits)
344
+ }
345
+
346
+ // Walk a function, hoisting invariant reduction partials out of each loop. Inner
347
+ // loops first (post-order) so a dot is hoisted relative to its tightest enclosing
348
+ // loop, and an already-rewritten dot can't re-match in an outer pass.
349
+ const hoistReductionInvariantsIn = (fn, fnLocals, freshIdRef, newLocalDecls) => {
350
+ const walk = (node, parent, idx) => {
351
+ if (!isArr(node)) return
352
+ for (let i = 0; i < node.length; i++) if (isArr(node[i])) walk(node[i], node, i)
353
+ if (node[0] === 'loop' && isArr(parent)) hoistDotInvariant(node, parent, idx, fnLocals, freshIdRef, newLocalDecls)
354
+ }
355
+ for (let i = 0; i < fn.length; i++) if (isArr(fn[i])) walk(fn[i], fn, i)
356
+ }
357
+
358
+ // =============================================================================
359
+ // SLP (superword-level parallelism): pack two ADJACENT isomorphic f64 element
360
+ // stores into one f64x2 store — the WITHIN-iteration 2-lane class the loop
361
+ // vectorizer (which packs ACROSS iterations) structurally cannot reach.
362
+ //
363
+ // Soundness has TWO obligations, because the pack reorders memory: it materializes
364
+ // BOTH lane values BEFORE either store, turning [read0, write0, read1, write1] into
365
+ // [read0, read1, write0, write1].
366
+ // 1. CROSS-base aliasing — guarded by one module fact: no typed-array VIEW exists
367
+ // (`ctx.features.typedView` false, checked at the dispatch). A view (subarray /
368
+ // buffer-backed ctor) is the only way two DISTINCT typed bases can overlap;
369
+ // without one, distinct bases own disjoint allocations and can't alias.
370
+ // 2. WITHIN-base read-after-write — the high value (read1) must not load the low
371
+ // store's target (write0), or the pack reads write0's PRE-store value. This is a
372
+ // same-base hazard a view gate can't see (`o[k+1]=o[k]; o[k+2]=o[k+1]` forward
373
+ // shift); slpReadsOffset rejects it. The sound own-index map reads its OWN offset,
374
+ // never the sibling's, so it survives.
375
+ // The pack is admitted ONLY when overhead-free (adjacent loads → v128.load,
376
+ // identical pure scalar → splat, matching op → recurse); anything that would
377
+ // need a per-lane `replace_lane` build bails, which makes the rewrite both
378
+ // PROFITABLE and unable to grow code. Every f64x2 lane op is bit-identical to its
379
+ // scalar f64 op (IEEE element-wise), so the result is byte-equal to the scalar form.
380
+ // =============================================================================
381
+ const F64X2_BIN = { 'f64.add': 'f64x2.add', 'f64.sub': 'f64x2.sub', 'f64.mul': 'f64x2.mul', 'f64.div': 'f64x2.div', 'f64.min': 'f64x2.min', 'f64.max': 'f64x2.max' }
382
+ const F64X2_UN = { 'f64.neg': 'f64x2.neg', 'f64.abs': 'f64x2.abs', 'f64.sqrt': 'f64x2.sqrt' }
383
+
384
+ // The subtree's value is the SAME evaluated once (splat) or twice (the two source
385
+ // statements, which are adjacent — no store/reassign between): a pure, side-effect-free,
386
+ // DETERMINISTIC expression. Rejects calls (a `new TypedArray()` alloc returns a fresh
387
+ // pointer per call — splatting it would make the two lanes ALIAS, the array-literal
388
+ // scatter miscompile), loads, and any store/set/memory op.
389
+ const slpSplatSafe = (n) => {
390
+ if (!isArr(n)) return true
391
+ const op = n[0]
392
+ if (typeof op !== 'string') return false
393
+ if (op.startsWith('call') || op.includes('.load') || op.includes('.store')
394
+ || op === 'local.set' || op === 'local.tee' || op === 'global.set'
395
+ || op.startsWith('memory.') || op.includes('.atomic.')) return false
396
+ for (let i = 1; i < n.length; i++) if (!slpSplatSafe(n[i])) return false
397
+ return true
398
+ }
399
+
400
+ // Decompose a load/store node, normalizing the optional `offset=K` attribute jz
401
+ // folds adjacent accesses into: `(op addr …)` → off 0, `(op offset=K addr …)` → K.
402
+ const slpMem = (n) => {
403
+ if (!isArr(n)) return null
404
+ if (typeof n[1] === 'string' && n[1].startsWith('offset=')) return { off: +n[1].slice(7), addr: n[2], val: n[3] }
405
+ return { off: 0, addr: n[1], val: n[2] }
406
+ }
407
+ // The two accesses (x = low/first, y = high/second) provably address the SAME base
408
+ // pointer. Sound shapes only — `y` must read what `x` produced, never redefine it:
409
+ // • x = (local.tee $X e), y = (local.get $X) — x defines the shared ptr, y reuses it
410
+ // • x = (local.get $X), y = (local.get $X) — both read the same already-set local
411
+ // • exprEq(x, y) with NEITHER a tee — identical side-effect-free addresses
412
+ // REJECTS `(local.tee $X eA), (local.tee $X eB)` (y redefines $X to a different address
413
+ // → the high lane would write the wrong place) and `(get $X), (tee $X e)` — the watr.js
414
+ // self-host miscompile came from accepting those by name alone.
415
+ const slpSameBase = (x, y) => {
416
+ if (!isArr(x) || !isArr(y)) return false
417
+ if (x[0] === 'local.tee' && y[0] === 'local.get' && typeof x[1] === 'string' && x[1] === y[1]) return true
418
+ if (x[0] === 'local.get' && y[0] === 'local.get' && typeof x[1] === 'string' && x[1] === y[1]) return true
419
+ return x[0] !== 'local.tee' && y[0] !== 'local.tee' && exprEq(x, y)
420
+ }
421
+
422
+ // Two address expressions name the SAME pointer base. Symmetric, tee/get-normalized
423
+ // (a `local.tee $X` defines what a `local.get $X` reads, so they're one base); else
424
+ // structural exprEq. Used by the RAW guard below — under the no-view gate, a different
425
+ // base is a different allocation, so "not same base" ⇒ provably disjoint memory.
426
+ const slpSameMem = (a, b) => {
427
+ if (!isArr(a) || !isArr(b)) return false
428
+ const an = (a[0] === 'local.tee' || a[0] === 'local.get') && typeof a[1] === 'string' ? a[1] : null
429
+ const bn = (b[0] === 'local.tee' || b[0] === 'local.get') && typeof b[1] === 'string' ? b[1] : null
430
+ if (an !== null || bn !== null) return an === bn
431
+ return exprEq(a, b)
432
+ }
433
+
434
+ // Does `value` load the element at (addr, off) — the slot u0 stores to? SLP materializes
435
+ // BOTH packed values BEFORE either store, so if the high store's VALUE reads the low
436
+ // store's TARGET, the original (which stored low first) and the pack (which reads low's
437
+ // OLD value) diverge — a within-iteration read-after-write hazard. `o[k+1]=o[k]; o[k+2]=
438
+ // o[k+1]` is the canonical miscompile: the second value reads o[k+1], which the first
439
+ // store just wrote. (The sound own-index map `o[i]=…; o[i+1]=…` reads u1's OWN offset,
440
+ // never u0's, so it never trips this.) f64 accesses are 8-byte and 8-aligned, so two
441
+ // overlap iff their offsets are equal.
442
+ const slpReadsOffset = (value, addr, off) => {
443
+ let hit = false
444
+ const walk = (n) => {
445
+ if (hit || !isArr(n)) return
446
+ if (n[0] === 'f64.load') {
447
+ const m = slpMem(n)
448
+ if (m && m.off === off && slpSameMem(m.addr, addr)) { hit = true; return }
449
+ }
450
+ for (let i = 1; i < n.length; i++) walk(n[i])
451
+ }
452
+ walk(value)
453
+ return hit
454
+ }
455
+
456
+ // Pack two isomorphic f64 trees [lo, hi] into an f64x2 value, or null if it isn't
457
+ // overhead-free (adjacent loads → v128.load, identical pure scalar → splat, matching
458
+ // op → recurse). The overhead-free restriction is what makes it both profitable and
459
+ // unable to grow code; every f64x2 lane op is bit-identical to its scalar f64 op.
460
+ const slpPackF64x2 = (lo, hi) => {
461
+ if (!isArr(lo) || !isArr(hi)) return null
462
+ if (lo[0] === 'f64.load' && hi[0] === 'f64.load') {
463
+ const a = slpMem(lo), b = slpMem(hi)
464
+ if (b.off - a.off !== 8 || !slpSameBase(a.addr, b.addr)) return null
465
+ return a.off ? ['v128.load', `offset=${a.off}`, a.addr] : ['v128.load', a.addr]
466
+ }
467
+ if (exprEq(lo, hi) && slpSplatSafe(lo)) return ['f64x2.splat', lo]
468
+ if (lo[0] === hi[0]) {
469
+ const bin = F64X2_BIN[lo[0]]
470
+ if (bin && lo.length === 3 && hi.length === 3) {
471
+ const x = slpPackF64x2(lo[1], hi[1]); if (!x) return null
472
+ const y = slpPackF64x2(lo[2], hi[2]); return y ? [bin, x, y] : null
473
+ }
474
+ const un = F64X2_UN[lo[0]]
475
+ if (un && lo.length === 2 && hi.length === 2) {
476
+ const x = slpPackF64x2(lo[1], hi[1]); return x ? [un, x] : null
477
+ }
478
+ }
479
+ return null
480
+ }
481
+
482
+ // Resolve the element store at `stmts[i]` to { off, addr, value, lo, hi } — the f64
483
+ // value to pack and the inclusive statement span it occupies. jz emits an element
484
+ // store in three shapes, all handled here so SLP fires both pre- and post-watr:
485
+ // • inline (f64.store addr V) → span [i,i]
486
+ // • flat tee'd (local.set $t V) ; (f64.store addr (local.get $t)) → span [i-1,i]
487
+ // • block-wrapped (block (local.set $t V) (f64.store addr (local.get $t)))
488
+ // The tee'd value to pack is V (the definition), not the `(local.get $t)`.
489
+ const slpUnitAt = (stmts, i, getCounts) => {
490
+ const s = stmts[i]
491
+ if (!isArr(s)) return null
492
+ if (s[0] === 'block' && s.length === 3
493
+ && isArr(s[1]) && s[1][0] === 'local.set' && isArr(s[2]) && s[2][0] === 'f64.store') {
494
+ const m = slpMem(s[2])
495
+ if (m && isArr(m.val) && m.val[0] === 'local.get' && m.val[1] === s[1][1]) return { off: m.off, addr: m.addr, value: s[1][2], lo: i, hi: i }
496
+ return null
497
+ }
498
+ if (s[0] !== 'f64.store') return null
499
+ const m = slpMem(s)
500
+ if (!m) return null
501
+ // Flat tee'd: `(local.set $t V) ; (f64.store … (local.get $t))`. Resolving the value
502
+ // to V and dropping the set is sound ONLY if $t is used nowhere else — otherwise a
503
+ // later `(local.get $t)` reads a value we deleted (the watr.js self-host miscompile).
504
+ if (isArr(m.val) && m.val[0] === 'local.get' && typeof m.val[1] === 'string'
505
+ && i > 0 && isArr(stmts[i - 1]) && stmts[i - 1][0] === 'local.set' && stmts[i - 1][1] === m.val[1]
506
+ && getCounts.get(m.val[1]) === 1)
507
+ return { off: m.off, addr: m.addr, value: stmts[i - 1][2], lo: i - 1, hi: i }
508
+ return { off: m.off, addr: m.addr, value: m.val, lo: i, hi: i }
509
+ }
510
+
511
+ // Count `(local.get NAME)` occurrences across the function, so the flat-tee'd
512
+ // resolution above can confirm a store value's temp is single-use before removing it.
513
+ const slpGetCounts = (fn) => {
514
+ const counts = new Map()
515
+ const walk = (n) => {
516
+ if (!isArr(n)) return
517
+ if (n[0] === 'local.get' && typeof n[1] === 'string') counts.set(n[1], (counts.get(n[1]) || 0) + 1)
518
+ for (let i = 1; i < n.length; i++) walk(n[i])
519
+ }
520
+ walk(fn)
521
+ return counts
522
+ }
523
+
524
+ // Rewrite two back-to-back element stores one f64 apart with isomorphic values into a
525
+ // single v128 store. The packed value is computed into a fresh v128 local FIRST, then
526
+ // stored — preserving jz's value-before-address evaluation order (the store address can
527
+ // read a `local.tee` the value defines, e.g. the shared `i<<3` offset). base is the LOW
528
+ // store's address (its tee that defines the shared pointer is kept); the high store +
529
+ // its value dissolve into the high lane. Sound only under the no-view gate at dispatch.
530
+ const slpStorePairsIn = (node, fnLocals, freshIdRef, newLocalDecls, getCounts) => {
531
+ if (!isArr(node)) return
532
+ for (let i = 0; i < node.length; i++) if (isArr(node[i])) slpStorePairsIn(node[i], fnLocals, freshIdRef, newLocalDecls, getCounts)
533
+ for (let i = 0; i < node.length; i++) {
534
+ const u0 = slpUnitAt(node, i, getCounts)
535
+ if (!u0) continue
536
+ const u1 = slpUnitAt(node, u0.hi + 1, getCounts)
537
+ if (!u1 || u1.lo !== u0.hi + 1) continue
538
+ if (u1.off - u0.off !== 8 || !slpSameBase(u0.addr, u1.addr)) continue
539
+ // RAW hazard: the high store's value must not read the low store's target — the pack
540
+ // would read its pre-store value. (u0 writes u0.off; reject if u1.value loads it.)
541
+ if (slpReadsOffset(u1.value, u0.addr, u0.off)) continue
542
+ const packed = slpPackF64x2(u0.value, u1.value)
543
+ if (!packed) continue
544
+ const t = `$__slp${freshIdRef.next++}`
545
+ newLocalDecls.push(['local', t, 'v128']); fnLocals.set(t, 'v128')
546
+ const store = u0.off
547
+ ? ['v128.store', `offset=${u0.off}`, u0.addr, ['local.get', t]]
548
+ : ['v128.store', u0.addr, ['local.get', t]]
549
+ node.splice(u0.lo, u1.hi - u0.lo + 1, ['local.set', t, packed], store)
550
+ i = u0.lo
551
+ }
552
+ }
553
+
270
554
  // ---- Lane type tables ------------------------------------------------------
271
555
 
272
556
  const LANE_INFO = {
@@ -352,6 +636,13 @@ const LANE_PURE = {
352
636
  ['f32.neg', { simd: 'f32x4.neg' }],
353
637
  ['f32.abs', { simd: 'f32x4.abs' }],
354
638
  ['f32.sqrt', { simd: 'f32x4.sqrt' }],
639
+ // rounding: each f32x4.* rounds lane-for-lane identically to the scalar f32.* (same
640
+ // IEEE rounding mode), so the lift is bit-exact. Math.floor/ceil/trunc and the bare
641
+ // f64.nearest jz emits all reach here in a Float32Array kernel.
642
+ ['f32.floor', { simd: 'f32x4.floor' }],
643
+ ['f32.ceil', { simd: 'f32x4.ceil' }],
644
+ ['f32.trunc', { simd: 'f32x4.trunc' }],
645
+ ['f32.nearest', { simd: 'f32x4.nearest' }],
355
646
  ]),
356
647
  f64: new Map([
357
648
  ['f64.add', { simd: 'f64x2.add' }],
@@ -363,9 +654,37 @@ const LANE_PURE = {
363
654
  ['f64.neg', { simd: 'f64x2.neg' }],
364
655
  ['f64.abs', { simd: 'f64x2.abs' }],
365
656
  ['f64.sqrt', { simd: 'f64x2.sqrt' }],
657
+ // rounding: f64x2.* rounds each lane identically to the scalar f64.* op (same IEEE
658
+ // mode), so bit-exact. Unblocks `out[i] = Math.floor/ceil/trunc(f(in[i]))` f64 maps.
659
+ ['f64.floor', { simd: 'f64x2.floor' }],
660
+ ['f64.ceil', { simd: 'f64x2.ceil' }],
661
+ ['f64.trunc', { simd: 'f64x2.trunc' }],
662
+ ['f64.nearest', { simd: 'f64x2.nearest' }],
366
663
  ]),
367
664
  }
368
665
 
666
+ // Integer-load → f32x4 widening, for `out[i] = intArr[i] (* k)` (Int16Array →
667
+ // Float32Array decode/normalize, the canonical audio/image map). jz emits the
668
+ // scalar as `f64.convert_i32_{s,u}(<intload>(addr))`; lift to: load `lanes` ints,
669
+ // widen to i32x4, then f32x4.convert. `steps` are applied innermost-first.
670
+ // `lossy`: i32→f32 rounds (the scalar converts via exact f64 then demotes — double
671
+ // rounding differs by ≤1 ulp), so it needs relaxedSimd; i8/i16 are exact in f32.
672
+ const INT_WIDEN_F32 = {
673
+ 'i32.load': { load: 'v128.load', steps: [], cvt: 's', lossy: true },
674
+ 'i32.load16_s': { load: 'v128.load64_zero', steps: ['i32x4.extend_low_i16x8_s'], cvt: 's', lossy: false },
675
+ 'i32.load16_u': { load: 'v128.load64_zero', steps: ['i32x4.extend_low_i16x8_u'], cvt: 'u', lossy: false },
676
+ 'i32.load8_s': { load: 'v128.load32_zero', steps: ['i16x8.extend_low_i8x16_s', 'i32x4.extend_low_i16x8_s'], cvt: 's', lossy: false },
677
+ 'i32.load8_u': { load: 'v128.load32_zero', steps: ['i16x8.extend_low_i8x16_u', 'i32x4.extend_low_i16x8_u'], cvt: 'u', lossy: false },
678
+ }
679
+
680
+ // f64 scalar op → f32x4 SIMD op, for Float32Array arithmetic jz computes in f64
681
+ // (promote→f64 op→demote). Used only in f32-lane context under relaxedSimd, since
682
+ // the f64→f32 intermediate-precision drop is not bit-exact (see _relaxF32).
683
+ const F64_TO_F32X4 = {
684
+ 'f64.add': 'f32x4.add', 'f64.sub': 'f32x4.sub', 'f64.mul': 'f32x4.mul', 'f64.div': 'f32x4.div',
685
+ 'f64.min': 'f32x4.min', 'f64.max': 'f32x4.max', 'f64.neg': 'f32x4.neg', 'f64.abs': 'f32x4.abs', 'f64.sqrt': 'f32x4.sqrt',
686
+ }
687
+
369
688
  // Horizontal reductions: associative+commutative ops applied to one
370
689
  // loop-carried accumulator. Each entry maps the SCALAR op (which is also
371
690
  // the op used to combine the SIMD result back into the accumulator at the
@@ -807,19 +1126,45 @@ function tryVectorize(bl, fnLocals, freshIdRef) {
807
1126
  // pointers (map loops over distinct arrays). Soundness re-checked post-scan.
808
1127
  const offsetTees = new Map()
809
1128
 
1129
+ // The compute/lane type is the WIDEST FLOAT among all loads+stores. A narrower
1130
+ // float/int LOAD is then a widening read (INT_WIDEN_F32 / f32→f64), a narrower
1131
+ // float/int STORE a narrowing write (demote / trunc+wrap) — both sub-width memory
1132
+ // ops around the float lane. Pinning it up front (vs whichever op the recursive
1133
+ // scan hits first) is what lets a narrowing map `i16[i] = f32arr[i]*k` keep the
1134
+ // f32 compute lane instead of locking onto the i16 store. No float → integer lane
1135
+ // (set by the scan below, unchanged).
1136
+ const isNarrowStore = (lane, sty) => (lane === 'f32' && (sty === 'i16' || sty === 'i8'))
1137
+ || (lane === 'f64' && (sty === 'f32' || sty === 'i32'))
1138
+ let preFloat = null
1139
+ const scanFloatWidth = (n) => {
1140
+ if (!isArr(n)) return
1141
+ const t = LOAD_OPS[n[0]] || STORE_OPS[n[0]]
1142
+ if (t === 'f64') preFloat = 'f64'
1143
+ else if (t === 'f32' && preFloat == null) preFloat = 'f32'
1144
+ for (let i = 1; i < n.length; i++) scanFloatWidth(n[i])
1145
+ }
1146
+ for (const s of body) scanFloatWidth(s)
1147
+ if (preFloat) { laneType = preFloat; stride = LANE_INFO[preFloat].stride }
1148
+
810
1149
  function scanForLoadsStores(node, parent, pi) {
811
1150
  if (!isArr(node)) return true
812
1151
  const op = node[0]
813
1152
  if (LOAD_OPS[op]) {
1153
+ const lt = LOAD_OPS[op]
1154
+ // int→f32 widening map (`out[i] = intArr[i] (*k)`): an integer load feeding a
1155
+ // Float32Array store. Accept under the f32 lane and validate at the int element
1156
+ // stride (the loop steps `lanes` f32 = 4 elements; load64_zero/load32_zero read
1157
+ // exactly 4 ints). liftExprV widens via INT_WIDEN_F32.
1158
+ const widenInt = laneType === 'f32' && lt !== 'f32' && INT_WIDEN_F32[op]
814
1159
  if (laneType == null) {
815
- laneType = LOAD_OPS[op]
1160
+ laneType = lt
816
1161
  stride = LANE_INFO[laneType].stride
817
- } else if (LOAD_OPS[op] !== laneType) {
1162
+ } else if (lt !== laneType && !widenInt) {
818
1163
  return false
819
1164
  }
820
1165
  const m = matchLaneAddr(node[1], incVar, addrLocals, offsetTees)
821
1166
  if (!m) return false
822
- if ((1 << m.strideLog2) !== stride) return false
1167
+ if ((1 << m.strideLog2) !== (widenInt ? LANE_INFO[lt].stride : stride)) return false
823
1168
  if (m.teeName) addrLocals.set(m.teeName, { strideLog2: m.strideLog2, base: m.base })
824
1169
  if (m.offsetTeeName) offsetTees.set(m.offsetTeeName, m.strideLog2)
825
1170
  loadStoreSites.push({ parent, idx: pi, kind: 'load' })
@@ -827,11 +1172,15 @@ function tryVectorize(bl, fnLocals, freshIdRef) {
827
1172
  }
828
1173
  if (STORE_OPS[op]) {
829
1174
  const sty = STORE_OPS[op]
830
- if (laneType != null && sty !== laneType) return false
1175
+ // narrowing store: a narrower element under a wider float lane (`o[i]=narrow(f(x))`,
1176
+ // codec encode / downsample). Validate the store address at the narrow element stride
1177
+ // (the loop steps `lanes` of the float lane; the partial store writes that many).
1178
+ const narrowing = laneType != null && sty !== laneType && isNarrowStore(laneType, sty)
1179
+ if (laneType != null && sty !== laneType && !narrowing) return false
831
1180
  if (laneType == null) { laneType = sty; stride = LANE_INFO[laneType].stride }
832
1181
  const m = matchLaneAddr(node[1], incVar, addrLocals, offsetTees)
833
1182
  if (!m) return false
834
- if ((1 << m.strideLog2) !== stride) return false
1183
+ if ((1 << m.strideLog2) !== (narrowing ? LANE_INFO[sty].stride : stride)) return false
835
1184
  if (m.teeName) addrLocals.set(m.teeName, { strideLog2: m.strideLog2, base: m.base })
836
1185
  if (m.offsetTeeName) offsetTees.set(m.offsetTeeName, m.strideLog2)
837
1186
  loadStoreSites.push({ parent, idx: pi, kind: 'store' })
@@ -913,12 +1262,35 @@ function tryVectorize(bl, fnLocals, freshIdRef) {
913
1262
  }
914
1263
  }
915
1264
 
1265
+ // A ToInt32 (`|0`) narrowing conversion is commonly CSE'd into its own lane-local just before
1266
+ // the store (`set $t (…trunc_sat…); store addr (local.get $t)`), hiding it from the
1267
+ // narrowing-store path (liftStmt would then lift the i32 wrap in the f64 lane and bail). When
1268
+ // such a lane-local is read exactly once by a narrowing store, inline the conversion back into
1269
+ // the store so peelNarrowConv/narrowStore handle it. The original set survives in the scalar
1270
+ // remainder; this only reshapes the SIMD lift (and bails cleanly if liftStmt still declines).
1271
+ let body2 = body
1272
+ {
1273
+ const getCount = new Map()
1274
+ const countGets = (n) => { if (!isArr(n)) return; if (n[0] === 'local.get' && typeof n[1] === 'string') getCount.set(n[1], (getCount.get(n[1]) || 0) + 1); for (let i = 1; i < n.length; i++) countGets(n[i]) }
1275
+ for (const s of body) countGets(s)
1276
+ const dropped = new Set()
1277
+ const inlined = body.map(s => {
1278
+ if (isArr(s) && STORE_OPS[s[0]] && s.length === 3 && STORE_OPS[s[0]] !== laneType &&
1279
+ isLocalGet(s[2]) && localKind.get(s[2][1]) === 'lane' && getCount.get(s[2][1]) === 1) {
1280
+ const def = body.find(x => isArr(x) && x[0] === 'local.set' && x[1] === s[2][1] && x.length === 3)
1281
+ if (def && peelNarrowConv(def[2], STORE_OPS[s[0]])) { dropped.add(def); return [s[0], s[1], def[2]] }
1282
+ }
1283
+ return s
1284
+ })
1285
+ if (dropped.size) body2 = inlined.filter(s => !dropped.has(s))
1286
+ }
1287
+
916
1288
  // Build lifted body. If anything fails to lift, bail.
917
- const newLanedLocals = new Map() // origName → { laneName, simdType }
1289
+ const newLanedLocals = new Map() // origName → laneName (bare string; see getOrAllocLanedLocal)
918
1290
  const extraLocals = [] // canon temps allocated during lift
919
- const ctx = { laneType, incVar, rampVar: null, rampTemp: null, widenLoads: false, localKind, newLanedLocals, extraLocals, freshIdRef, fail: false, failReason: null }
1291
+ const ctx = { laneType, incVar, rampVar: null, rampTemp: null, widenLoads: false, localKind, fnLocals, newLanedLocals, extraLocals, freshIdRef, fail: false, failReason: null }
920
1292
  const lifted = []
921
- for (const s of body) {
1293
+ for (const s of body2) {
922
1294
  const r = liftStmt(s, ctx)
923
1295
  if (ctx.fail) return null
924
1296
  if (r != null) {
@@ -966,7 +1338,7 @@ function tryVectorize(bl, fnLocals, freshIdRef) {
966
1338
  // Locals to add to function header.
967
1339
  const newLocalDecls = [
968
1340
  ['local', simdBoundName, 'i32'],
969
- ...[...newLanedLocals.values()].map(({ laneName }) => ['local', laneName, 'v128']),
1341
+ ...[...newLanedLocals.values()].map(laneName => ['local', laneName, 'v128']),
970
1342
  ...extraLocals,
971
1343
  ]
972
1344
 
@@ -1204,7 +1576,10 @@ function tryStencil(node, fnLocals, freshIdRef, enabled) {
1204
1576
  if (name === incVar) continue
1205
1577
  const ty = fnLocals.get(name)
1206
1578
  if (ty === 'i32') { localKind.set(name, 'addr'); continue }
1207
- if (ty === laneType) {
1579
+ // A stencil temp computed in f64 then stored to an f32 array carries `ty === 'f64'`
1580
+ // in an f32 lane (jz computes Float32Array math in f64). Treat it as lane/invariant
1581
+ // data the same as a native-typed local — the lift lanes it as f32x4 (relaxedSimd).
1582
+ if (ty === laneType || (laneType === 'f32' && ty === 'f64')) {
1208
1583
  if (writes.has(name)) {
1209
1584
  let fk = null; for (const s of body) { const k = firstAccess(s, name); if (k) { fk = k; break } }
1210
1585
  if (fk === 'read') return null // loop-carried float local
@@ -1218,7 +1593,7 @@ function tryStencil(node, fnLocals, freshIdRef, enabled) {
1218
1593
 
1219
1594
  // Lift through the shared lifter (addresses kept verbatim; loads → v128.load).
1220
1595
  const newLanedLocals = new Map(), extraLocals = []
1221
- const ctx = { laneType, incVar, rampVar: null, rampTemp: null, widenLoads: false, localKind, newLanedLocals, extraLocals, freshIdRef, fail: false, failReason: null }
1596
+ const ctx = { laneType, incVar, rampVar: null, rampTemp: null, widenLoads: false, localKind, fnLocals, newLanedLocals, extraLocals, freshIdRef, fail: false, failReason: null }
1222
1597
  const lifted = []
1223
1598
  for (const s of body) {
1224
1599
  const r = liftStmt(s, ctx)
@@ -1255,7 +1630,7 @@ function tryStencil(node, fnLocals, freshIdRef, enabled) {
1255
1630
  // LICM-hoisted $__li invariants run ahead of the SIMD block (the scalar tail's
1256
1631
  // copy inside bl.blockNode re-runs them harmlessly — pure & loop-invariant).
1257
1632
  const wrapper = ['block', ...preamble.map(cloneNode), ...peelStmts, boundSetup, simdBlock, bl.blockNode]
1258
- const newLocalDecls = [['local', simdBoundName, 'i32'], ...[...newLanedLocals.values()].map(({ laneName }) => ['local', laneName, 'v128']), ...extraLocals]
1633
+ const newLocalDecls = [['local', simdBoundName, 'i32'], ...[...newLanedLocals.values()].map(laneName => ['local', laneName, 'v128']), ...extraLocals]
1259
1634
  return { wrapper, newLocalDecls }
1260
1635
  }
1261
1636
 
@@ -1287,17 +1662,67 @@ function tryReduceVectorize(bl, fnLocals, freshIdRef, multiAcc = false) {
1287
1662
  // — or a NaN-canonicalized two-statement min/max reduction —
1288
1663
  // (local.set $cn (OP (local.get $acc) EXPR))
1289
1664
  // (local.set $acc (select C (local.get $cn) (T.ne $cn $cn)))
1290
- const bodyLen = incIdx - 3
1665
+ // A conditional-store min/max (`if (a[i] > m) m = a[i]`) is the SAME reduction as the ternary
1666
+ // `m = a[i] > m ? a[i] : m`; rewrite it to the select-assign form so one recognizer covers both.
1667
+ // Sound for recognition: the lane EXPR (an array load) is pure, and the SIMD lift reads it
1668
+ // unconditionally anyway (pmax), while the scalar remainder keeps the original conditional store.
1669
+ const asSelectAssign = (stmt) => {
1670
+ if (isArr(stmt) && stmt[0] === 'if' && stmt.length === 3 && isArr(stmt[2]) && stmt[2][0] === 'then' && stmt[2].length === 2) {
1671
+ const set = stmt[2][1]
1672
+ if (isArr(set) && set[0] === 'local.set' && set.length === 3 && !hasSideEffect(set[2]))
1673
+ return ['local.set', set[1], ['select', set[2], ['local.get', set[1]], stmt[1]]]
1674
+ }
1675
+ return stmt
1676
+ }
1677
+ const bodyStmts = []
1678
+ for (let i = 3; i < incIdx; i++) bodyStmts.push(asSelectAssign(loopNode[i]))
1679
+ // CSE collapse: `m = a[i] > m ? a[i] : m` hoists the load into its own `(local.set $t LOAD)`
1680
+ // ahead of the reduction, making a 2-statement body the single-statement min/max recognizer
1681
+ // misses. When $t is pure (no side effect, no accumulator reference) inline it back into the
1682
+ // reduction so the canonical one-statement shape is recognized. Sound: the lift only consumes
1683
+ // the inlined lane expr for the SIMD prefix; the original $t set survives in the scalar
1684
+ // remainder (the unchanged blockNode), so $t stays defined wherever else it is read.
1685
+ let body0 = bodyStmts
1686
+ if (bodyStmts.length === 2) {
1687
+ const [s1, s2] = bodyStmts
1688
+ if (isArr(s1) && s1[0] === 'local.set' && typeof s1[1] === 'string' && s1.length === 3 &&
1689
+ isArr(s2) && s2[0] === 'local.set' && typeof s2[1] === 'string' && s2.length === 3 && s1[1] !== s2[1]) {
1690
+ const t = s1[1], expr = s1[2]
1691
+ const usesName = (n, name) => isArr(n) && ((n[0] === 'local.get' && n[1] === name) || n.some(c => usesName(c, name)))
1692
+ if (!hasSideEffect(expr) && !usesName(expr, s2[1]) && !usesName(expr, t)) {
1693
+ const subst = (n) => isArr(n) ? (n[0] === 'local.get' && n[1] === t ? expr : n.map(subst)) : n
1694
+ body0 = [['local.set', s2[1], subst(s2[2])]]
1695
+ }
1696
+ }
1697
+ }
1698
+ const bodyLen = body0.length
1291
1699
  let accName, opName, reduceEntry, exprNode, canonC = null
1292
1700
  if (bodyLen === 1) {
1293
- const stmt = loopNode[3]
1701
+ const stmt = body0[0]
1294
1702
  if (!isArr(stmt) || stmt[0] !== 'local.set' || stmt.length !== 3) return null
1295
1703
  accName = stmt[1]
1296
1704
  if (typeof accName !== 'string') return null
1297
1705
  const rhs = stmt[2]
1298
1706
  if (!isArr(rhs)) return null
1299
1707
  const minmax = matchIntMinMaxReduce(rhs, accName)
1300
- if (minmax) {
1708
+ if (minmax && minmax.laneType === 'f64') {
1709
+ // Comparison min/max over an f64 array (`m = a[i] > m ? a[i] : m`). f64x2.pmax/pmin
1710
+ // replicate the scalar `(a>m)?a:m` EXACTLY per element — pmax(m,a) = (m<a)?a:m keeps the
1711
+ // accumulator on NaN (m<NaN is false) and on a ±0 tie, never NaN-poisoning the way
1712
+ // f64x2.max would. They preserve the data's exact NaN bits (a selection, not a compute),
1713
+ // so no canon is needed. The ONLY divergence from the sequential scalar is the SIGN of a
1714
+ // zero RESULT when the extremum is hit by both +0 and −0 in different lanes (a cross-lane
1715
+ // reorder) — strictly less than the ULP reassociation the sum reductions already accept,
1716
+ // so it rides the relaxedSimd tier (on at 'speed'); strict callers opt out (scalar).
1717
+ if (!_relaxF32) return null
1718
+ reduceEntry = {
1719
+ simd: minmax.isMax ? 'f64x2.pmax' : 'f64x2.pmin',
1720
+ extract: 'f64x2.extract_lane', laneType: 'f64',
1721
+ identity: ['f64.const', minmax.isMax ? '-inf' : 'inf'],
1722
+ minmaxSelect: true, isMax: minmax.isMax, pmaxF64: true,
1723
+ }
1724
+ exprNode = minmax.exprNode
1725
+ } else if (minmax) {
1301
1726
  // Synthetic entry: WASM has the SIMD i32x4.max_s/min_s but no scalar i32.max, so the
1302
1727
  // horizontal fold + merge below use select (flagged by minmaxSelect). Identity is the
1303
1728
  // op's neutral — INT_MIN for max, INT_MAX for min. A bare narrow load instead folds
@@ -1343,7 +1768,7 @@ function tryReduceVectorize(bl, fnLocals, freshIdRef, multiAcc = false) {
1343
1768
  exprNode = rhs[2]
1344
1769
  }
1345
1770
  } else if (bodyLen === 2) {
1346
- const s1 = loopNode[3], s2 = loopNode[4]
1771
+ const s1 = body0[0], s2 = body0[1]
1347
1772
  if (!isArr(s1) || s1[0] !== 'local.set' || s1.length !== 3) return null
1348
1773
  if (!isArr(s2) || s2[0] !== 'local.set' || s2.length !== 3) return null
1349
1774
  const cnName = s1[1], rhs = s1[2]
@@ -1432,15 +1857,22 @@ function tryReduceVectorize(bl, fnLocals, freshIdRef, multiAcc = false) {
1432
1857
  const stride = LANE_INFO[laneType].stride
1433
1858
  const addrLocals = new Map()
1434
1859
  const offsetTees = new Map()
1435
- let loadCount = 0
1860
+ let loadCount = 0, sawWidenF32 = false
1436
1861
  function scanExpr(node) {
1437
1862
  if (!isArr(node)) return true
1438
1863
  const op = node[0]
1439
1864
  if (LOAD_OPS[op]) {
1440
- if (LOAD_OPS[op] !== laneType) return false
1865
+ // f32→f64 widening reduction (`s += f32arr[i]`, acc f64): liftExprV promotes
1866
+ // the f32.load to f64x2.promote_low_f32x4, so accept it under an f64 lane and
1867
+ // validate at the f32 element stride (4) — the loop still steps `lanes` (2)
1868
+ // elements, advancing the f32 address by 8 bytes (the load64_zero the lift reads).
1869
+ const ltw = LOAD_OPS[op]
1870
+ const widenF32 = ltw === 'f32' && laneType === 'f64'
1871
+ if (ltw !== laneType && !widenF32) return false
1872
+ if (widenF32) sawWidenF32 = true
1441
1873
  const m = matchLaneAddr(node[1], incVar, addrLocals, offsetTees)
1442
1874
  if (!m) return false
1443
- if ((1 << m.strideLog2) !== stride) return false
1875
+ if ((1 << m.strideLog2) !== (widenF32 ? 4 : stride)) return false
1444
1876
  if (m.teeName) addrLocals.set(m.teeName, { strideLog2: m.strideLog2, base: m.base })
1445
1877
  if (m.offsetTeeName) offsetTees.set(m.offsetTeeName, m.strideLog2)
1446
1878
  loadCount++
@@ -1477,9 +1909,14 @@ function tryReduceVectorize(bl, fnLocals, freshIdRef, multiAcc = false) {
1477
1909
  for (const name of addrLocals.keys()) localKind.set(name, 'addr')
1478
1910
  for (const name of offsetTees.keys()) localKind.set(name, 'addr')
1479
1911
 
1480
- const ctx = { laneType, incVar, rampVar: null, rampTemp: null, widenLoads: false, localKind, newLanedLocals: new Map(), extraLocals: [], freshIdRef, fail: false, failReason: null }
1912
+ const ctx = { laneType, incVar, rampVar: null, rampTemp: null, widenLoads: false, localKind, fnLocals, newLanedLocals: new Map(), extraLocals: [], freshIdRef, fail: false, failReason: null }
1913
+
1481
1914
  const liftedExpr = liftExprV(exprNode, ctx)
1482
- if (ctx.fail) return null
1915
+ // liftExprV's contract is "null ⟺ ctx.fail"; under self-host (jz.wasm) it can diverge and
1916
+ // return null WITHOUT the flag, which would otherwise splice a literal `null` operand into the
1917
+ // emitted `(<reduce>.add acc null)` — invalid wasm ("not enough arguments on the stack"). Treat
1918
+ // a null lift as a bail (the loop stays scalar — correct, just unvectorized on that leg).
1919
+ if (ctx.fail || liftedExpr == null) return null
1483
1920
  if (ctx.newLanedLocals.size > 0 || ctx.extraLocals.length > 0) return null
1484
1921
 
1485
1922
  // Synthesize SIMD prefix block + horizontal reduce + (preserved scalar tail).
@@ -1501,7 +1938,9 @@ function tryReduceVectorize(bl, fnLocals, freshIdRef, multiAcc = false) {
1501
1938
  // the same kind the existing 2-lane fold already does, identical on every engine.
1502
1939
  // Restricted to the plain horizontal-fold FP path (not min/max-select, the
1503
1940
  // narrow-widening sums, or NaN-canon — those have their own fold shapes).
1504
- const plainReduce = !reduceEntry.minmaxSelect && !widen && canonC == null
1941
+ // The f32→f64 widening sum uses half-width load64_zero loads; the multi-accumulator
1942
+ // offsetLoads/laneBytes logic assumes full v128 loads, so keep it single-accumulator.
1943
+ const plainReduce = !reduceEntry.minmaxSelect && !widen && !sawWidenF32 && canonC == null
1505
1944
  const NACC = (multiAcc && plainReduce && (laneType === 'f64' || laneType === 'f32')) ? 4 : 1
1506
1945
  const accK = (k) => k === 0 ? simdAccName : `$__simd_acc${id}_${k}`
1507
1946
  const laneBytes = lanes * stride
@@ -1539,14 +1978,17 @@ function tryReduceVectorize(bl, fnLocals, freshIdRef, multiAcc = false) {
1539
1978
  const extraDecls = []
1540
1979
  let mergeStmts
1541
1980
  if (reduceEntry.minmaxSelect) {
1542
- // No scalar i32.max/min — fold via select through an i32 temp (no exponential
1543
- // operand duplication): ht = lane0; ht = minmax(ht, lane_k); acc = minmax(acc, ht).
1544
- // `select(a,b,(gt|lt)_s a b)` = a when it's the larger/smaller, i.e. minmax(a,b).
1545
- const cmpOp = reduceEntry.isMax ? 'i32.gt_s' : 'i32.lt_s'
1981
+ // No scalar max/min op — fold via select through a temp (no exponential operand
1982
+ // duplication): ht = lane0; ht = minmax(ht, lane_k); acc = minmax(acc, ht). For int,
1983
+ // `select(a,b,(gt|lt)_s a b)` = max/min(a,b). For the f64 pmax/pmin reduction the scalar
1984
+ // equivalent is the pmax/pmin select — `pmax(a,b) = (a<b)?b:a` — so the merge keeps the
1985
+ // same NaN/±0 tie semantics as the f64x2.pmax lanes.
1546
1986
  const ht = `$__simd_h${id}`
1547
- extraDecls.push(['local', ht, 'i32'])
1987
+ extraDecls.push(['local', ht, reduceEntry.pmaxF64 ? 'f64' : 'i32'])
1548
1988
  const lane = (k) => [reduceEntry.extract, k, ['local.get', simdAccName]]
1549
- const minmaxSel = (a, b) => ['select', a, b, [cmpOp, a, b]]
1989
+ const minmaxSel = reduceEntry.pmaxF64
1990
+ ? (a, b) => reduceEntry.isMax ? ['select', b, a, ['f64.lt', a, b]] : ['select', b, a, ['f64.lt', b, a]]
1991
+ : (a, b) => ['select', a, b, [reduceEntry.isMax ? 'i32.gt_s' : 'i32.lt_s', a, b]]
1550
1992
  mergeStmts = [['local.set', ht, lane(0)]]
1551
1993
  for (let k = 1; k < lanes; k++) mergeStmts.push(['local.set', ht, minmaxSel(lane(k), ['local.get', ht])])
1552
1994
  if (reduceEntry.accF64) {
@@ -1735,13 +2177,17 @@ function _isAddressLocal(body, name, ind) {
1735
2177
 
1736
2178
  // ---- Lifter ----------------------------------------------------------------
1737
2179
 
1738
- function getOrAllocLanedLocal(name, ctx) {
1739
- let r = ctx.newLanedLocals.get(name)
1740
- if (!r) {
1741
- r = { laneName: `${name}__v`, origName: name }
1742
- ctx.newLanedLocals.set(name, r)
1743
- }
1744
- return r
2180
+ // Returns the v128 lane-local NAME (a string) for `name`, allocating once. We store the bare
2181
+ // string NOT a `{laneName}` object — because a schema-object read back through the Map in a
2182
+ // DIFFERENT function returns undefined under self-host. Takes `newLanedLocals` directly
2183
+ // (not ctx) so callers don't need to pass the full ctx object to a helper at call-depth 2.
2184
+ function getOrAllocLanedLocal(name, newLanedLocals) {
2185
+ let laneName = newLanedLocals.get(name)
2186
+ if (!laneName) {
2187
+ laneName = `${name}__v`
2188
+ newLanedLocals.set(name, laneName)
2189
+ }
2190
+ return laneName
1745
2191
  }
1746
2192
 
1747
2193
  // Wrap an already-lifted v128 value `coreV` in per-lane NaN canonicalization:
@@ -1752,7 +2198,10 @@ function getOrAllocLanedLocal(name, ctx) {
1752
2198
  // Otherwise we materialize a fresh v128 temp so the core evaluates once.
1753
2199
  function liftCanon(coreV, C, ctx, info) {
1754
2200
  const laneNe = ctx.laneType === 'f32' ? 'f32x4.ne' : 'f64x2.ne'
1755
- const splatC = [info.splat, C]
2201
+ // The f32-via-f64 canon carries an f64 NaN const splat it as f32 (demote is
2202
+ // exact for the canonical NaN, and the lane value coreV is already f32x4).
2203
+ const cF = ctx.laneType === 'f32' && isArr(C) && C[0] === 'f64.const' ? ['f32.const', C[1]] : C
2204
+ const splatC = [info.splat, cF]
1756
2205
  if (isArr(coreV) && coreV[0] === 'local.get') {
1757
2206
  return ['v128.bitselect', splatC, coreV, [laneNe, coreV, coreV]]
1758
2207
  }
@@ -1770,6 +2219,15 @@ function liftCanon(coreV, C, ctx, info) {
1770
2219
  // bail for the block currently under the recognizer chain; the walk reads it after.
1771
2220
  let _whyNotActive = false
1772
2221
  let _whyNotReason = null
2222
+ // Precision-relaxed f32 SIMD. jz computes Float32Array arithmetic in f64
2223
+ // (`f32.demote_f64 (f64.mul (f64.promote_f32 …) …)`); lifting that chain to
2224
+ // `f32x4.mul` over `v128.load` changes the intermediate from f64 to f32 — a
2225
+ // sub-ulp difference at f32 precision (inaudible for audio/DSP, the canonical
2226
+ // f32-SIMD trade every audio engine makes), but NOT bit-exact, so it is gated
2227
+ // on the same `relaxedSimd` opt-in that enables relaxed-FMA. The promote/demote
2228
+ // *strip* for a pure f32 copy (no arithmetic) round-trips losslessly and stays
2229
+ // on unconditionally. Armed for the duration of a vectorizeLaneLocal call.
2230
+ let _relaxF32 = false
1773
2231
 
1774
2232
  // Mark a lift bail and record its reason. First-write-wins: the innermost failing op
1775
2233
  // sets ctx.failReason; outer frames see ctx.fail already set and return without
@@ -1782,6 +2240,70 @@ const liftFail = (ctx, reason) => {
1782
2240
  }
1783
2241
 
1784
2242
  /** Lift a statement. Returns lifted stmt, or null to skip, or ['__seq__', ...] for multiple. */
2243
+ // Conditional lane-local assignment in a stencil/map body — the sibling of the
2244
+ // if-STORE path below, but the destination is a 'lane' LOCAL, not memory:
2245
+ // if (C) L = A [else L = B | else <nested if assigning L>]
2246
+ // the saturation / clamp shape (waves' amplitude clamp `if(nb>CAP)nb=CAP; else
2247
+ // if(nb<-CAP)nb=-CAP`). Built as ONE nested `v128.bitselect` EXPRESSION so every
2248
+ // mask reads the PRE-assignment lane value (no intermediate stores ⇒ order-free,
2249
+ // and bit-exact with the scalar select chain — a comparison mask is all-ones /
2250
+ // all-zeros per lane, so bitselect is an exact lane select), then a single
2251
+ // `local.set` of the laned local. Returns the lifted node, or null when `stmt` is
2252
+ // not a lane-assignment if (so the if-STORE path can try). Sets ctx.fail only once
2253
+ // committed (a lane-if shape that cannot be lifted).
2254
+ function tryLiftLaneIf(stmt, ctx) {
2255
+ const armBody = (arm) => {
2256
+ let body = arm.slice(1)
2257
+ if (body.length === 1 && isArr(body[0]) && body[0][0] === 'block') {
2258
+ const b = body[0]; let i = 1
2259
+ if (typeof b[i] === 'string' && b[i].startsWith('$')) i++
2260
+ if (isArr(b[i]) && b[i][0] === 'result') i++
2261
+ body = b.slice(i)
2262
+ }
2263
+ return body
2264
+ }
2265
+ // The lane being assigned: the innermost then-arm's single local.set target.
2266
+ const laneOf = (node) => {
2267
+ if (!isArr(node)) return null
2268
+ if (node[0] === 'local.set' && typeof node[1] === 'string' && ctx.localKind.get(node[1]) === 'lane') return node[1]
2269
+ if (node[0] === 'if' && isArr(node[2]) && node[2][0] === 'then') {
2270
+ const body = armBody(node[2])
2271
+ if (body.length === 1) return laneOf(body[0])
2272
+ }
2273
+ return null
2274
+ }
2275
+ const lane = laneOf(stmt)
2276
+ if (!lane) return null
2277
+ // Recurse the if-chain into nested bitselects; each `local.set L = V` is a leaf value.
2278
+ const buildVal = (node) => {
2279
+ if (isArr(node) && node[0] === 'local.set' && node[1] === lane) return liftExprV(node[2], ctx)
2280
+ if (isArr(node) && node[0] === 'if' && isArr(node[2]) && node[2][0] === 'then') {
2281
+ const thenBody = armBody(node[2])
2282
+ if (thenBody.length !== 1) return liftFail(ctx, 'lane-if: non-single then arm')
2283
+ let cond = node[1]
2284
+ if (isArr(cond) && cond[0] === 'i32.ne' && isI32Const(cond[2]) && cond[2][1] === 0) cond = cond[1]
2285
+ const cmp = isArr(cond) && cond.length === 3 ? LANE_COMPARE[ctx.laneType]?.[cond[0]] : null
2286
+ if (!cmp) return liftFail(ctx, 'lane-if: condition is not a lane comparison')
2287
+ const ca = liftExprV(cond[1], ctx); if (ctx.fail) return null
2288
+ const cb = liftExprV(cond[2], ctx); if (ctx.fail) return null
2289
+ const thenVal = buildVal(thenBody[0]); if (ctx.fail) return null
2290
+ const elseArm = (isArr(node[3]) && node[3][0] === 'else') ? armBody(node[3]) : null
2291
+ let elseVal
2292
+ if (elseArm) {
2293
+ if (elseArm.length !== 1) return liftFail(ctx, 'lane-if: non-single else arm')
2294
+ elseVal = buildVal(elseArm[0]); if (ctx.fail) return null
2295
+ } else {
2296
+ elseVal = ['local.get', getOrAllocLanedLocal(lane, ctx.newLanedLocals)] // no else ⇒ keep current
2297
+ }
2298
+ return ['v128.bitselect', thenVal, elseVal, [cmp, ca, cb]]
2299
+ }
2300
+ return liftFail(ctx, 'lane-if: unrecognized arm shape')
2301
+ }
2302
+ const val = buildVal(stmt)
2303
+ if (ctx.fail || val == null) return null
2304
+ return ['local.set', getOrAllocLanedLocal(lane, ctx.newLanedLocals), val]
2305
+ }
2306
+
1785
2307
  function liftStmt(stmt, ctx) {
1786
2308
  if (!isArr(stmt)) {
1787
2309
  // Bare strings like "drop" — produced by stack-form WAT. We unwrap value-blocks
@@ -1799,7 +2321,7 @@ function liftStmt(stmt, ctx) {
1799
2321
  return ['local.set', name, stmt[2]]
1800
2322
  }
1801
2323
  if (kind === 'lane') {
1802
- const { laneName } = getOrAllocLanedLocal(name, ctx)
2324
+ const laneName = getOrAllocLanedLocal(name, ctx.newLanedLocals)
1803
2325
  const v = liftExprV(stmt[2], ctx)
1804
2326
  if (ctx.fail) return null
1805
2327
  return ['local.set', laneName, v]
@@ -1808,14 +2330,32 @@ function liftStmt(stmt, ctx) {
1808
2330
  }
1809
2331
 
1810
2332
  if (STORE_OPS[op]) {
1811
- const simdStore = 'v128.store'
1812
2333
  const addr = stmt[1] // we leave addresses as-is (scalar i32 expressions)
1813
- const val = liftExprV(stmt[2], ctx)
1814
- if (ctx.fail) return null
1815
2334
  // Handle memarg if present (last positional after addr/val): unlikely in
1816
2335
  // pre-watr IR for this shape; bail if more than 3 children.
1817
2336
  if (stmt.length !== 3) return liftFail(ctx, `${op} with memarg`)
1818
- return [simdStore, addr, val]
2337
+ const sty = STORE_OPS[op]
2338
+ // Narrowing store: a narrower element written from a wider float lane (`o[i] =
2339
+ // narrow(f(x))` — codec encode / downsample). The scalar store value carries a
2340
+ // conversion (f32.demote_f64, or the float→int ToInt32 idiom); peel it, lift the
2341
+ // inner float expr, and let narrowStore apply the SIMD narrow + low-byte store.
2342
+ if (sty !== ctx.laneType) {
2343
+ // Integer narrowing (`o[i] = (f(x)) | 0` into Int32Array/…) lowers via the saturating
2344
+ // i32x4.trunc_sat_f64x2_s_zero, which clamps +Inf / |x|≥2³¹ to INT_MAX where scalar
2345
+ // ToInt32 wraps mod 2³² — bit-exact for in-range finite values (every pixel/coordinate/
2346
+ // typical-DSP value), divergent only at that edge, so it rides relaxedSimd. Float demote
2347
+ // (f64→f32) is bit-exact (round-to-nearest both ways) and stays ungated.
2348
+ if (sty !== 'f32' && !_relaxF32) return liftFail(ctx, `narrowing ${ctx.laneType}->${sty} store saturates out-of-range (needs relaxedSimd)`)
2349
+ const inner = peelNarrowConv(stmt[2], sty)
2350
+ if (!inner) return liftFail(ctx, `narrowing store ${ctx.laneType}->${sty}: unrecognized conversion`)
2351
+ const innerV = liftExprV(inner, ctx)
2352
+ if (ctx.fail) return null
2353
+ const ns = narrowStore(addr, innerV, ctx.laneType, sty, ctx)
2354
+ return ns || liftFail(ctx, `no narrowing ${ctx.laneType}->${sty}`)
2355
+ }
2356
+ const val = liftExprV(stmt[2], ctx)
2357
+ if (ctx.fail) return null
2358
+ return ['v128.store', addr, val]
1819
2359
  }
1820
2360
 
1821
2361
  // (block (result T) STMTS... TAIL_EXPR) followed by sibling "drop" — we get
@@ -1847,6 +2387,10 @@ function liftStmt(stmt, ctx) {
1847
2387
  // are trap-free) and emit ONE store of `bitselect(A, B, mask(COND))`. Unlocks per-pixel conditional
1848
2388
  // maps like lorenz's i32x4 trail fade (`if (p & 0xffffff) px[i] = fade(p)`).
1849
2389
  if (op === 'if' && isArr(stmt[2]) && stmt[2][0] === 'then') {
2390
+ // First: conditional lane-LOCAL assignment (clamp/saturation) → bitselect into the laned local.
2391
+ const laneLifted = tryLiftLaneIf(stmt, ctx)
2392
+ if (ctx.fail) return null
2393
+ if (laneLifted) return laneLifted
1850
2394
  const armOf = (arm) => {
1851
2395
  let body = arm.slice(1)
1852
2396
  if (body.length === 1 && isArr(body[0]) && body[0][0] === 'block') { // unwrap a single block arm
@@ -1914,6 +2458,39 @@ function liftExprV(expr, ctx) {
1914
2458
  return ['f64x2.promote_low_f32x4', addr]
1915
2459
  }
1916
2460
 
2461
+ // f32-lane: jz computes Float32Array arithmetic in f64, wrapping the f32 load in
2462
+ // `f64.promote_f32` and the result in `f32.demote_f64`. The promote/demote are
2463
+ // lane-space identities — strip them (a promote-of-load + demote round-trips
2464
+ // losslessly: a pure `b[i]=a[i]` copy vectorizes bit-exactly, always). The f64
2465
+ // arithmetic op and any f64 constant map to their f32x4 forms only under
2466
+ // relaxedSimd, since computing in f32 (vs f64-then-demote) drops sub-ulp precision.
2467
+ if (ctx.laneType === 'f32') {
2468
+ if (op === 'f64.promote_f32') return liftExprV(expr[1], ctx)
2469
+ if (op === 'f32.demote_f64') return liftExprV(expr[1], ctx)
2470
+ // int→f32 widening load: `f64.convert_i32_{s,u}(<intload>(addr))` → load 4 ints,
2471
+ // widen to i32x4, f32x4.convert. i8/i16 are exact in f32; i32 rounds (gated).
2472
+ if ((op === 'f64.convert_i32_s' || op === 'f64.convert_i32_u') && isArr(expr[1]) && INT_WIDEN_F32[expr[1][0]]) {
2473
+ const ld = expr[1], w = INT_WIDEN_F32[ld[0]]
2474
+ if (w.lossy && !_relaxF32) return liftFail(ctx, `${ld[0]}→f32 SIMD rounds (i32 exceeds f32 mantissa) — needs relaxedSimd`)
2475
+ const addr = typeof ld[1] === 'string' && ld[1].startsWith('offset=') ? [w.load, ld[1], ld[2]] : [w.load, ld[1]]
2476
+ let v = addr
2477
+ for (const step of w.steps) v = [step, v]
2478
+ return [w.cvt === 'u' ? 'f32x4.convert_i32x4_u' : 'f32x4.convert_i32x4_s', v]
2479
+ }
2480
+ if (op === 'f64.const') {
2481
+ if (!_relaxF32) return liftFail(ctx, 'f64 constant in f32 lane needs relaxedSimd (f32 round of the constant)')
2482
+ return ['f32x4.splat', ['f32.const', expr[1]]]
2483
+ }
2484
+ const f32op = F64_TO_F32X4[op]
2485
+ if (f32op) {
2486
+ if (!_relaxF32) return liftFail(ctx, `${op}: f32 SIMD computes in f32 not f64 (sub-ulp) — needs relaxedSimd`)
2487
+ const a = liftExprV(expr[1], ctx); if (ctx.fail) return null
2488
+ if (expr.length === 2) return [f32op, a] // unary: neg / abs / sqrt
2489
+ const b = liftExprV(expr[2], ctx); if (ctx.fail) return null
2490
+ return [f32op, a, b]
2491
+ }
2492
+ }
2493
+
1917
2494
  // Loads → v128.load (preserving address, including any local.tee).
1918
2495
  if (LOAD_OPS[op]) {
1919
2496
  if (LOAD_OPS[op] !== ctx.laneType) return liftFail(ctx, `${op}: load type ≠ lane type ${ctx.laneType}`)
@@ -1942,10 +2519,18 @@ function liftExprV(expr, ctx) {
1942
2519
  }
1943
2520
  const kind = ctx.localKind.get(name)
1944
2521
  if (kind === 'lane') {
1945
- const { laneName } = getOrAllocLanedLocal(name, ctx)
2522
+ const laneName = getOrAllocLanedLocal(name, ctx.newLanedLocals)
1946
2523
  return ['local.get', laneName]
1947
2524
  }
1948
2525
  if (kind === 'invariant') {
2526
+ // An invariant whose wasm type ≠ the lane element type needs a scalar
2527
+ // convert before the splat. The common case: an f64 multiplier/gain splat
2528
+ // into an f32 lane (`out[i] = in[i] * k`). Demoting k to f32 is the same
2529
+ // precision relaxation as the f32 arithmetic itself, so gate it on relaxedSimd.
2530
+ if (ctx.laneType === 'f32' && ctx.fnLocals?.get(name) === 'f64') {
2531
+ if (!_relaxF32) return liftFail(ctx, `${name}: f64 invariant in f32 lane needs relaxedSimd`)
2532
+ return [info.splat, ['f32.demote_f64', ['local.get', name]]]
2533
+ }
1949
2534
  return [info.splat, ['local.get', name]]
1950
2535
  }
1951
2536
  if (kind === 'addr' || name === ctx.incVar) {
@@ -1974,19 +2559,44 @@ function liftExprV(expr, ctx) {
1974
2559
  // it). Both the flattened `select` form and the un-flattened `block` form
1975
2560
  // lift to a per-lane v128.bitselect — canonical value in NaN lanes, X
1976
2561
  // elsewhere — exactly reproducing the scalar canonicalization lane-by-lane.
1977
- if (ctx.laneType === 'f64' || ctx.laneType === 'f32') {
1978
- if (op === 'select') {
2562
+ if (op === 'select') {
2563
+ // NaN-canonicalization idiom `(select C X (T.ne X X))` — FLOAT lanes only (an
2564
+ // integer lane never carries it, since no i32 value is NaN).
2565
+ if (ctx.laneType === 'f64' || ctx.laneType === 'f32') {
1979
2566
  const m = matchCanonSelect(expr, ctx.laneType)
1980
- if (!m) return liftFail(ctx, 'non-canonical select (not a NaN-canon idiom)')
1981
- const coreV = liftExprV(m.val, ctx)
1982
- return ctx.fail ? null : liftCanon(coreV, m.C, ctx, info)
2567
+ if (m) {
2568
+ const coreV = liftExprV(m.val, ctx)
2569
+ return ctx.fail ? null : liftCanon(coreV, m.C, ctx, info)
2570
+ }
1983
2571
  }
1984
- if (op === 'block') {
1985
- const m = matchCanonBlock(expr, ctx.laneType)
1986
- if (!m) return liftFail(ctx, 'non-canonical value-block')
1987
- const coreV = liftExprV(m.core, ctx)
1988
- return ctx.fail ? null : liftCanon(coreV, m.C, ctx, info)
2572
+ // General `select(X, Y, COND)` (wasm: X if COND else Y) — jz lowers a value
2573
+ // ternary `COND ? X : Y` to this when both arms are cheap/pure. Lift to
2574
+ // v128.bitselect(X, Y, mask) like the `if` form below — valid for EVERY lane type
2575
+ // (i32 included: COND maps via LANE_COMPARE[laneType], NaN is irrelevant). Both
2576
+ // arms are lane-pure (recursion forbids stores/sets) and trap-free, so evaluating
2577
+ // both then selecting is sound. f32 lane promotes operands → f64.* compare → f32x4.
2578
+ if (expr.length === 4) {
2579
+ const cond = expr[3]
2580
+ const cmpOp = isArr(cond) && ctx.laneType === 'f32' && typeof cond[0] === 'string' && cond[0].startsWith('f64.') ? 'f32.' + cond[0].slice(4) : (isArr(cond) ? cond[0] : null)
2581
+ const cmpSimd = cmpOp && cond.length === 3 ? LANE_COMPARE[ctx.laneType]?.[cmpOp] : null
2582
+ if (!cmpSimd) return liftFail(ctx, `select condition ${isArr(cond) ? cond[0] : '?'} not a lane comparison`)
2583
+ const x = liftExprV(expr[1], ctx); if (ctx.fail) return null
2584
+ const y = liftExprV(expr[2], ctx); if (ctx.fail) return null
2585
+ const ca = liftExprV(cond[1], ctx); if (ctx.fail) return null
2586
+ const cb = liftExprV(cond[2], ctx); if (ctx.fail) return null
2587
+ const mtmp = `$__mask${ctx.freshIdRef.next++}`
2588
+ ctx.extraLocals.push(['local', mtmp, 'v128'])
2589
+ return ['block', ['result', 'v128'],
2590
+ ['local.set', mtmp, [cmpSimd, ca, cb]],
2591
+ ['v128.bitselect', x, y, ['local.get', mtmp]]]
1989
2592
  }
2593
+ return liftFail(ctx, 'non-canonical select (not a NaN-canon idiom)')
2594
+ }
2595
+ if ((ctx.laneType === 'f64' || ctx.laneType === 'f32') && op === 'block') {
2596
+ const m = matchCanonBlock(expr, ctx.laneType)
2597
+ if (!m) return liftFail(ctx, 'non-canonical value-block')
2598
+ const coreV = liftExprV(m.core, ctx)
2599
+ return ctx.fail ? null : liftCanon(coreV, m.C, ctx, info)
1990
2600
  }
1991
2601
 
1992
2602
  // Conditional select — jz lowers `cond ? X : Y` to (if (result LT) COND (then X)
@@ -1998,13 +2608,21 @@ function liftExprV(expr, ctx) {
1998
2608
  // evaluates X,Y before its 3rd operand, but any address `local.tee` lives in
1999
2609
  // COND and must run before the branches read it (matching scalar order).
2000
2610
  if (op === 'if') {
2001
- if (!isArr(expr[1]) || expr[1][0] !== 'result' || expr[1][1] !== ctx.laneType) return liftFail(ctx, 'conditional without lane-typed result')
2611
+ // jz lowers `cond ? X : Y` to (if (result T) COND (then X)(else Y)). In an f32
2612
+ // lane it computes in f64 (promote/demote around the store), so the `if` carries
2613
+ // `(result f64)` and COND is an `f64.*` compare — accept both, mapping to f32x4.
2614
+ // The branch values are f32-mapped by recursion; gated by relaxedSimd via those.
2615
+ const resTy = isArr(expr[1]) && expr[1][0] === 'result' ? expr[1][1] : null
2616
+ if (resTy !== ctx.laneType && !(ctx.laneType === 'f32' && resTy === 'f64')) return liftFail(ctx, 'conditional without lane-typed result')
2002
2617
  const thenN = expr[3], elseN = expr[4]
2003
2618
  if (!isArr(thenN) || thenN[0] !== 'then' || thenN.length !== 2) return liftFail(ctx, 'malformed conditional then-branch')
2004
2619
  if (!isArr(elseN) || elseN[0] !== 'else' || elseN.length !== 2) return liftFail(ctx, 'malformed conditional else-branch')
2005
2620
  let cond = expr[2]
2006
2621
  if (isArr(cond) && cond[0] === 'i32.ne' && isI32Const(cond[2]) && cond[2][1] === 0) cond = cond[1] // strip `!= 0`
2007
- const cmpSimd = isArr(cond) && cond.length === 3 ? LANE_COMPARE[ctx.laneType]?.[cond[0]] : null
2622
+ // f32 lane: operands were promoted, so the compare is `f64.*` — use its f32x4 form
2623
+ // (operands are exact f32→f64 promotions, so the lane comparison is unchanged).
2624
+ const cmpOp = isArr(cond) && ctx.laneType === 'f32' && typeof cond[0] === 'string' && cond[0].startsWith('f64.') ? 'f32.' + cond[0].slice(4) : (isArr(cond) ? cond[0] : null)
2625
+ const cmpSimd = cmpOp && cond.length === 3 ? LANE_COMPARE[ctx.laneType]?.[cmpOp] : null
2008
2626
  if (!cmpSimd) return liftFail(ctx, `${isArr(cond) ? cond[0] : 'condition'}: not a lane-vectorizable comparison`)
2009
2627
  const ca = liftExprV(cond[1], ctx); if (ctx.fail) return null
2010
2628
  const cb = liftExprV(cond[2], ctx); if (ctx.fail) return null
@@ -2510,15 +3128,17 @@ function tryRampMap(blockNode, fnLocals, freshIdRef) {
2510
3128
  // pointer is still expressed in terms of the IV. We keep the address verbatim
2511
3129
  // (scalar i32) and advance the IV by LANES, so `base + (i<<K)` lands on the
2512
3130
  // next group's first element each SIMD step — for any element width.
2513
- let storeStmt = null, storeIdx = -1
3131
+ // Collect every store. One store → the original single-map paths (ramp pack / widening /
3132
+ // 4-wide). Two or more independent store8s → a multi-channel in-place fade (boids' 4-channel
3133
+ // u8 trail), handled by the multi-store WIDEN16 branch below; one pass over memory, N widening
3134
+ // stores. Stores beyond the first don't reach the single-store paths.
3135
+ const storeStmts = []
2514
3136
  for (let i = 0; i < body.length; i++) {
2515
3137
  const s = body[i]
2516
- if (isArr(s) && STORE_OPS[s[0]]) {
2517
- if (storeStmt) return null
2518
- storeStmt = s; storeIdx = i
2519
- }
3138
+ if (isArr(s) && STORE_OPS[s[0]]) storeStmts.push({ stmt: s, idx: i })
2520
3139
  }
2521
- if (!storeStmt) return null
3140
+ if (!storeStmts.length) return null
3141
+ const storeStmt = storeStmts[0].stmt, storeIdx = storeStmts[0].idx
2522
3142
  const storeOp = storeStmt[0]
2523
3143
  if (storeStmt.length !== 3) return null
2524
3144
  const elemLog2 = { 'i32.store8': 0, 'i32.store': 2 }[storeOp]
@@ -2535,8 +3155,24 @@ function tryRampMap(blockNode, fnLocals, freshIdRef) {
2535
3155
  for (const s of body) gatherNames(s)
2536
3156
  for (const name of allNames) { const k = _offsetLocalStride(body, name, ivName); if (k != null) offsetTees.set(name, k) }
2537
3157
 
3158
+ // CSE'd FULL lane address: an in-place map `a[i] = f(a[i])` shares one `(local.tee $A
3159
+ // (i32.add base i))` between the load and the store, reused as `(local.get $A)`. Without
3160
+ // resolving it the store/load address matchers reject the bare get (the empty-addrLocals
3161
+ // bug that kept every in-place trail-fade scalar). Record each such tee — its lifted
3162
+ // address (the tee) runs in the hoisted v128.load, so the store's get reads it back.
3163
+ const addrLocals = new Map()
3164
+ const recordAddrTees = (n) => {
3165
+ if (!isArr(n)) return
3166
+ if (n[0] === 'local.tee' && typeof n[1] === 'string' && isArr(n[2]) && n[2][0] === 'i32.add') {
3167
+ const m = matchLaneAddr(n[2], ivName, addrLocals, offsetTees)
3168
+ if (m && m.teeName == null) addrLocals.set(n[1], { strideLog2: m.strideLog2, base: m.base })
3169
+ }
3170
+ for (let i = 1; i < n.length; i++) recordAddrTees(n[i])
3171
+ }
3172
+ for (const s of body) recordAddrTees(s)
3173
+
2538
3174
  const storeAddr = storeStmt[1]
2539
- const addrM = matchLaneAddr(storeAddr, ivName, new Map(), offsetTees)
3175
+ const addrM = matchLaneAddr(storeAddr, ivName, addrLocals, offsetTees)
2540
3176
  if (!addrM || addrM.strideLog2 !== elemLog2) return null
2541
3177
 
2542
3178
  // Memory loads turn this into a widening byte-map: out[i] = narrow(f(widen(a[i])…)).
@@ -2549,7 +3185,7 @@ function tryRampMap(blockNode, fnLocals, freshIdRef) {
2549
3185
  if (LOAD_OPS[n[0]]) {
2550
3186
  hasLoads = true
2551
3187
  if (storeOp !== 'i32.store8' || n[0] !== 'i32.load8_u') { loadsOk = false; return }
2552
- const m = matchLaneAddr(n[1], ivName, new Map(), offsetTees)
3188
+ const m = matchLaneAddr(n[1], ivName, addrLocals, offsetTees)
2553
3189
  if (!m || m.strideLog2 !== 0) loadsOk = false
2554
3190
  return // address validated; the IV-strided subtree is not data
2555
3191
  }
@@ -2588,7 +3224,7 @@ function tryRampMap(blockNode, fnLocals, freshIdRef) {
2588
3224
  const newLanedLocals = new Map()
2589
3225
  const extraLocals = []
2590
3226
  const freshV128 = (tag) => { const n = `$__${tag}${freshIdRef.next++}`; extraLocals.push(['local', n, 'v128']); return n }
2591
- const ctx = { laneType: 'i32', incVar: ivName, rampVar: ivName, rampTemp: null, widenLoads: true, localKind, newLanedLocals, extraLocals, freshIdRef, fail: false, failReason: null }
3227
+ const ctx = { laneType: 'i32', incVar: ivName, rampVar: ivName, rampTemp: null, widenLoads: true, localKind, fnLocals: null, newLanedLocals, extraLocals, freshIdRef, fail: false, failReason: null }
2592
3228
 
2593
3229
  // A byte store fed by one value expression (inline, or via a single lane-local
2594
3230
  // temp `tw = EXPR; store(addr, tw)`) carries no loop-carried state, so we can
@@ -2631,21 +3267,12 @@ function tryRampMap(blockNode, fnLocals, freshIdRef) {
2631
3267
  else return null
2632
3268
  return (r[0] < 0 || r[1] > 65535) ? null : r
2633
3269
  }
2634
- const wideValueExpr = (!hasLoads && byteValueExpr) ? byteValueExpr : null // pure ramp → 16-wide pack
2635
- const widenRange = (hasLoads && byteValueExpr) ? byteValueRange(byteValueExpr) : null
2636
- const WIDEN16 = widenRange != null && widenRange[1] <= 255 // result fits a byte narrow_u exact
2637
- const WIDE16 = wideValueExpr != null
2638
- const LANES = (WIDE16 || WIDEN16) ? 16 : 4
2639
- const ramp = (off) => ['i32x4.add', ['i32x4.splat', ['local.get', ivName]],
2640
- ['v128.const', 'i32x4', String(off), String(off + 1), String(off + 2), String(off + 3)]]
2641
-
2642
- let lifted
2643
- if (WIDEN16) {
2644
- // 16-wide widening byte-map: load each u8 input once (v128.load of 16 bytes),
2645
- // extend_low/high to two i16x8 halves, run the affine map in i16x8 on each half,
2646
- // narrow_u the two result halves to one i8x16, store 16. Each load is hoisted to a
2647
- // temp (so extend_low + extend_high share one load); the loads run in source order,
2648
- // so the offset `local.tee` in the first one is set before later loads read it.
3270
+ // The i16x8 widening byte-map emit for ONE store, factored so a multi-channel fade reuses it
3271
+ // per channel: load each u8 input once (v128.load 16), extend_low/high to two i16x8 halves,
3272
+ // run the affine map in i16x8 on each half, narrow_u to i8x16, store 16. Each load is hoisted
3273
+ // (extend_low + extend_high share one load); loads run in source order, so an offset/address
3274
+ // `local.tee` in a load is set before the store's `local.get` reads it.
3275
+ const widen16Emit = (sAddr, valueExpr) => {
2649
3276
  const loadTemps = new Map()
2650
3277
  const loadSets = []
2651
3278
  const collectLoads = (e) => {
@@ -2655,7 +3282,7 @@ function tryRampMap(blockNode, fnLocals, freshIdRef) {
2655
3282
  if (!loadTemps.has(k)) { const t = freshV128('win'); loadTemps.set(k, t); loadSets.push(['local.set', t, ['v128.load', e[1]]]) }
2656
3283
  } else for (let i = 1; i < e.length; i++) collectLoads(e[i])
2657
3284
  }
2658
- collectLoads(byteValueExpr)
3285
+ collectLoads(valueExpr)
2659
3286
  const liftW = (e, half) => {
2660
3287
  const op = e[0]
2661
3288
  if (op === 'i32.const') return ['i16x8.splat', e]
@@ -2667,45 +3294,87 @@ function tryRampMap(blockNode, fnLocals, freshIdRef) {
2667
3294
  if (op === 'i32.and') return ['v128.and', liftW(e[1], half), ['i16x8.splat', e[2]]]
2668
3295
  return null // byteValueRange already proved every op is one of the above
2669
3296
  }
2670
- lifted = [...loadSets,
2671
- ['v128.store', storeAddr, ['i8x16.narrow_i16x8_u', liftW(byteValueExpr, 'low'), liftW(byteValueExpr, 'high')]]]
2672
- } else if (WIDE16) {
3297
+ return [...loadSets,
3298
+ ['v128.store', sAddr, ['i8x16.narrow_i16x8_u', liftW(valueExpr, 'low'), liftW(valueExpr, 'high')]]]
3299
+ }
3300
+
3301
+ let lifted, LANES
3302
+ if (storeStmts.length > 1) {
3303
+ // MULTI-CHANNEL in-place byte fade: N independent store8s in one loop (boids' 4-channel u8
3304
+ // trail). Each store must be a u8 store of a WIDEN16-eligible value (range ≤ 255); it emits
3305
+ // its own load→i16x8→narrow→store sequence, all concatenated into ONE 16-wide pass — so the
3306
+ // memory traffic stays single-pass (vs N separate vectorized loops). Every body statement
3307
+ // must be a store or the lane-local set feeding one; any other (shared/invariant) compute
3308
+ // bails to scalar, since it would be dropped.
3309
+ LANES = 16
2673
3310
  lifted = []
2674
- const vv = []
2675
- for (let j = 0; j < 4; j++) {
2676
- const rt = freshV128('ramp')
2677
- lifted.push(['local.set', rt, ramp(j * 4)])
2678
- ctx.rampTemp = rt
2679
- const v = liftExprV(wideValueExpr, ctx)
2680
- if (ctx.fail) return null
2681
- const vn = freshV128('rampv')
2682
- lifted.push(['local.set', vn, v])
2683
- vv.push(vn)
2684
- }
2685
- // Pack the low byte of all 16 i32 lanes (4 vectors) into one i8x16, in order.
2686
- const g = (n) => ['local.get', n]
2687
- const sh = (a, b, idx) => ['i8x16.shuffle', ...idx.map(String), a, b]
2688
- const lo = freshV128('ramplo'), hi = freshV128('ramphi')
2689
- lifted.push(['local.set', lo, sh(g(vv[0]), g(vv[1]), [0, 4, 8, 12, 16, 20, 24, 28, 0, 0, 0, 0, 0, 0, 0, 0])])
2690
- lifted.push(['local.set', hi, sh(g(vv[2]), g(vv[3]), [0, 4, 8, 12, 16, 20, 24, 28, 0, 0, 0, 0, 0, 0, 0, 0])])
2691
- lifted.push(['v128.store', storeAddr, sh(g(lo), g(hi), [0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23])])
3311
+ const consumed = new Set()
3312
+ for (const { stmt, idx } of storeStmts) {
3313
+ if (stmt[0] !== 'i32.store8' || stmt.length !== 3) return null
3314
+ const a = matchLaneAddr(stmt[1], ivName, addrLocals, offsetTees)
3315
+ if (!a || a.strideLog2 !== 0) return null
3316
+ consumed.add(idx)
3317
+ let val = stmt[2]
3318
+ if (isArr(val) && val[0] === 'local.get' && typeof val[1] === 'string' && localKind.get(val[1]) === 'lane') {
3319
+ let setIdx = -1
3320
+ for (let j = 0; j < idx; j++) { const s = body[j]; if (isArr(s) && s[0] === 'local.set' && s[1] === val[1] && s.length === 3) { setIdx = j; val = s[2] } }
3321
+ if (setIdx < 0) return null
3322
+ consumed.add(setIdx)
3323
+ }
3324
+ const rng = byteValueRange(val)
3325
+ if (!rng || rng[1] > 255) return null // not WIDEN16-eligible (overflows u16 or u8) → scalar
3326
+ lifted.push(...widen16Emit(stmt[1], val))
3327
+ }
3328
+ if (consumed.size !== body.length) return null
2692
3329
  } else {
2693
- ctx.rampTemp = freshV128('ramp')
2694
- // ramp = [i, i+1, i+2, i+3], computed once per SIMD iteration.
2695
- lifted = [['local.set', ctx.rampTemp, ramp(0)]]
2696
- for (let i = 0; i < body.length; i++) {
2697
- if (i === storeIdx) {
2698
- const vval = liftExprV(storeStmt[2], ctx)
3330
+ const wideValueExpr = (!hasLoads && byteValueExpr) ? byteValueExpr : null // pure ramp → 16-wide pack
3331
+ const widenRange = (hasLoads && byteValueExpr) ? byteValueRange(byteValueExpr) : null
3332
+ const WIDEN16 = widenRange != null && widenRange[1] <= 255 // result fits a byte ⇒ narrow_u exact
3333
+ const WIDE16 = wideValueExpr != null
3334
+ LANES = (WIDE16 || WIDEN16) ? 16 : 4
3335
+ const ramp = (off) => ['i32x4.add', ['i32x4.splat', ['local.get', ivName]],
3336
+ ['v128.const', 'i32x4', String(off), String(off + 1), String(off + 2), String(off + 3)]]
3337
+
3338
+ if (WIDEN16) {
3339
+ lifted = widen16Emit(storeAddr, byteValueExpr)
3340
+ } else if (WIDE16) {
3341
+ lifted = []
3342
+ const vv = []
3343
+ for (let j = 0; j < 4; j++) {
3344
+ const rt = freshV128('ramp')
3345
+ lifted.push(['local.set', rt, ramp(j * 4)])
3346
+ ctx.rampTemp = rt
3347
+ const v = liftExprV(wideValueExpr, ctx)
2699
3348
  if (ctx.fail) return null
2700
- lifted.push(buildRampStore(storeOp, storeAddr, vval, ctx))
2701
- } else {
2702
- const r = liftStmt(body[i], ctx)
2703
- if (ctx.fail) return null
2704
- if (r != null) { if (Array.isArray(r) && r[0] === '__seq__') lifted.push(...r.slice(1)); else lifted.push(r) }
3349
+ const vn = freshV128('rampv')
3350
+ lifted.push(['local.set', vn, v])
3351
+ vv.push(vn)
3352
+ }
3353
+ // Pack the low byte of all 16 i32 lanes (4 vectors) into one i8x16, in order.
3354
+ const g = (n) => ['local.get', n]
3355
+ const sh = (a, b, idx) => ['i8x16.shuffle', ...idx.map(String), a, b]
3356
+ const lo = freshV128('ramplo'), hi = freshV128('ramphi')
3357
+ lifted.push(['local.set', lo, sh(g(vv[0]), g(vv[1]), [0, 4, 8, 12, 16, 20, 24, 28, 0, 0, 0, 0, 0, 0, 0, 0])])
3358
+ lifted.push(['local.set', hi, sh(g(vv[2]), g(vv[3]), [0, 4, 8, 12, 16, 20, 24, 28, 0, 0, 0, 0, 0, 0, 0, 0])])
3359
+ lifted.push(['v128.store', storeAddr, sh(g(lo), g(hi), [0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23])])
3360
+ } else {
3361
+ ctx.rampTemp = freshV128('ramp')
3362
+ // ramp = [i, i+1, i+2, i+3], computed once per SIMD iteration.
3363
+ lifted = [['local.set', ctx.rampTemp, ramp(0)]]
3364
+ for (let i = 0; i < body.length; i++) {
3365
+ if (i === storeIdx) {
3366
+ const vval = liftExprV(storeStmt[2], ctx)
3367
+ if (ctx.fail) return null
3368
+ lifted.push(buildRampStore(storeOp, storeAddr, vval, ctx))
3369
+ } else {
3370
+ const r = liftStmt(body[i], ctx)
3371
+ if (ctx.fail) return null
3372
+ if (r != null) { if (Array.isArray(r) && r[0] === '__seq__') lifted.push(...r.slice(1)); else lifted.push(r) }
3373
+ }
2705
3374
  }
2706
3375
  }
2707
3376
  }
2708
- if (!lifted.length) return null
3377
+ if (!lifted || !lifted.length) return null
2709
3378
 
2710
3379
  const id = freshIdRef.next++
2711
3380
  const simdBoundName = `$__simd_bound${id}`
@@ -2726,7 +3395,7 @@ function tryRampMap(blockNode, fnLocals, freshIdRef) {
2726
3395
  const wrapper = ['block', boundSetup, simdBlock, blockNode]
2727
3396
  const newLocalDecls = [
2728
3397
  ['local', simdBoundName, 'i32'],
2729
- ...[...newLanedLocals.values()].map(({ laneName }) => ['local', laneName, 'v128']),
3398
+ ...[...newLanedLocals.values()].map(laneName => ['local', laneName, 'v128']),
2730
3399
  ...extraLocals,
2731
3400
  ]
2732
3401
  return { wrapper, newLocalDecls }
@@ -2736,6 +3405,62 @@ function tryRampMap(blockNode, fnLocals, freshIdRef) {
2736
3405
  // `storeOp` at scalar address `addr`. i32.store is the full vector; i32.store8
2737
3406
  // truncates (low byte of each lane) via i8x16.shuffle — exactly matching scalar
2738
3407
  // store8, with no value-range assumption (shuffle selects, never saturates).
3408
+ // Narrowing-map store: pack a wider float lane vector `val` down to a narrower
3409
+ // store element and write the low bytes (extract_lane 0 + scalar store, like
3410
+ // buildRampStore). f64→f32 demotes (bit-exact vs scalar); f64→i32 truncates;
3411
+ // f32→i16/i8 truncate to i32x4 then WRAP via i8x16.shuffle (low bytes = scalar
3412
+ // store{8,16}, never saturates). Returns the store stmt or null (unsupported).
3413
+ // Peel the scalar narrowing conversion off a store value, returning the inner float
3414
+ // expr to lift (narrowStore then applies the SIMD narrow). f32 store: f32.demote_f64(X).
3415
+ // int store: the ToInt32 idiom `i32.wrap_i64(X<0 ? trunc_sat_f64_s X : trunc_sat_f64_u X)`
3416
+ // (or a bare trunc_sat). The inner X is the f64/f32 lane value computed before the cast.
3417
+ function peelNarrowConv(val, sty) {
3418
+ if (!isArr(val)) return null
3419
+ if (sty === 'f32') return val[0] === 'f32.demote_f64' ? val[1] : null
3420
+ // int element (i8/i16/i32): peel ToInt32 (`x | 0`). jz's general lowering is an
3421
+ // Infinity-guarded saturating trunc:
3422
+ // (select (i32.wrap_i64 (i64.trunc_sat_f64_s X)) (i32.const 0) (f64.ne X' Inf))
3423
+ // where X is `(local.tee $inf <f64 expr>)` and X' the matching get. Peel to the inner f64.
3424
+ // (The SIMD narrow i32x4.trunc_sat_f64x2_s_zero saturates +Inf / |x|≥2³¹ to INT_MAX where
3425
+ // ToInt32 wraps mod 2³² — caller gates the int narrowing on relaxedSimd for that edge.)
3426
+ if (val[0] === 'select' && val.length === 4 && isI32Const(val[2]) && val[2][1] === 0 &&
3427
+ isArr(val[1]) && val[1][0] === 'i32.wrap_i64' && isArr(val[1][1]) && val[1][1][0] === 'i64.trunc_sat_f64_s') {
3428
+ let inner = val[1][1][1] // the f64 operand of the trunc, captured in a `(local.tee $inf …)`
3429
+ if (isArr(inner) && inner[0] === 'local.tee' && inner.length === 3) inner = inner[2] // peel to the tee's VALUE
3430
+ return inner
3431
+ }
3432
+ if (val[0] === 'i32.wrap_i64' && isArr(val[1]) && val[1][0] === 'if') {
3433
+ const iff = val[1], thenA = iff[3], elseA = iff[4]
3434
+ const s = isArr(thenA) && isArr(thenA[1]) && thenA[1][0] === 'i64.trunc_sat_f64_s' ? thenA[1][1] : null
3435
+ const u = isArr(elseA) && isArr(elseA[1]) && elseA[1][0] === 'i64.trunc_sat_f64_u' ? elseA[1][1] : null
3436
+ if (s && u && exprEq(s, u)) return s
3437
+ }
3438
+ if (val[0] === 'i32.trunc_sat_f64_s' || val[0] === 'i32.trunc_sat_f64_u') return val[1]
3439
+ return null
3440
+ }
3441
+
3442
+ // i8x16.shuffle masks that pack a 4×i32 vector down to the low bytes of each lane
3443
+ // (truncating-wrap = scalar store{8,16}), tail zero-filled: _I16 keeps the low 2
3444
+ // bytes of each lane (→ 8 bytes / i64.store), _I8 the low byte (→ 4 bytes / i32.store).
3445
+ const PACK_I32_TO_I16 = [0, 1, 4, 5, 8, 9, 12, 13, 0, 0, 0, 0, 0, 0, 0, 0]
3446
+ const PACK_I32_TO_I8 = [0, 4, 8, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
3447
+
3448
+ function narrowStore(addr, val, laneType, sty, ctx) {
3449
+ const tmp = `$__nv${ctx.freshIdRef.next++}`
3450
+ ctx.extraLocals.push(['local', tmp, 'v128'])
3451
+ const g = ['local.get', tmp]
3452
+ const sh = (idx) => ['i8x16.shuffle', ...idx.map(String), g, g]
3453
+ let pre, lane8, store
3454
+ if (laneType === 'f64' && sty === 'f32') { pre = ['f32x4.demote_f64x2_zero', val]; lane8 = g; store = 'i64.store' }
3455
+ else if (laneType === 'f64' && sty === 'i32') { pre = ['i32x4.trunc_sat_f64x2_s_zero', val]; lane8 = g; store = 'i64.store' }
3456
+ else if (laneType === 'f32' && sty === 'i16') { pre = ['i32x4.trunc_sat_f32x4_s', val]; lane8 = sh(PACK_I32_TO_I16); store = 'i64.store' }
3457
+ else if (laneType === 'f32' && sty === 'i8') { pre = ['i32x4.trunc_sat_f32x4_s', val]; lane8 = sh(PACK_I32_TO_I8); store = 'i32.store' }
3458
+ else return null
3459
+ // 8-byte stores extract an i64 lane; the 4-byte i8 pack extracts an i32 lane.
3460
+ const packed = store === 'i64.store' ? ['i64x2.extract_lane', 0, lane8] : ['i32x4.extract_lane', 0, lane8]
3461
+ return ['block', ['local.set', tmp, pre], [store, addr, packed]]
3462
+ }
3463
+
2739
3464
  function buildRampStore(storeOp, addr, vval, ctx) {
2740
3465
  if (storeOp === 'i32.store') return ['v128.store', addr, vval] // 4 i32 lanes → 16 bytes
2741
3466
  // i32.store8: hoist vval to a temp so the shuffle reads it once; low byte of
@@ -3726,8 +4451,8 @@ function tryDivergentEscapeVectorize(blockNode, fnLocals, freshIdRef) {
3726
4451
  // public `$math.{sin,cos}` wrap the same core. Their f64x2 mirrors $math.sin2/$math.cos2 (the
3727
4452
  // vectorized reduce+horner, module/math.js:543) are BIT-EXACT per lane to the scalar core — so we
3728
4453
  // can lift the call straight to the *2 helper. Phase-2 adds pow/log/atan2 here (see PPC_CALL2).
3729
- // NOTE: scalar targets here must be kept out of inlineOnce's single-caller inlining (SIMD_PROTECTED
3730
- // in src/wat/optimize.js) else the call node is gone before this lift runs.
4454
+ // NOTE: scalar targets here must be kept out of watr's single-caller inlining — jz passes these
4455
+ // keys (SIMD_PINNED, below) as watOptimize's `pin` list, else the call node is gone before this lift runs.
3731
4456
  const PPC_CALL2 = {
3732
4457
  '$math.sin_core': '$math.sin2', '$math.cos_core': '$math.cos2',
3733
4458
  '$math.sin': '$math.sin2', '$math.cos': '$math.cos2',
@@ -3738,6 +4463,11 @@ const PPC_CALL2 = {
3738
4463
  '$math.log': '$math.log_v', '$math.exp': '$math.exp_v', '$math.exp2': '$math.exp2_v',
3739
4464
  }
3740
4465
 
4466
+ // Scalar transcendentals the auto-vectorizer rewrites to f64x2 mirrors above. watr's inline
4467
+ // passes must NOT dissolve their (single-caller) call nodes before this lift runs — jz passes
4468
+ // these to watOptimize's `pin` option (the protection policy lives here, not hardcoded in watr).
4469
+ export const SIMD_PINNED = Object.keys(PPC_CALL2)
4470
+
3741
4471
  // Per-pixel-color vectorizer. The dual of tryDivergentEscapeVectorize for kernels with NO inner
3742
4472
  // escape loop: an outer pixel loop whose body computes an f64 value from the pixel index (via
3743
4473
  // cos/sin/sqrt/…), packs it to a u32 colour, and stores it — every pixel independent. We lift the
@@ -4134,13 +4864,19 @@ function tryOuterStrip(blockNode, fnLocals, freshIdRef, enabled) {
4134
4864
  // accumulator seeds (→ splat), scalar inner-IV init. MUST run before the inner-body lift so the
4135
4865
  // per-pixel coord lanes are registered when the inner loop references them. ----
4136
4866
  const laneInit = []
4867
+ const seededAccs = new Set()
4137
4868
  for (let i = 0; i < innerIdx; i++) {
4138
4869
  const s = obody[i]
4139
4870
  if (!(isArr(s) && s[0] === 'local.set' && typeof s[1] === 'string' && s.length === 3)) { laneInit.push(s); continue }
4140
4871
  const name = s[1]
4141
4872
  if (accNames.has(name)) { // accumulator seed → splat
4873
+ // The seed must be a FRESH per-pixel value, independent of the accumulator's own carry.
4874
+ // A seed that reads `name` (e.g. `acc = acc * decay`) propagates the previous pixel's
4875
+ // running value across pixels — that's a loop-carried recurrence, not a per-pixel reset.
4876
+ if (readsName(s[2], name)) return null
4142
4877
  const seed = liftOS(s[2])
4143
4878
  if (!seed) return null
4879
+ seededAccs.add(name)
4144
4880
  laneInit.push(['local.set', laneMap.get(name), seed]); continue
4145
4881
  }
4146
4882
  if (fnLocals.get(name) === 'f64' && readsName(s[2], pxVar)) { // per-pixel coord (cx = xi/W) → ramp lane
@@ -4153,6 +4889,14 @@ function tryOuterStrip(blockNode, fnLocals, freshIdRef, enabled) {
4153
4889
  laneInit.push(s) // scalar (inner IV init `b=0`, invariant setup)
4154
4890
  }
4155
4891
 
4892
+ // LEGALITY: every accumulator must be FRESHLY SEEDED inside the outer-loop body. An accumulator
4893
+ // with no per-pixel seed is live-in — carried across outer iterations (a recurrence like the
4894
+ // lorenz `x = x + S·(…)` evolving over the sample loop). The two pixel lanes are then DEPENDENT:
4895
+ // lane xi+1 must continue from lane xi's final value, not restart from a splat of the shared
4896
+ // carry. Strip-mining it runs both lanes from the same seed in lockstep — halving the real work
4897
+ // and producing a wrong result (a bogus speedup on a serial recurrence). Reject.
4898
+ for (const a of accNames) if (!seededAccs.has(a)) return null
4899
+
4156
4900
  // ---- lift the inner-loop body: temp f64 lane locals + accumulate into the acc shadows; the
4157
4901
  // inner IV bump stays scalar. Per-pixel coords now resolve via laneMap. ----
4158
4902
  const liftedInner = []
@@ -4218,6 +4962,339 @@ function tryOuterStrip(blockNode, fnLocals, freshIdRef, enabled) {
4218
4962
  return { wrapper, newLocalDecls }
4219
4963
  }
4220
4964
 
4965
+ // ---- Per-pixel iterated-map reduction (tryIteratedReduce, experimental) ----------------------
4966
+ //
4967
+ // Generalizes the outer-strip to the ITERATED-MAP fractal shape — lyapunov, bifurcation, smooth-
4968
+ // escape attractors — whose per-pixel value runs a recurrence many times and accumulates a
4969
+ // transcendental. Beyond tryOuterStrip (one inner loop, a plain additive accumulator) it handles:
4970
+ // • MULTIPLE inner loops carrying per-pixel f64 state between them (lyapunov warmup → accumulate),
4971
+ // • loop-carried f64 RECURRENCES x = r·x·(1−x) (not just acc = acc + …),
4972
+ // • lane-invariant scalar bookkeeping kept SCALAR — integer counters with wraparound and the
4973
+ // forcing-sequence gather seq[si] (same index for both lanes → one scalar load),
4974
+ // • a scalar-condition select seq[si]<1 ? a : b (a ramps per lane, b splats) → a scalar
4975
+ // `if (result v128)`, and a per-lane conditional accumulate if(d>0) L += log(d) → bitselect.
4976
+ // Two adjacent pixels (xi, xi+1) run as f64x2 lanes; the colour pack+store runs scalar per lane,
4977
+ // and the original scalar loop, kept as the tail, finishes the odd last pixel.
4978
+ //
4979
+ // BIT-EXACT: f64x2 arithmetic is per-lane IEEE-identical, $math.log_v/exp_v are the per-lane mirrors
4980
+ // of the scalar polys, and the conditional accumulate adds bitselect(f(x), 0, mask) — exactly the
4981
+ // scalar add-or-skip. The speculatively-evaluated transcendental of a masked-out lane is discarded
4982
+ // (the helpers never trap). Gated behind cfg.experimentalOuterStrip; only fires when an inner loop
4983
+ // carries a transcendental (the latency-bound work SIMD actually accelerates — cheap-arithmetic
4984
+ // pixel loops are left to the scalar JIT, which already pipelines independent iterations).
4985
+ function tryIteratedReduce(blockNode, fnLocals, freshIdRef, enabled) {
4986
+ if (!enabled) return null
4987
+ const outer = matchOuterPixelLoop(blockNode)
4988
+ if (!outer) return null
4989
+ const { oLabel, loopNode, preamble, pixelIVs, pxVar, widthBound, pivType, obody, oExit } = outer
4990
+
4991
+ const innerIdxs = []
4992
+ for (let i = 0; i < obody.length; i++) {
4993
+ const s = obody[i]
4994
+ if (isArr(s) && s[0] === 'block' && s.slice(1).some(c => isArr(c) && c[0] === 'loop')) innerIdxs.push(i)
4995
+ }
4996
+ if (!innerIdxs.length) return null
4997
+ const lastInner = innerIdxs[innerIdxs.length - 1]
4998
+ const innerSet = new Set(innerIdxs)
4999
+
5000
+ const impureCall = (n) => isArr(n) && ((n[0] === 'call' && typeof n[1] === 'string' && !n[1].startsWith('$math.')) || n.some(impureCall))
5001
+ if (obody.some(impureCall)) return null
5002
+
5003
+ const id = freshIdRef.next++
5004
+ const nm = (s) => `$__ir${id}_${s}`
5005
+ const laneMap = new Map() // f64 per-pixel local → its v128 shadow
5006
+ const shadowOf = (v) => { let s = laneMap.get(v); if (!s) { s = nm(v.replace(/\W/g, '')); laneMap.set(v, s) } return s }
5007
+ let sawHeavy = false // a transcendental lifted inside a loop → SIMD is worth it
5008
+
5009
+ const bump = (n, k) => k === 0 ? n
5010
+ : (isArr(n) && n[0] === 'local.get' && pivType.has(n[1])) ? [pivType.get(n[1]) + '.add', n, [pivType.get(n[1]) + '.const', k]]
5011
+ : (isArr(n) ? n.map(c => bump(c, k)) : n)
5012
+ const rampOf = (piv) => pivType.get(piv) === 'f64'
5013
+ ? ['f64x2.replace_lane', 1, ['f64x2.splat', ['local.get', piv]], ['f64.add', ['local.get', piv], ['f64.const', 1]]]
5014
+ : ['f64x2.replace_lane', 1, ['f64x2.splat', ['f64.convert_i32_s', ['local.get', piv]]], ['f64.convert_i32_s', ['i32.add', ['local.get', piv], ['i32.const', 1]]]]
5015
+ const readsName = (n, name) => isArr(n) && ((n[0] === 'local.get' && n[1] === name) || n.some(c => readsName(c, name)))
5016
+ // Lane-invariant: reads no per-pixel lane local and no pixel IV → identical value in both lanes.
5017
+ const laneInvariant = (n) => !isArr(n) ? true
5018
+ : n[0] === 'local.get' ? !(laneMap.has(n[1]) || pivType.has(n[1]))
5019
+ : n.slice(1).every(laneInvariant)
5020
+
5021
+ // Build the f64x2 form of `cond ? x : y` from already-lifted arms `x`,`y` and the raw `cond`.
5022
+ // A lane-INVARIANT cond (same both lanes — e.g. seq[si]<1) → a v128-typed scalar branch; a
5023
+ // per-lane f64 compare → bitselect (x where cond, y elsewhere).
5024
+ const liftSelect = (x, y, cond) => {
5025
+ if (!x || !y) return null
5026
+ if (isArr(cond) && cond[0] === 'i32.ne' && isI32Const(cond[2]) && cond[2][1] === 0) cond = cond[1]
5027
+ if (laneInvariant(cond)) return ['if', ['result', 'v128'], cond, ['then', x], ['else', y]]
5028
+ const cmp = isArr(cond) && cond.length === 3 ? CMP_LANE[cond[0]] : null
5029
+ if (!cmp) return null
5030
+ const ca = lift(cond[1]), cb = lift(cond[2])
5031
+ return (ca && cb) ? ['v128.bitselect', x, y, [cmp, ca, cb]] : null
5032
+ }
5033
+ // Lift an f64 expression to f64x2 (null = not liftable).
5034
+ const lift = (n) => {
5035
+ if (!isArr(n)) return null
5036
+ const op = n[0]
5037
+ if (op === 'f64.const') return ['f64x2.splat', n]
5038
+ if (op === 'local.get') {
5039
+ const v = n[1]
5040
+ if (laneMap.has(v)) return ['local.get', laneMap.get(v)]
5041
+ if (pivType.get(v) === 'f64') return rampOf(v)
5042
+ if (writesName(loopNode, v)) return null
5043
+ return ['f64x2.splat', n]
5044
+ }
5045
+ if (op === 'f64.convert_i32_s' && isArr(n[1]) && n[1][0] === 'local.get' && pivType.get(n[1][1]) === 'i32') return rampOf(n[1][1])
5046
+ if (op === 'global.get') return writesName(loopNode, n[1]) ? null : ['f64x2.splat', n]
5047
+ if (LOAD_OPS[op] === 'f64') {
5048
+ const addr = typeof n[1] === 'string' && n[1].startsWith('offset=') ? n[2] : n[1]
5049
+ if (readsName(addr, pxVar) || [...laneMap.keys()].some(lv => readsName(addr, lv))) return null // per-lane gather: unsupported
5050
+ return ['f64x2.splat', n]
5051
+ }
5052
+ if (op === 'call') {
5053
+ const v2 = PPC_CALL2[n[1]]
5054
+ if (!v2) return null
5055
+ if (n.length === 3) { const a = lift(n[2]); if (!a) return null; sawHeavy = true; return ['call', v2, a] }
5056
+ if (n.length === 4) { const a = lift(n[2]), b = lift(n[3]); if (!a || !b) return null; sawHeavy = true; return ['call', v2, a, b] }
5057
+ return null
5058
+ }
5059
+ if (op === 'if') {
5060
+ if (!isArr(n[1]) || n[1][0] !== 'result' || n[1][1] !== 'f64') return null
5061
+ const thenN = n[3], elseN = n[4]
5062
+ if (!isArr(thenN) || thenN[0] !== 'then' || thenN.length !== 2) return null
5063
+ if (!isArr(elseN) || elseN[0] !== 'else' || elseN.length !== 2) return null
5064
+ let cond = n[2]
5065
+ if (isArr(cond) && cond[0] === 'i32.ne' && isI32Const(cond[2]) && cond[2][1] === 0) cond = cond[1]
5066
+ return liftSelect(lift(thenN[1]), lift(elseN[1]), cond)
5067
+ }
5068
+ // jz lowers `cond ? A : B` to a `select` (A if cond else B). Same two cases as the `if` form:
5069
+ // lane-invariant cond → a scalar v128-typed branch; per-lane f64 compare → bitselect.
5070
+ if (op === 'select' && n.length === 4) return liftSelect(lift(n[1]), lift(n[2]), n[3])
5071
+ if (LANE_PURE.f64.has(op)) {
5072
+ const ks = n.slice(1).map(lift)
5073
+ return ks.some(k => k === null) ? null : [LANE_PURE.f64.get(op).simd, ...ks]
5074
+ }
5075
+ return null
5076
+ }
5077
+
5078
+ // Lift one inner-loop body statement → its lifted form(s), or null to bail.
5079
+ const liftInnerStmt = (s, innerIV) => {
5080
+ if (matchInc1(s) === innerIV || matchIncN(s)?.name === innerIV) return [s] // IV bump: scalar
5081
+ if (isArr(s) && s[0] === 'local.set' && typeof s[1] === 'string' && s.length === 3) {
5082
+ const name = s[1], rhs = s[2]
5083
+ if (fnLocals.get(name) !== 'f64') return laneInvariant(rhs) ? [s] : null // scalar i32 counter
5084
+ const lifted = lift(rhs) // recurrence (rhs reads name) resolves to the shadow — fine
5085
+ return lifted ? [['local.set', shadowOf(name), lifted]] : null
5086
+ }
5087
+ // Lane-invariant scalar `if` (counter wraparound `if(si>=N) si=0`) → keep scalar.
5088
+ if (isArr(s) && s[0] === 'if' && laneInvariant(s[1]) &&
5089
+ s.slice(2).every(arm => isArr(arm) && (arm[0] === 'then' || arm[0] === 'else') &&
5090
+ arm.slice(1).every(st => isArr(st) && st[0] === 'local.set' && fnLocals.get(st[1]) !== 'f64'))) return [s]
5091
+ // Per-lane conditional accumulate `if(cond) acc = acc + E` → acc += bitselect(liftE, 0, mask).
5092
+ if (isArr(s) && s[0] === 'if' && s.length === 3 && isArr(s[2]) && s[2][0] === 'then' && s[2].length === 2) {
5093
+ const st = s[2][1]
5094
+ if (isArr(st) && st[0] === 'local.set' && st.length === 3 && fnLocals.get(st[1]) === 'f64' && laneMap.has(st[1]) &&
5095
+ isArr(st[2]) && st[2][0] === 'f64.add' && isLocalGet(st[2][1], st[1])) {
5096
+ const cond = s[1], cmp = isArr(cond) && cond.length === 3 ? CMP_LANE[cond[0]] : null
5097
+ if (!cmp || laneInvariant(cond)) return null // need a per-lane mask
5098
+ const liftE = lift(st[2][2]), ca = lift(cond[1]), cb = lift(cond[2])
5099
+ if (!liftE || !ca || !cb) return null
5100
+ const sh = laneMap.get(st[1])
5101
+ return [['local.set', sh, ['f64x2.add', ['local.get', sh], ['v128.bitselect', liftE, ['f64x2.splat', ['f64.const', 0]], [cmp, ca, cb]]]]]
5102
+ }
5103
+ }
5104
+ return null
5105
+ }
5106
+
5107
+ const liftInnerLoop = (block) => {
5108
+ const ibl = matchBlockLoop(block, { allowPreamble: true })
5109
+ if (!ibl || ibl.preamble.length) return null
5110
+ const lifted = []
5111
+ for (const s of ibl.body) { const out = liftInnerStmt(s, ibl.incVar); if (!out) return null; lifted.push(...out) }
5112
+ return ['block', ibl.blockLabel, ['loop', ibl.loopLabel, ibl.loopNode[2], ...lifted, ibl.loopNode[ibl.incIdx], ['br', ibl.loopLabel]]]
5113
+ }
5114
+
5115
+ // ---- laneCompute = obody[0..lastInner]: f64 seeds → shadow lift; scalar seeds kept; loops lifted ----
5116
+ const laneCompute = []
5117
+ for (let i = 0; i <= lastInner; i++) {
5118
+ const s = obody[i]
5119
+ if (innerSet.has(i)) { const li = liftInnerLoop(s); if (!li) return null; laneCompute.push(li); continue }
5120
+ if (isArr(s) && s[0] === 'local.set' && typeof s[1] === 'string' && s.length === 3) {
5121
+ const name = s[1], rhs = s[2]
5122
+ if (fnLocals.get(name) === 'f64') {
5123
+ if (readsName(rhs, name)) return null // self-reading seed = carry across the OUTER loop → reject
5124
+ const lifted = lift(rhs); if (!lifted) return null
5125
+ laneCompute.push(['local.set', shadowOf(name), lifted])
5126
+ } else { if (!laneInvariant(rhs)) return null; laneCompute.push(s) } // scalar counter seed
5127
+ continue
5128
+ }
5129
+ return null
5130
+ }
5131
+ if (!sawHeavy || !laneMap.size) return null // no transcendental reduction → leave to the scalar JIT
5132
+
5133
+ // ---- epilogue = obody[lastInner+1..]: colour pack+store, run scalar per lane ----
5134
+ const epilogue = obody.slice(lastInner + 1)
5135
+ let hasStore = false
5136
+ const findStore = (n) => { if (!isArr(n)) return; if (STORE_OPS[n[0]]) hasStore = true; n.forEach(findStore) }
5137
+ epilogue.forEach(findStore)
5138
+ if (!hasStore) return null
5139
+ const epiWritten = new Set()
5140
+ const wr = (n) => { if (!isArr(n)) return; const st = (n[0] === 'local.set' || n[0] === 'local.tee') && typeof n[1] === 'string'; if (st) epiWritten.add(n[1]); for (const c of (st ? n.slice(2) : n.slice(1))) wr(c) }
5141
+ epilogue.forEach(wr)
5142
+ const epiReadSet = new Set(); const rd = (n) => { if (!isArr(n)) return; if (n[0] === 'local.get') epiReadSet.add(n[1]); else for (const c of n) rd(c) }
5143
+ epilogue.forEach(rd)
5144
+ for (const v of epiReadSet) if (writesName(loopNode, v) && !laneMap.has(v) && !epiWritten.has(v) && !pivType.has(v)) return null
5145
+ const epiReads = [...laneMap.keys()].filter(v => epiReadSet.has(v))
5146
+ if (!epiReads.length) return null
5147
+
5148
+ // ============================ emit ============================
5149
+ const newLocalDecls = [...new Set(laneMap.values())].map(n => ['local', n, 'v128'])
5150
+ const epiLane = (k) => [
5151
+ ...epiReads.map(v => ['local.set', v, ['f64x2.extract_lane', k, ['local.get', laneMap.get(v)]]]),
5152
+ ...epilogue.map(s => bump(s, k)),
5153
+ ]
5154
+ const sOut = nm('ob'), sOl = nm('ol')
5155
+ const simdOuter = ['block', sOut, ['loop', sOl,
5156
+ ['br_if', sOut, ['i32.eqz', [oExit.cmpOp, bump(['local.get', pxVar], 1), widthBound]]],
5157
+ ...laneCompute, ...epiLane(0), ...epiLane(1),
5158
+ ...pixelIVs.map(p => ['local.set', p.name, [p.type + '.add', ['local.get', p.name], [p.type + '.const', 2]]]),
5159
+ ['br', sOl]]]
5160
+ const wrapper = ['block', nm('w'), ...preamble, simdOuter, ['block', oLabel, loopNode]]
5161
+ return { wrapper, newLocalDecls }
5162
+ }
5163
+
5164
+ // ---- Integer convolution column-strip-mine (tryConvColumn, experimental) ---------------------
5165
+ //
5166
+ // The int8 quantized convolution / dense-MAC kernel (conv2d): an OUTER output-pixel loop (ox)
5167
+ // whose body — after the inner receptive-field loops fully unroll at speed — is a straight-line
5168
+ // f64 reduction acc = bias + Σ inp[…+ox]·wt[…] over int8 taps, then requantize (acc>>SHIFT),
5169
+ // ReLU-clamp, and store one uint8. jz accumulates in f64, but every product is int8×int8 (≤ 16129)
5170
+ // and the sum fits i32, so the f64 carries an EXACT integer. That lets us strip-mine the column
5171
+ // loop 8-wide as pure integer SIMD: 8 adjacent outputs (ox..ox+7) in lanes. Per tap, the per-pixel
5172
+ // input gather inp[base+ox] is 8 CONTIGUOUS bytes — `v128.load64_zero` + `i16x8.extend_low_i8x16`
5173
+ // — and the (pixel-invariant) weight broadcasts via `i16x8.splat`; `i16x8.mul` forms 8 products
5174
+ // (each fits i16), widened (`i32x4.extend_low/high_i16x8_s`) into two i32x4 accumulators so 36 taps
5175
+ // never overflow. Requant + clamp + store run scalar per lane; the kept scalar loop is the <8 tail.
5176
+ //
5177
+ // BIT-EXACT: integer arithmetic reorders nothing — each lane's i32 sum equals the scalar f64's exact
5178
+ // integer, and ToInt32(acc)>>SHIFT == lane>>SHIFT. Gated behind cfg.experimentalOuterStrip. ~5×
5179
+ // over the scalar reduction (the serial f64 add-chain is latency-bound; 8 columns hide it).
5180
+ function tryConvColumn(blockNode, fnLocals, freshIdRef, enabled) {
5181
+ if (!enabled) return null
5182
+ const outer = matchOuterPixelLoop(blockNode)
5183
+ if (!outer) return null
5184
+ const { oLabel, loopNode, preamble, pixelIVs, pxVar, widthBound, pivType, obody, oExit } = outer
5185
+ if (pivType.get(pxVar) !== 'i32') return null // strip-mine an integer column
5186
+ for (const s of obody) if (isArr(s) && s[0] === 'block' && s.slice(1).some(c => isArr(c) && c[0] === 'loop')) return null // body must be unrolled (no inner loop)
5187
+ const impureCall = (n) => isArr(n) && ((n[0] === 'call' && typeof n[1] === 'string' && !n[1].startsWith('$math.')) || n.some(impureCall))
5188
+ if (obody.some(impureCall)) return null
5189
+ const readsName = (n, name) => isArr(n) && ((n[0] === 'local.get' && n[1] === name) || n.some(c => readsName(c, name)))
5190
+
5191
+ // Locals whose value depends on the column IV (transitively) — these address the per-pixel gather.
5192
+ const oxDep = new Set([pxVar])
5193
+ const allSets = []
5194
+ const collectSets = (n) => { if (!isArr(n)) return; if ((n[0] === 'local.set' || n[0] === 'local.tee') && typeof n[1] === 'string') allSets.push([n[1], n[2]]); for (const c of n.slice(1)) collectSets(c) }
5195
+ obody.forEach(collectSets)
5196
+ for (let changed = true; changed;) { changed = false; for (const [name, rhs] of allSets) if (!oxDep.has(name) && [...oxDep].some(d => readsName(rhs, d))) { oxDep.add(name); changed = true } }
5197
+ const isGatherAddr = (addr) => [...oxDep].some(d => readsName(addr, d))
5198
+
5199
+ // A byte tap operand: convert_i32_{s,u}(i32.load8_{s,u}(addr)). Returns { load, addr, signed }.
5200
+ const matchByteLoad = (n) => {
5201
+ if (!isArr(n) || (n[0] !== 'f64.convert_i32_s' && n[0] !== 'f64.convert_i32_u') || !isArr(n[1])) return null
5202
+ const ld = n[1]
5203
+ if (ld[0] !== 'i32.load8_s' && ld[0] !== 'i32.load8_u') return null
5204
+ const addr = (typeof ld[1] === 'string' && ld[1].startsWith('offset=')) ? ld[2] : ld[1]
5205
+ return { load: ld, addr, signed: ld[0] === 'i32.load8_s' }
5206
+ }
5207
+ const load64 = (ld) => (typeof ld[1] === 'string' && ld[1].startsWith('offset=')) ? ['v128.load64_zero', ld[1], ld[2]] : ['v128.load64_zero', ld[1]]
5208
+ // Lift a single product addend `inp·wt` (exactly one side gathers on ox) to an i16x8 of 8 products.
5209
+ const liftProduct = (prod) => {
5210
+ if (!isArr(prod) || prod[0] !== 'f64.mul') return null
5211
+ const a = matchByteLoad(prod[1]), b = matchByteLoad(prod[2])
5212
+ if (!a || !b) return null
5213
+ const ag = isGatherAddr(a.addr), bg = isGatherAddr(b.addr)
5214
+ const g = ag && !bg ? a : bg && !ag ? b : null // exactly one per-pixel gather
5215
+ if (!g) return null
5216
+ const inv = g === a ? b : a
5217
+ const gI16 = [g.signed ? 'i16x8.extend_low_i8x16_s' : 'i16x8.extend_low_i8x16_u', load64(g.load)]
5218
+ return ['i16x8.mul', gI16, ['i16x8.splat', inv.load]] // splat the invariant weight (fits i16)
5219
+ }
5220
+
5221
+ // THE accumulator: an f64 local written as `acc = acc + product` (either operand order). Its FIRST
5222
+ // write is the init — a plain invariant `acc = bias`, or (bias folded into the first tap by the
5223
+ // reassociator) `acc = bias + product`.
5224
+ const macAddend = (rhs, name) => (isArr(rhs) && rhs[0] === 'f64.add') ? (isLocalGet(rhs[1], name) ? rhs[2] : isLocalGet(rhs[2], name) ? rhs[1] : null) : null
5225
+ let accName = null
5226
+ for (const [name, rhs] of allSets) if (fnLocals.get(name) === 'f64' && macAddend(rhs, name) != null) { if (accName && accName !== name) return null; accName = name }
5227
+ if (!accName) return null
5228
+ const accIdx = []
5229
+ for (let i = 0; i < obody.length; i++) { const s = obody[i]; if (isArr(s) && s[0] === 'local.set' && s[1] === accName && s.length === 3) accIdx.push(i) }
5230
+ if (accIdx.length < 4) return null
5231
+ const initIdx = accIdx[0], initRhs = obody[initIdx][2]
5232
+ if (readsName(initRhs, accName)) return null // first write must not read acc
5233
+
5234
+ const id = freshIdRef.next++
5235
+ const nm = (s) => `$__cv${id}_${s}`
5236
+ const loV = nm('lo'), hiV = nm('hi'), pV = nm('p')
5237
+ const splatI32 = (e) => ['i32x4.splat', (isArr(e) && (e[0] === 'f64.convert_i32_s' || e[0] === 'f64.convert_i32_u')) ? e[1] : ['i32.trunc_sat_f64_s', e]]
5238
+ const accStmts = (prod) => [
5239
+ ['local.set', pV, prod],
5240
+ ['local.set', loV, ['i32x4.add', ['local.get', loV], ['i32x4.extend_low_i16x8_s', ['local.get', pV]]]],
5241
+ ['local.set', hiV, ['i32x4.add', ['local.get', hiV], ['i32x4.extend_high_i16x8_s', ['local.get', pV]]]],
5242
+ ]
5243
+ // Init → lo/hi seeded to the invariant bias, plus the folded first tap (if the bias was fused in).
5244
+ const initStmts = () => {
5245
+ if (isArr(initRhs) && initRhs[0] === 'f64.add') {
5246
+ const pA = liftProduct(initRhs[1]), pB = liftProduct(initRhs[2])
5247
+ const bias = pA && !pB ? initRhs[2] : pB && !pA ? initRhs[1] : null
5248
+ const prod = pA && !pB ? pA : pB && !pA ? pB : null
5249
+ if (!prod || isGatherAddr(bias)) return null
5250
+ return [['local.set', loV, splatI32(bias)], ['local.set', hiV, splatI32(bias)], ...accStmts(prod)]
5251
+ }
5252
+ if (isGatherAddr(initRhs)) return null // plain seed must be loop-invariant
5253
+ return [['local.set', loV, splatI32(initRhs)], ['local.set', hiV, splatI32(initRhs)]]
5254
+ }
5255
+
5256
+ // Build the SIMD body: keep scalar address setup; init→lo/hi seed; each MAC→i16x8 product → lo/hi.
5257
+ const lastMac = accIdx[accIdx.length - 1]
5258
+ const laneCompute = []
5259
+ for (let i = 0; i <= lastMac; i++) {
5260
+ const s = obody[i]
5261
+ if (i === initIdx) { const init = initStmts(); if (!init) return null; laneCompute.push(...init); continue }
5262
+ if (isArr(s) && s[0] === 'local.set' && s[1] === accName) {
5263
+ const addend = macAddend(s[2], accName)
5264
+ if (addend == null) return null // an acc write that isn't acc+product
5265
+ const prod = liftProduct(addend); if (!prod) return null
5266
+ laneCompute.push(...accStmts(prod)); continue
5267
+ }
5268
+ if (readsName(s, accName)) return null // scalar setup must not touch acc
5269
+ laneCompute.push(s)
5270
+ }
5271
+
5272
+ // Epilogue (requant + clamp + store) runs scalar per lane: acc ← the lane's i32 column sum.
5273
+ const epilogue = obody.slice(lastMac + 1)
5274
+ let hasStore = false
5275
+ const findStore = (n) => { if (!isArr(n)) return; if (STORE_OPS[n[0]]) hasStore = true; n.forEach(findStore) }
5276
+ epilogue.forEach(findStore)
5277
+ if (!hasStore) return null
5278
+ const bump = (n, k) => k === 0 ? n
5279
+ : (isArr(n) && n[0] === 'local.get' && pivType.has(n[1])) ? [pivType.get(n[1]) + '.add', n, [pivType.get(n[1]) + '.const', k]]
5280
+ : (isArr(n) ? n.map(c => bump(c, k)) : n)
5281
+ const epiLane = (k) => [
5282
+ ['local.set', accName, ['f64.convert_i32_s', ['i32x4.extract_lane', k & 3, ['local.get', k < 4 ? loV : hiV]]]],
5283
+ ...epilogue.map(s => bump(s, k)),
5284
+ ]
5285
+
5286
+ const newLocalDecls = [['local', loV, 'v128'], ['local', hiV, 'v128'], ['local', pV, 'v128']]
5287
+ const sOut = nm('ob'), sOl = nm('ol')
5288
+ // Guard requires 8 columns available (ox+7 < width); the kept scalar loop finishes the <8 tail.
5289
+ const simdOuter = ['block', sOut, ['loop', sOl,
5290
+ ['br_if', sOut, ['i32.eqz', [oExit.cmpOp, bump(['local.get', pxVar], 7), widthBound]]],
5291
+ ...laneCompute, ...epiLane(0), ...epiLane(1), ...epiLane(2), ...epiLane(3), ...epiLane(4), ...epiLane(5), ...epiLane(6), ...epiLane(7),
5292
+ ...pixelIVs.map(p => ['local.set', p.name, [p.type + '.add', ['local.get', p.name], [p.type + '.const', 8]]]),
5293
+ ['br', sOl]]]
5294
+ const wrapper = ['block', nm('w'), ...preamble, simdOuter, ['block', oLabel, loopNode]]
5295
+ return { wrapper, newLocalDecls }
5296
+ }
5297
+
4221
5298
  // ---- Mixed-lane tone-map (tryToneMap, experimental) ------------------------
4222
5299
  //
4223
5300
  // Vectorizes the log-tonemap TAIL shared by fern / bifurcation / attractors:
@@ -4269,14 +5346,31 @@ function matchInfCanonTone(sel) {
4269
5346
  return tr ? tr.inner : null
4270
5347
  }
4271
5348
 
4272
- // `base + (i<<2)` with `base` a loop-invariant array pointer (local or global). Returns
4273
- // the base node (stride-4 load64_zero/i64.store cover exactly 2 consecutive u32) or null.
4274
- function matchToneAddr(addr, ind) {
5349
+ // Loads tryToneMap accepts as the f64-island input. `i32.load` is the original (stride-4, no
5350
+ // widening). Narrow typed-array reads (Uint8/Int8/Uint16/Int16) are widened to the low two i32x4
5351
+ // lanes before the f64x2.convert — exactly the F/B/T `dist` term over an 8-bit/16-bit buffer.
5352
+ // `shift` = the address stride exponent (0/1/2). `over` = extra ELEMENTS the widening load reads
5353
+ // past its lane pair (u8 reads 4 bytes via load32_zero for 2 lanes) — the SIMD bound is shrunk by
5354
+ // it so the read never runs off the array; the scalar tail finishes the remainder. The widen step
5355
+ // signedness matches the load op, so the i32x4 lanes hold the exact scalar value.
5356
+ const TONE_LOAD = {
5357
+ 'i32.load': { shift: 2, widen: null, over: 0 },
5358
+ 'i32.load8_u': { shift: 0, widen: ['v128.load32_zero', ['i16x8.extend_low_i8x16_u', 'i32x4.extend_low_i16x8_u']], over: 2 },
5359
+ 'i32.load8_s': { shift: 0, widen: ['v128.load32_zero', ['i16x8.extend_low_i8x16_s', 'i32x4.extend_low_i16x8_s']], over: 2 },
5360
+ 'i32.load16_u': { shift: 1, widen: ['v128.load32_zero', ['i32x4.extend_low_i16x8_u']], over: 0 },
5361
+ 'i32.load16_s': { shift: 1, widen: ['v128.load32_zero', ['i32x4.extend_low_i16x8_s']], over: 0 },
5362
+ }
5363
+
5364
+ // `base + (i << shift)` (shift 0 ⇒ bare `base + i`) with `base` a loop-invariant array pointer.
5365
+ // The address shape for a load/store at its element stride: the u32 store uses shift 2 (stride-4 ⇒
5366
+ // load64_zero/i64.store cover exactly 2 consecutive u32); a narrow load uses its own stride (0/1).
5367
+ function matchToneAddrShift(addr, ind, shift) {
4275
5368
  if (!isArr(addr) || addr[0] !== 'i32.add' || addr.length !== 3) return null
4276
5369
  const pair = (baseN, offN) => {
4277
5370
  if (!isArr(baseN) || (baseN[0] !== 'local.get' && baseN[0] !== 'global.get')) return null
4278
5371
  if (baseN[0] === 'local.get' && baseN[1] === ind) return null
4279
- if (isArr(offN) && offN[0] === 'i32.shl' && offN.length === 3 && isLocalGet(offN[1], ind) && constNum(offN[2]) === 2) return baseN
5372
+ if (shift === 0) return isLocalGet(offN, ind) ? baseN : null
5373
+ if (isArr(offN) && offN[0] === 'i32.shl' && offN.length === 3 && isLocalGet(offN[1], ind) && constNum(offN[2]) === shift) return baseN
4280
5374
  return null
4281
5375
  }
4282
5376
  return pair(addr[1], addr[2]) || pair(addr[2], addr[1])
@@ -4321,14 +5415,20 @@ function tryToneMap(bl, fnLocals, freshIdRef, enabled) {
4321
5415
  // island signature (`f64.convert_i32_*`). Any other-width load/store declines. The
4322
5416
  // f64-convert requirement is what distinguishes this from a plain i32 map (tryVectorize,
4323
5417
  // which runs earlier and already owns those).
4324
- let hasConvert = false, storeCount = 0, loadCount = 0, ok = true
5418
+ let hasConvert = false, storeCount = 0, loadCount = 0, overread = 0, ok = true
4325
5419
  const scan = (n) => {
4326
5420
  if (!ok || !isArr(n)) return
4327
5421
  const o = n[0]
4328
5422
  if (o === 'f64.convert_i32_s' || o === 'f64.convert_i32_u') hasConvert = true
4329
- if (o === 'i32.store') { storeCount++; if (!matchToneAddr(n[1], incVar)) ok = false; scan(n[2]); return }
4330
- if (o === 'i32.load') { loadCount++; if (!matchToneAddr(n[1], incVar)) ok = false; return }
4331
- if (LOAD_OPS[o] || STORE_OPS[o]) { ok = false; return } // any wider/narrower memop not this shape
5423
+ if (o === 'i32.store') { storeCount++; if (!matchToneAddrShift(n[1], incVar, 2)) ok = false; scan(n[2]); return }
5424
+ const ld = TONE_LOAD[o]
5425
+ if (ld && n.length === 2) { // i32 (stride-4) or a narrow typed-array read at its own stride
5426
+ loadCount++
5427
+ if (!matchToneAddrShift(n[1], incVar, ld.shift)) { ok = false; return }
5428
+ if (ld.over > overread) overread = ld.over
5429
+ return
5430
+ }
5431
+ if (LOAD_OPS[o] || STORE_OPS[o]) { ok = false; return } // any other-width memop → not this shape
4332
5432
  for (let i = 1; i < n.length; i++) scan(n[i])
4333
5433
  }
4334
5434
  for (const s of body) scan(s)
@@ -4371,10 +5471,16 @@ function tryToneMap(bl, fnLocals, freshIdRef, enabled) {
4371
5471
  }
4372
5472
  }
4373
5473
 
4374
- const ctx = { extraLocals: [], fail: false, failReason: null } // only liftFail (fail/failReason) + freshMask (extraLocals) read it
4375
- const newLanedLocals = new Map() // origName { laneName }
5474
+ const newLanedLocals = new Map() // origName laneName (bare string; see getOrAllocLanedLocal)
5475
+ // SAME field set + ORDER as the ctx in tryVectorize / tryReduceVectorize / tryRampMap. The
5476
+ // self-host kernel infers ONE struct layout per shared callee, and `liftFail` is shared with
5477
+ // liftExprV — so every ctx reaching it MUST have the identical shape, or the inferred layout is
5478
+ // wrong for some and field reads corrupt (this is the exact regression 11657cf fixed; the
5479
+ // tone-map's old 3-field ctx re-broke the ENTIRE self-host vectorizer). tryToneMap itself only
5480
+ // reads fail/failReason/extraLocals, but the unused fields must still be present, in order.
5481
+ const ctx = { laneType: 'f64', incVar, rampVar: null, rampTemp: null, widenLoads: false, localKind, fnLocals: null, newLanedLocals, extraLocals: [], freshIdRef, fail: false, failReason: null }
4376
5482
  const toneSetBefore = new Set() // lane locals already assigned (conditional-merge gate)
4377
- const laned = (name) => { let r = newLanedLocals.get(name); if (!r) { r = { laneName: `${name}__v` }; newLanedLocals.set(name, r) } return r.laneName }
5483
+ const laned = (name) => { let ln = newLanedLocals.get(name); if (!ln) { ln = `${name}__v`; newLanedLocals.set(name, ln) } return ln }
4378
5484
  const freshMask = () => { const mt = `$__mask${freshIdRef.next++}`; ctx.extraLocals.push(['local', mt, 'v128']); return mt }
4379
5485
 
4380
5486
  // The ctx-using lifters are NESTED function declarations that CAPTURE the state above (like
@@ -4412,7 +5518,13 @@ function tryToneMap(bl, fnLocals, freshIdRef, enabled) {
4412
5518
  }
4413
5519
  const tr = matchTruncF64(expr) // f64 → i32 `|0` bridge
4414
5520
  if (tr) { const a = liftV(tr.inner); if (ctx.fail) return null; return [tr.signed ? 'i32x4.trunc_sat_f64x2_s_zero' : 'i32x4.trunc_sat_f64x2_u_zero', a] }
4415
- if (op === 'i32.load' && expr.length === 2) return ['v128.load64_zero', expr[1]] // address kept scalar
5521
+ if (TONE_LOAD[op] && expr.length === 2) { // i32 → load64_zero (low 2 lanes); narrow → widen to i32x4 low lanes
5522
+ const w = TONE_LOAD[op].widen
5523
+ if (!w) return ['v128.load64_zero', expr[1]] // address kept scalar
5524
+ let v = [w[0], expr[1]]
5525
+ for (let i = 0; i < w[1].length; i++) v = [w[1][i], v]
5526
+ return v
5527
+ }
4416
5528
  if (op === 'i32.const') return ['i32x4.splat', expr]
4417
5529
  if (op === 'f64.const') return ['f64x2.splat', expr]
4418
5530
  if (op === 'local.get' && typeof expr[1] === 'string') {
@@ -4421,6 +5533,15 @@ function tryToneMap(bl, fnLocals, freshIdRef, enabled) {
4421
5533
  if (kind === 'invariant') return [fnLocals.get(name) === 'f64' ? 'f64x2.splat' : 'i32x4.splat', expr]
4422
5534
  return liftFail(ctx, `tonemap: ${name} address/induction var used as lane data`)
4423
5535
  }
5536
+ if (op === 'local.tee' && typeof expr[1] === 'string' && expr.length === 3) {
5537
+ // `let v = X` reused in the same statement folds to `(local.tee $v X)`; lift it to set the
5538
+ // lane local AND yield it (bit-exact — the scalar tee sets $v and returns the same value).
5539
+ const name = expr[1]
5540
+ if (localKind.get(name) !== 'lane') return liftFail(ctx, `tonemap: tee of non-lane ${name}`)
5541
+ const v = liftV(expr[2]); if (ctx.fail) return null
5542
+ toneSetBefore.add(name)
5543
+ return ['local.tee', laned(name), v]
5544
+ }
4424
5545
  if (op === 'call' && PPC_CALL2[expr[1]]) { // transcendental → its 2-wide mirror
4425
5546
  const args = []
4426
5547
  for (let i = 2; i < expr.length; i++) { const a = liftV(expr[i]); if (ctx.fail) return null; args.push(a) }
@@ -4557,11 +5678,15 @@ function tryToneMap(bl, fnLocals, freshIdRef, enabled) {
4557
5678
  ...lifted,
4558
5679
  ['local.set', incVar, ['i32.add', ['local.get', incVar], ['i32.const', LANES]]],
4559
5680
  ['br', simdLoop]]]
4560
- const boundSetup = ['local.set', simdBoundName, ['i32.and', boundExpr, ['i32.const', -LANES]]]
5681
+ // A widening narrow load reads more elements than its lane pair (u8: 4 bytes for 2 lanes), so shrink
5682
+ // the SIMD bound by that over-read — the widest load stays in-bounds and the scalar tail finishes the
5683
+ // remainder. (A negative bound from a tiny array just leaves the whole loop scalar — safe.)
5684
+ const boundExprAdj = overread > 0 ? ['i32.sub', boundExpr, ['i32.const', overread]] : boundExpr
5685
+ const boundSetup = ['local.set', simdBoundName, ['i32.and', boundExprAdj, ['i32.const', -LANES]]]
4561
5686
  const wrapper = ['block', ...preamble.map(cloneNode), boundSetup, simdBlock, bl.blockNode]
4562
5687
  const newLocalDecls = [
4563
5688
  ['local', simdBoundName, 'i32'],
4564
- ...[...newLanedLocals.values()].map(({ laneName }) => ['local', laneName, 'v128']),
5689
+ ...[...newLanedLocals.values()].map(laneName => ['local', laneName, 'v128']),
4565
5690
  ...ctx.extraLocals,
4566
5691
  ]
4567
5692
  return { wrapper, newLocalDecls }
@@ -4573,7 +5698,12 @@ function tryToneMap(bl, fnLocals, freshIdRef, enabled) {
4573
5698
  * Walk a function looking for vectorizable (block (loop)) pairs, in-place.
4574
5699
  * Adds new locals to the function header.
4575
5700
  */
4576
- export function vectorizeLaneLocal(fn, multiAcc = false, relaxedFma = false, blurMP = true, whyNot = false, stencil = false, outerStrip = false, pureFuncMap = null, toneMap = false) {
5701
+ // opts gates the recognizer set + lift variants (all default-safe so a bare
5702
+ // `vectorizeLaneLocal(fn)` is the conservative scalar-preserving pass):
5703
+ // multiAcc, relaxedFma, blurMP, whyNot, stencil, outerStrip, pureFuncMap, toneMap.
5704
+ export function vectorizeLaneLocal(fn, opts = {}) {
5705
+ const { multiAcc = false, relaxedFma = false, blurMP = true, whyNot = false,
5706
+ stencil = false, outerStrip = false, pureFuncMap = null, toneMap = false, slp = false } = opts
4577
5707
  if (!isArr(fn) || fn[0] !== 'func') return
4578
5708
  const bodyStart = findBodyStart(fn)
4579
5709
  if (bodyStart < 0) return
@@ -4594,7 +5724,15 @@ export function vectorizeLaneLocal(fn, multiAcc = false, relaxedFma = false, blu
4594
5724
  const freshIdRef = { next: 0 }
4595
5725
  const newLocalDeclsAll = []
4596
5726
 
5727
+ // Hoist loop-invariant partial products out of unrolled dot reductions (rust/LLVM's
5728
+ // mat4 prologue trick). Reassociates the float sum, so tied to the relaxedFma tier;
5729
+ // runs BEFORE the dot-pair vectorizer so a hoisted dot drops below DOT_UNROLL steps
5730
+ // and stays scalar (faster here than the pack/extract SIMD form — see the lab).
5731
+ if (relaxedFma) hoistReductionInvariantsIn(fn, fnLocals, freshIdRef, newLocalDeclsAll)
4597
5732
  vectorizeStraightLineF64DotPairsIn(fn, fnLocals, freshIdRef, newLocalDeclsAll, relaxedFma)
5733
+ // SLP within-iteration store pairs. Sound only with no aliasing typed-array view
5734
+ // in the module (else a shifted view could reorder-hazard the packed read/write).
5735
+ if (slp && !ctx.features.typedView) slpStorePairsIn(fn, fnLocals, freshIdRef, newLocalDeclsAll, slpGetCounts(fn))
4598
5736
 
4599
5737
  // Walk body recursively. Process inner-most matches first (post-order)
4600
5738
  // so we don't try to vectorize an outer loop whose inner is the lane-local one.
@@ -4612,7 +5750,13 @@ export function vectorizeLaneLocal(fn, multiAcc = false, relaxedFma = false, blu
4612
5750
  // recognizers (divergent-escape, ramp-map, blur, channel-reduce, per-pixel)
4613
5751
  // do their own matching on the raw node. Order is preserved exactly — it is
4614
5752
  // load-bearing (first match wins).
4615
- const bl = matchBlockLoop(node, { allowPreamble: true })
5753
+ // allowInlinedLi: accept an inlined LICM preamble (`$__inl*___li*`) too — jz's
5754
+ // LICM hoists ToInt32/casts of loop-invariant params just before the loop (e.g.
5755
+ // `a[i] & m` with a runtime `m`), and after inlining the snap is renamed off the
5756
+ // bare `$__li*` form. The preamble is pure & loop-invariant by construction
5757
+ // (hasSideEffect-guarded) and cloned ahead of the SIMD block, so this only widens
5758
+ // which loops the recognizers see, never changes a lifted result.
5759
+ const bl = matchBlockLoop(node, { allowPreamble: true, allowInlinedLi: true })
4616
5760
  let r = tryDivergentEscapeVectorize(node, fnLocals, freshIdRef)
4617
5761
  ?? tryMemCopyFill(bl, fnLocals, freshIdRef)
4618
5762
  ?? tryVectorize(bl, fnLocals, freshIdRef)
@@ -4625,6 +5769,8 @@ export function vectorizeLaneLocal(fn, multiAcc = false, relaxedFma = false, blu
4625
5769
  ?? tryByteScan(bl, fnLocals, freshIdRef)
4626
5770
  ?? tryPerPixelColor(node, fnLocals, freshIdRef, pureFuncMap)
4627
5771
  ?? tryOuterStrip(node, fnLocals, freshIdRef, outerStrip)
5772
+ ?? tryIteratedReduce(node, fnLocals, freshIdRef, outerStrip)
5773
+ ?? tryConvColumn(node, fnLocals, freshIdRef, outerStrip)
4628
5774
  ?? tryToneMap(bl, fnLocals, freshIdRef, toneMap)
4629
5775
  // --why-not-simd: a canonical loop-shaped candidate that no SIMD pass took.
4630
5776
  // Reported BEFORE the scalar strength-reduce fallback (which fires on most
@@ -4644,8 +5790,10 @@ export function vectorizeLaneLocal(fn, multiAcc = false, relaxedFma = false, blu
4644
5790
  }
4645
5791
  }
4646
5792
  _whyNotActive = whyNot
5793
+ _relaxF32 = relaxedFma
4647
5794
  for (let i = bodyStart; i < fn.length; i++) walk(fn, i)
4648
5795
  _whyNotActive = false
5796
+ _relaxF32 = false
4649
5797
 
4650
5798
  if (newLocalDeclsAll.length) {
4651
5799
  // Sibling loops (and the straight-line dot pass) can each lift the SAME source