jz 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -43
- package/bench/README.md +128 -78
- package/bench/bench.svg +32 -42
- package/cli.js +73 -7
- package/dist/interop.js +1 -0
- package/dist/jz.js +6024 -0
- package/index.d.ts +126 -0
- package/index.js +190 -34
- package/interop.js +71 -27
- package/layout.js +5 -0
- package/module/array.js +71 -39
- package/module/collection.js +41 -5
- package/module/core.js +2 -4
- package/module/math.js +138 -2
- package/module/number.js +21 -0
- package/module/object.js +26 -0
- package/module/simd.js +37 -5
- package/module/string.js +41 -12
- package/module/typedarray.js +415 -26
- package/package.json +21 -6
- package/src/autoload.js +3 -0
- package/src/compile/analyze-scans.js +174 -11
- package/src/compile/analyze.js +38 -3
- package/src/compile/emit-assign.js +9 -6
- package/src/compile/emit.js +347 -36
- package/src/compile/index.js +307 -29
- package/src/compile/infer.js +36 -1
- package/src/compile/loop-divmod.js +155 -0
- package/src/compile/narrow.js +108 -11
- package/src/compile/peel-stencil.js +261 -0
- package/src/compile/plan/advise.js +55 -1
- package/src/compile/plan/index.js +4 -0
- package/src/compile/plan/inline.js +5 -2
- package/src/compile/plan/literals.js +220 -5
- package/src/compile/plan/scope.js +115 -39
- package/src/compile/program-facts.js +21 -2
- package/src/ctx.js +45 -7
- package/src/ir.js +55 -7
- package/src/kind-traits.js +27 -0
- package/src/kind.js +65 -3
- package/src/optimize/index.js +344 -45
- package/src/optimize/vectorize.js +2968 -182
- package/src/prepare/index.js +64 -7
- package/src/prepare/lift-iife.js +149 -0
- package/src/reps.js +3 -2
- package/src/static.js +9 -0
- package/src/type.js +10 -6
- package/src/wat/assemble.js +195 -13
- package/src/wat/optimize.js +363 -185
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { findBodyStart } from '../ir.js'
|
|
2
|
+
import { warn } from '../ctx.js'
|
|
2
3
|
|
|
3
4
|
/**
|
|
4
5
|
* Lane-local SIMD-128 vectorizer.
|
|
@@ -191,19 +192,27 @@ const matchF64DotSeq = (stmts, i) => {
|
|
|
191
192
|
|
|
192
193
|
const f64x2Pair = (lo, hi) => ['f64x2.replace_lane', 1, ['f64x2.splat', ['local.get', lo]], ['local.get', hi]]
|
|
193
194
|
|
|
194
|
-
|
|
195
|
+
// Build the 2-lane dot expression `a0*p0 + a1*p1 + a2*p2 + a3*p3`.
|
|
196
|
+
// Default: explicit mul/add pairs (one rounding per op) — bit-identical to the
|
|
197
|
+
// scalar `a*b+c` a JS engine emits. With `useRelaxedFma`, each accumulate folds
|
|
198
|
+
// to `f64x2.relaxed_madd(splat(a[i]), p[i], acc)` — one VFMADD instruction with
|
|
199
|
+
// a single rounding. Faster and more accurate, but the fused rounding diverges
|
|
200
|
+
// from the non-fused reference (the bench `fma` parity class). Opt-in only.
|
|
201
|
+
const dotPairExpr = (a, pairs, useRelaxedFma = false) => {
|
|
195
202
|
let expr = ['f64x2.mul', ['f64x2.splat', ['local.get', a[0]]], pairs[0]]
|
|
196
203
|
for (let i = 1; i < 4; i++) {
|
|
197
|
-
expr =
|
|
204
|
+
expr = useRelaxedFma
|
|
205
|
+
? ['f64x2.relaxed_madd', ['f64x2.splat', ['local.get', a[i]]], pairs[i], expr]
|
|
206
|
+
: ['f64x2.add', expr, ['f64x2.mul', ['f64x2.splat', ['local.get', a[i]]], pairs[i]]]
|
|
198
207
|
}
|
|
199
208
|
return expr
|
|
200
209
|
}
|
|
201
210
|
|
|
202
|
-
const vectorizeStraightLineF64DotPairsIn = (node, fnLocals, freshIdRef, newLocalDecls) => {
|
|
211
|
+
const vectorizeStraightLineF64DotPairsIn = (node, fnLocals, freshIdRef, newLocalDecls, useRelaxedFma = false) => {
|
|
203
212
|
if (!isArr(node)) return
|
|
204
213
|
for (let i = 0; i < node.length; i++) {
|
|
205
214
|
const child = node[i]
|
|
206
|
-
if (isArr(child)) vectorizeStraightLineF64DotPairsIn(child, fnLocals, freshIdRef, newLocalDecls)
|
|
215
|
+
if (isArr(child)) vectorizeStraightLineF64DotPairsIn(child, fnLocals, freshIdRef, newLocalDecls, useRelaxedFma)
|
|
207
216
|
}
|
|
208
217
|
const addendTemps = new Map()
|
|
209
218
|
const pairTemps = new Map()
|
|
@@ -246,7 +255,7 @@ const vectorizeStraightLineF64DotPairsIn = (node, fnLocals, freshIdRef, newLocal
|
|
|
246
255
|
}
|
|
247
256
|
pairs.push(['local.get', tmp])
|
|
248
257
|
}
|
|
249
|
-
const dot = dotPairExpr(a.left, pairs)
|
|
258
|
+
const dot = dotPairExpr(a.left, pairs, useRelaxedFma)
|
|
250
259
|
const expr = addend ? ['f64x2.add', dot, ['f64x2.splat', addend]] : dot
|
|
251
260
|
node.splice(i, b.end - i,
|
|
252
261
|
...prefix,
|
|
@@ -505,6 +514,20 @@ function matchInc1(stmt) {
|
|
|
505
514
|
return x
|
|
506
515
|
}
|
|
507
516
|
|
|
517
|
+
/**
|
|
518
|
+
* Match increment shape `(local.set $X (i32.add (local.get $X) (i32.const C)))`
|
|
519
|
+
* for any constant C. Returns { name, c } or null. Generalizes matchInc1 to the
|
|
520
|
+
* strided-pointer bumps (`p += stride`) the ramp-map recognizer must scale.
|
|
521
|
+
*/
|
|
522
|
+
function matchIncN(stmt) {
|
|
523
|
+
if (!isArr(stmt) || stmt[0] !== 'local.set' || stmt.length !== 3) return null
|
|
524
|
+
const x = stmt[1], v = stmt[2]
|
|
525
|
+
if (!isArr(v) || v[0] !== 'i32.add' || v.length !== 3) return null
|
|
526
|
+
if (!isLocalGet(v[1], x)) return null
|
|
527
|
+
const c = constNum(v[2])
|
|
528
|
+
return c == null ? null : { name: x, c }
|
|
529
|
+
}
|
|
530
|
+
|
|
508
531
|
/**
|
|
509
532
|
* Match `(br_if $LABEL (i32.eqz (i32.lt_{s,u} (local.get $I) BOUND)))`.
|
|
510
533
|
* Returns { ind, bound } or null.
|
|
@@ -661,62 +684,116 @@ const hasGlobalSet = (node) => {
|
|
|
661
684
|
return false
|
|
662
685
|
}
|
|
663
686
|
|
|
687
|
+
// True if EXPR carries any side effect — a call, a memory store/op, or a global
|
|
688
|
+
// write. Used to reject an impure preamble we would otherwise clone (run twice).
|
|
689
|
+
const hasSideEffect = (node) => {
|
|
690
|
+
if (!isArr(node)) return false
|
|
691
|
+
const op = node[0]
|
|
692
|
+
if (op === 'call' || op === 'call_indirect' || op === 'global.set'
|
|
693
|
+
|| (typeof op === 'string' && (op.includes('.store') || op.startsWith('memory.')))) return true
|
|
694
|
+
for (let i = 1; i < node.length; i++) if (hasSideEffect(node[i])) return true
|
|
695
|
+
return false
|
|
696
|
+
}
|
|
697
|
+
|
|
664
698
|
// ---- Recognize a (block (loop)) pair --------------------------------------
|
|
665
699
|
|
|
666
700
|
/**
|
|
667
|
-
*
|
|
668
|
-
*
|
|
701
|
+
* Match the canonical vectorizable loop SCAFFOLD shared by every inner-loop
|
|
702
|
+
* recognizer:
|
|
703
|
+
* (block $blk [preamble…]
|
|
704
|
+
* (loop $loop
|
|
705
|
+
* (br_if $blk (i32.eqz (i32.lt_{s,u} $i BOUND))) ; exit guard
|
|
706
|
+
* BODY…
|
|
707
|
+
* (local.set $i (i32.add $i 1)) ; bottom increment
|
|
708
|
+
* (br $loop)))
|
|
709
|
+
*
|
|
710
|
+
* Returns the structural FACTS only — no policy — or null when the shape
|
|
711
|
+
* doesn't match:
|
|
712
|
+
* { blockNode, blockLabel, loopNode, loopLabel, endIdx, incIdx, incVar,
|
|
713
|
+
* exitInfo, bound, boundLocal, body, preamble }
|
|
714
|
+
* - `blockNode` is the original block, embedded verbatim as the scalar tail by
|
|
715
|
+
* each lifter's wrapper (the never-miscompile remainder loop).
|
|
716
|
+
* - `body` = loopNode.slice(3, incIdx) (between exit guard and increment)
|
|
717
|
+
* - `bound` = the raw BOUND expr; `boundLocal` = its local name when it is
|
|
718
|
+
* `(local.get $L)`, else null. Bound shape is NOT rejected here — callers that
|
|
719
|
+
* require a local-or-const bound check it themselves (tryStrengthReduceIV is
|
|
720
|
+
* bound-shape-agnostic, so baking a rejection in would change its behavior).
|
|
721
|
+
*
|
|
722
|
+
* `opts.allowPreamble` (default false): when true, LICM-hoisted invariant
|
|
723
|
+
* `(local.set $__li* EXPR)` statements BEFORE the loop are collected into
|
|
724
|
+
* `preamble` (pure & loop-invariant by construction — safe to clone/re-run);
|
|
725
|
+
* a non-`$__li` preamble, an impure value, or any array content AFTER the loop
|
|
726
|
+
* bails. When false, ANY non-loop array content in the block bails.
|
|
669
727
|
*/
|
|
670
|
-
function
|
|
728
|
+
function matchBlockLoop(blockNode, opts = {}) {
|
|
671
729
|
if (!isArr(blockNode) || blockNode[0] !== 'block') return null
|
|
672
|
-
|
|
673
|
-
let blockLabel = null
|
|
674
|
-
|
|
730
|
+
const allowPreamble = !!opts.allowPreamble
|
|
731
|
+
let blockLabel = null, loopNode = null
|
|
732
|
+
const preamble = []
|
|
675
733
|
for (let i = 1; i < blockNode.length; i++) {
|
|
676
734
|
const c = blockNode[i]
|
|
677
|
-
if (typeof c === 'string' && c.startsWith('$') && blockLabel == null && i === 1) {
|
|
678
|
-
blockLabel = c; continue
|
|
679
|
-
}
|
|
735
|
+
if (typeof c === 'string' && c.startsWith('$') && blockLabel == null && i === 1) { blockLabel = c; continue }
|
|
680
736
|
if (isArr(c) && c[0] === 'loop') {
|
|
681
737
|
if (loopNode) return null // multiple loops
|
|
682
|
-
|
|
738
|
+
loopNode = c
|
|
683
739
|
} else if (isArr(c)) {
|
|
684
|
-
|
|
740
|
+
// `loopNode` truthy ⇒ this content is AFTER the loop ⇒ bail (even for a $__li set).
|
|
741
|
+
// A LICM-hoisted invariant is `$__liN`; INLINING renames it (e.g. `$__inl7___li0`). Default
|
|
742
|
+
// accepts only un-inlined `$__li*` (keeps the existing recognizers byte-identical);
|
|
743
|
+
// `allowInlinedLi` (gated callers only) also accepts the `__liN` marker anywhere — both are
|
|
744
|
+
// pure & loop-invariant by construction (belt-and-suspenders: hasSideEffect guard).
|
|
745
|
+
// Under allowInlinedLi a block preamble is loop-invariant by construction (jz hoists only
|
|
746
|
+
// invariants before the loop; IV-dependent work lives in the body), so any PURE local.set is
|
|
747
|
+
// safe to clone ahead of the SIMD — covers $__inl*__li* (schrodinger) AND $_pg0-style
|
|
748
|
+
// peephole-hoisted bounds (slime). The hasSideEffect guard rejects impure setups.
|
|
749
|
+
const liOk = typeof c[1] === 'string' && (opts.allowInlinedLi ? true : c[1].startsWith('$__li'))
|
|
750
|
+
if (!allowPreamble || loopNode || c[0] !== 'local.set' || !liOk || hasSideEffect(c[2])) return null
|
|
751
|
+
preamble.push(c)
|
|
685
752
|
}
|
|
686
753
|
}
|
|
687
754
|
if (!loopNode || !blockLabel) return null
|
|
688
755
|
|
|
689
|
-
// Loop layout: ['loop', '$label', ...stmts]
|
|
690
756
|
const loopLabel = typeof loopNode[1] === 'string' && loopNode[1].startsWith('$') ? loopNode[1] : null
|
|
691
757
|
if (!loopLabel) return null
|
|
692
758
|
|
|
693
|
-
|
|
694
|
-
let endIdx = loopNode.length - 1
|
|
759
|
+
const endIdx = loopNode.length - 1
|
|
695
760
|
if (!(isArr(loopNode[endIdx]) && loopNode[endIdx][0] === 'br' && loopNode[endIdx][1] === loopLabel)) return null
|
|
696
761
|
const incIdx = endIdx - 1
|
|
697
|
-
|
|
762
|
+
let incVar = matchInc1(loopNode[incIdx])
|
|
763
|
+
// CSE'd increment (gated): O3 may fold `x+1` into a body tee (the `xe = x+1` wrap) and write the
|
|
764
|
+
// increment as `x = $t` reusing it. Recover the IV when `$t` is `(tee/set $t (i32.add x 1))` in body.
|
|
765
|
+
if (!incVar && opts.allowInlinedLi) {
|
|
766
|
+
const inc = loopNode[incIdx]
|
|
767
|
+
if (isArr(inc) && inc[0] === 'local.set' && inc.length === 3 && isLocalGet(inc[2])) {
|
|
768
|
+
const copyOf = inc[2][1], iv = inc[1]
|
|
769
|
+
const findInc1 = (m) => isArr(m) && (((m[0] === 'local.set' || m[0] === 'local.tee') && m[1] === copyOf && isArr(m[2]) && m[2][0] === 'i32.add' && isLocalGet(m[2][1], iv) && constNum(m[2][2]) === 1) || m.some(findInc1))
|
|
770
|
+
if (loopNode.slice(3, incIdx).some(findInc1)) incVar = iv
|
|
771
|
+
}
|
|
772
|
+
}
|
|
698
773
|
if (!incVar) return null
|
|
699
774
|
|
|
700
|
-
// First stmt must be the exit br_if.
|
|
701
775
|
const exitInfo = matchExitBrIf(loopNode[2], blockLabel)
|
|
702
|
-
if (!exitInfo) return null
|
|
703
|
-
if (exitInfo.ind !== incVar) return null
|
|
776
|
+
if (!exitInfo || exitInfo.ind !== incVar) return null
|
|
704
777
|
|
|
705
|
-
|
|
706
|
-
const
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
778
|
+
const bound = exitInfo.bound
|
|
779
|
+
const boundLocal = isArr(bound) && bound[0] === 'local.get' && typeof bound[1] === 'string' ? bound[1] : null
|
|
780
|
+
const body = loopNode.slice(3, incIdx)
|
|
781
|
+
return { blockNode, blockLabel, loopNode, loopLabel, endIdx, incIdx, incVar, exitInfo, bound, boundLocal, body, preamble }
|
|
782
|
+
}
|
|
783
|
+
|
|
784
|
+
/**
|
|
785
|
+
* Try to vectorize the inner loop. Returns the replacement node array
|
|
786
|
+
* (synthetic outer block) or null on no match.
|
|
787
|
+
*/
|
|
788
|
+
function tryVectorize(bl, fnLocals, freshIdRef) {
|
|
789
|
+
// Consumes the shared scaffold descriptor (matchBlockLoop, computed once by the
|
|
790
|
+
// dispatch). The LICM `$__li` preamble is cloned ahead of the SIMD block; each
|
|
791
|
+
// set is pure & loop-invariant, so the kept scalar tail harmlessly re-runs it.
|
|
792
|
+
if (!bl) return null
|
|
793
|
+
const { incVar, bound, boundLocal, body, preamble } = bl
|
|
794
|
+
|
|
795
|
+
// Bound must be loop-invariant: (local.get $L) or (i32.const N).
|
|
796
|
+
if (!boundLocal && !isI32Const(bound)) return null
|
|
720
797
|
|
|
721
798
|
// Detect lane type from the FIRST load in body.
|
|
722
799
|
let laneType = null
|
|
@@ -839,7 +916,7 @@ function tryVectorize(blockNode, fnLocals, freshIdRef) {
|
|
|
839
916
|
// Build lifted body. If anything fails to lift, bail.
|
|
840
917
|
const newLanedLocals = new Map() // origName → { laneName, simdType }
|
|
841
918
|
const extraLocals = [] // canon temps allocated during lift
|
|
842
|
-
const ctx = { laneType, incVar, localKind, newLanedLocals, extraLocals, freshIdRef, fail: false, failReason: null }
|
|
919
|
+
const ctx = { laneType, incVar, rampVar: null, rampTemp: null, widenLoads: false, localKind, newLanedLocals, extraLocals, freshIdRef, fail: false, failReason: null }
|
|
843
920
|
const lifted = []
|
|
844
921
|
for (const s of body) {
|
|
845
922
|
const r = liftStmt(s, ctx)
|
|
@@ -881,8 +958,10 @@ function tryVectorize(blockNode, fnLocals, freshIdRef) {
|
|
|
881
958
|
['i32.and', boundExpr, ['i32.const', mask]]]
|
|
882
959
|
|
|
883
960
|
// Synthetic outer wrapper — has no result, no label, just sequences.
|
|
884
|
-
//
|
|
885
|
-
|
|
961
|
+
// A clone of any LICM-hoisted preamble runs first (so the SIMD block sees the
|
|
962
|
+
// invariant); the original block is preserved unchanged as the scalar tail (which
|
|
963
|
+
// re-runs the preamble harmlessly — it is loop-invariant).
|
|
964
|
+
const wrapper = ['block', ...preamble.map(cloneNode), boundSetup, simdBlock, bl.blockNode]
|
|
886
965
|
|
|
887
966
|
// Locals to add to function header.
|
|
888
967
|
const newLocalDecls = [
|
|
@@ -894,6 +973,292 @@ function tryVectorize(blockNode, fnLocals, freshIdRef) {
|
|
|
894
973
|
return { wrapper, newLocalDecls }
|
|
895
974
|
}
|
|
896
975
|
|
|
976
|
+
// ---- Stencil recognizer (neighbour loads: a[i±δ], a[c±δ], a[rn+x]) --------
|
|
977
|
+
//
|
|
978
|
+
// Vectorizes a map whose loads read NEIGHBOURING elements — `b[i] = f(a[i-1],
|
|
979
|
+
// a[i], a[i+1])` and the 2-D form `b[c] = f(a[c-1], a[c+1], a[rn+x], …)` where
|
|
980
|
+
// `c = rc + x` is a derived induction var (rc loop-invariant, x the IV).
|
|
981
|
+
//
|
|
982
|
+
// The lift is bit-exact BY CONSTRUCTION: a scalar `f64.load` at `base+(idx<<K)`
|
|
983
|
+
// becomes `v128.load` at the SAME address; for f64x2 lanes (x, x+1) that covers
|
|
984
|
+
// `(elem[idx], elem[idx+1])` — exactly the bytes the two scalar iterations read.
|
|
985
|
+
// No new memory is touched (scalar tail handles the remainder) ⇒ no boundary
|
|
986
|
+
// special-casing, no new OOB. The neighbour `a[i+1]` arrives as
|
|
987
|
+
// `(f64.load offset=8 …)` → `v128.load offset=8` (the +1-shifted pair). Stride-1
|
|
988
|
+
// in the IV is required (consecutive lanes ⇒ consecutive elements): every index
|
|
989
|
+
// must be affine in the IV with coefficient exactly 1 (`ivCoeff`).
|
|
990
|
+
//
|
|
991
|
+
// Correctness gates:
|
|
992
|
+
// • f64/f32 lanes only — float data locals vs i32 index/address locals are
|
|
993
|
+
// type-distinct, so localKind is by type. An i32-used-as-data case bails in
|
|
994
|
+
// the lifter (never miscompiles). Integer-lane stencils (types collide) decline.
|
|
995
|
+
// • In-place bail: if the WRITTEN base is also accessed at a DIFFERENT element,
|
|
996
|
+
// SIMD reads the old value where scalar reads the just-written one (loop-
|
|
997
|
+
// carried) ⇒ null. Offset-0 read of the written array is safe.
|
|
998
|
+
// • Distinct base subtrees ⇒ assumed non-aliasing — the SAME assumption the
|
|
999
|
+
// plain map path already relies on. A ping-pong buffer swap (waves) is OUTSIDE
|
|
1000
|
+
// the loop, so in-loop bases stay distinct globals — safe without a runtime guard.
|
|
1001
|
+
// • Reassociation: summing neighbours reorders f64 adds across lanes (ulp, like
|
|
1002
|
+
// float reductions) — gated behind cfg.experimentalStencil until proven.
|
|
1003
|
+
function tryStencil(node, fnLocals, freshIdRef, enabled) {
|
|
1004
|
+
if (!enabled) return null
|
|
1005
|
+
// Gated, so match here with allowInlinedLi (accepts inlined LICM preambles `$__inl7___li*` —
|
|
1006
|
+
// grid loops like schrodinger's stepR carry the row-base `y*w` as such a hoisted invariant).
|
|
1007
|
+
const bl = matchBlockLoop(node, { allowPreamble: true, allowInlinedLi: true })
|
|
1008
|
+
if (!bl) return null
|
|
1009
|
+
const { incVar, bound, body, preamble } = bl // preamble: LICM-hoisted $__li invariants
|
|
1010
|
+
if (body.some(hasGlobalSet)) return null
|
|
1011
|
+
|
|
1012
|
+
// Leaf-stencil guard: a stencil body is pure array arithmetic. A NESTED LOOP (the outer loop of a
|
|
1013
|
+
// 2-D sweep, whose body contains the inner loop) or a non-$math call must NOT be lifted as a
|
|
1014
|
+
// stencil — its "neighbour reads" would be the nested loop's loads, misaligned. waves/schrodinger/
|
|
1015
|
+
// metaballs bodies are pure arithmetic (math calls allowed), so they pass.
|
|
1016
|
+
const hasNestedLoopOrCall = (n) => isArr(n) && (n[0] === 'loop'
|
|
1017
|
+
|| (n[0] === 'call' && (typeof n[1] !== 'string' || !n[1].startsWith('$math.'))) || n[0] === 'call_indirect'
|
|
1018
|
+
|| n.some(hasNestedLoopOrCall))
|
|
1019
|
+
if (body.some(hasNestedLoopOrCall)) return null
|
|
1020
|
+
|
|
1021
|
+
const writes = new Set()
|
|
1022
|
+
for (const s of body) collectWrites(s, writes)
|
|
1023
|
+
|
|
1024
|
+
// Bound is re-evaluated for the SIMD guard, so it must be a PURE loop-invariant
|
|
1025
|
+
// i32 expression (const / unwritten local / global / +,-,* thereof). Unlike the
|
|
1026
|
+
// plain-map path (bare local-or-const only), stencils commonly bound by `w-1`.
|
|
1027
|
+
const boundPureInv = (n) =>
|
|
1028
|
+
isI32Const(n) ? true
|
|
1029
|
+
: isLocalGet(n) ? !writes.has(n[1])
|
|
1030
|
+
: (isArr(n) && n[0] === 'global.get') ? true
|
|
1031
|
+
: (isArr(n) && (n[0] === 'i32.add' || n[0] === 'i32.sub' || n[0] === 'i32.mul') && n.length === 3)
|
|
1032
|
+
? boundPureInv(n[1]) && boundPureInv(n[2])
|
|
1033
|
+
: false
|
|
1034
|
+
if (!boundPureInv(bound)) return null
|
|
1035
|
+
|
|
1036
|
+
// Element-index coefficient in the IV: 0 (loop-invariant), 1 (stride-1 affine —
|
|
1037
|
+
// IV, a derived IV, or either ± invariant), or null (anything else).
|
|
1038
|
+
const derived = new Set()
|
|
1039
|
+
let needsPeel = false
|
|
1040
|
+
const rightBs = []
|
|
1041
|
+
const unTee = (b) => (isArr(b) && b[0] === 'local.tee' && b.length === 3) ? b[2] : b // CSE folds x±1 into a tee
|
|
1042
|
+
const isStep = (b, op) => { b = unTee(b); return isArr(b) && b[0] === op && b.length === 3 && isLocalGet(b[1], incVar) && isI32Const(b[2]) && constNum(b[2]) === 1 }
|
|
1043
|
+
const isZeroGuard = (g) => isArr(g) && ((g[0] === 'i32.eqz' && isLocalGet(g[1], incVar)) || (g[0] === 'i32.eq' && isLocalGet(g[1], incVar) && isI32Const(g[2]) && constNum(g[2]) === 0))
|
|
1044
|
+
// Toroidal wrap-select: `xw = x>0?x-1:w-1` / `xe = x<w-1?x+1:0`. Fires its wrap value only at a
|
|
1045
|
+
// boundary column the peel covers — LEFT (interior x-1) at x=0, RIGHT (interior x+1) at x=B.
|
|
1046
|
+
// Returns null | {dir:'L'} | {dir:'R',B}. Sound for ANY B: simdBound caps at min(bound,…B)-(lanes-1)
|
|
1047
|
+
// so no chunk reaches x=B (no need to prove B==bound-1, which may be hoisted out of reach).
|
|
1048
|
+
const isWrapSelect = (e) => {
|
|
1049
|
+
if (!isArr(e) || e[0] !== 'select' || e.length !== 4) return null
|
|
1050
|
+
const g = e[3]
|
|
1051
|
+
if (isStep(e[1], 'i32.sub') && ivCoeff(e[2]) === 0 && isArr(g) && g[0] === 'i32.gt_s' && isLocalGet(g[1], incVar) && isI32Const(g[2]) && constNum(g[2]) === 0) return { dir: 'L' }
|
|
1052
|
+
if (isStep(e[2], 'i32.sub') && ivCoeff(e[1]) === 0 && isZeroGuard(g)) return { dir: 'L' }
|
|
1053
|
+
if (isStep(e[1], 'i32.add') && ivCoeff(e[2]) === 0 && isArr(g) && g[0] === 'i32.lt_s' && isLocalGet(g[1], incVar)) return { dir: 'R', B: g[2] }
|
|
1054
|
+
if (isStep(e[2], 'i32.add') && ivCoeff(e[1]) === 0 && isArr(g) && g[0] === 'i32.eq' && isLocalGet(g[1], incVar)) return { dir: 'R', B: g[2] }
|
|
1055
|
+
return null
|
|
1056
|
+
}
|
|
1057
|
+
const ivCoeff = (n) => {
|
|
1058
|
+
if (isLocalGet(n)) {
|
|
1059
|
+
const nm = n[1]
|
|
1060
|
+
if (nm === incVar || derived.has(nm)) return 1
|
|
1061
|
+
return writes.has(nm) ? null : 0 // unwritten ⇒ loop-invariant
|
|
1062
|
+
}
|
|
1063
|
+
if (isI32Const(n)) return 0
|
|
1064
|
+
if (isArr(n) && n[0] === 'global.get') return 0
|
|
1065
|
+
if (isArr(n) && (n[0] === 'i32.add' || n[0] === 'i32.sub') && n.length === 3) {
|
|
1066
|
+
const a = ivCoeff(n[1]), b = ivCoeff(n[2])
|
|
1067
|
+
if (a == null || b == null) return null
|
|
1068
|
+
const c = n[0] === 'i32.add' ? a + b : a - b
|
|
1069
|
+
return c === 0 || c === 1 ? c : null
|
|
1070
|
+
}
|
|
1071
|
+
// `y*w` (inline row base, e.g. idx = y*w + x): invariant×invariant ⇒ coeff 0.
|
|
1072
|
+
// Any IV-dependent factor would be non-unit-stride (stride-w) ⇒ reject.
|
|
1073
|
+
if (isArr(n) && (n[0] === 'i32.mul' || n[0] === 'f64.mul') && n.length === 3)
|
|
1074
|
+
return ivCoeff(n[1]) === 0 && ivCoeff(n[2]) === 0 ? 0 : null
|
|
1075
|
+
// Float-derived index (grid loops compute the row base `y*w` in f64): the index arrives as
|
|
1076
|
+
// `idx = select(wrap(trunc_sat(INV + convert(x))), 0, ≠Inf)`. For an integer counter x,
|
|
1077
|
+
// trunc(C + x) = trunc(C) + x ⇒ stride-1 (the i32 lane offset is added before the trunc); the
|
|
1078
|
+
// Infinity-canon select takes the trunc branch for finite coords (grid indices are finite).
|
|
1079
|
+
// f64.add/sub mirror i32.add/sub; convert/wrap/trunc_sat/tee are coeff-transparent.
|
|
1080
|
+
if (isArr(n) && (n[0] === 'f64.add' || n[0] === 'f64.sub') && n.length === 3) {
|
|
1081
|
+
const a = ivCoeff(n[1]), b = ivCoeff(n[2])
|
|
1082
|
+
if (a == null || b == null) return null
|
|
1083
|
+
const c = n[0] === 'f64.add' ? a + b : a - b
|
|
1084
|
+
return c === 0 || c === 1 ? c : null
|
|
1085
|
+
}
|
|
1086
|
+
if (isArr(n) && (n[0] === 'f64.convert_i32_s' || n[0] === 'i32.wrap_i64' || n[0] === 'i64.trunc_sat_f64_s') && n.length === 2)
|
|
1087
|
+
return ivCoeff(n[1])
|
|
1088
|
+
if (isArr(n) && n[0] === 'local.tee' && n.length === 3) return ivCoeff(n[2])
|
|
1089
|
+
if (isArr(n) && n[0] === 'select') {
|
|
1090
|
+
// Toroidal wrap-select (inline in an address or named): stride-1 interior; flag the peel.
|
|
1091
|
+
const w = isWrapSelect(n)
|
|
1092
|
+
if (w) { needsPeel = true; if (w.dir === 'R' && !rightBs.some(b => exprEq(b, w.B))) rightBs.push(w.B); return 1 }
|
|
1093
|
+
// jz overflow-canon `select(wrap(trunc_sat(…)), 0, ≠Inf)`: finite (grids) ⇒ the trunc branch.
|
|
1094
|
+
if (n.length === 4 && isI32Const(n[2]) && isArr(n[3]) && n[3][0] === 'f64.ne' && isArr(n[3][2]) && n[3][2][0] === 'f64.const' && /inf/i.test(String(n[3][2][1])))
|
|
1095
|
+
return ivCoeff(n[1])
|
|
1096
|
+
}
|
|
1097
|
+
return null
|
|
1098
|
+
}
|
|
1099
|
+
const countSets = (name) => {
|
|
1100
|
+
let k = 0
|
|
1101
|
+
const w = (x) => { if (!isArr(x)) return; if ((x[0] === 'local.set' || x[0] === 'local.tee') && x[1] === name) k++; for (let i = 1; i < x.length; i++) w(x[i]) }
|
|
1102
|
+
for (const s of body) w(s)
|
|
1103
|
+
return k
|
|
1104
|
+
}
|
|
1105
|
+
// Derived IVs: `c = INV + x` (coeff 1) or a toroidal wrap-select; set exactly once, first access a
|
|
1106
|
+
// write. RECURSES into nested tees — O3 CSEs `rc+x` into `(local.tee $pe (i32.add rc x))` inside a
|
|
1107
|
+
// load address, reused by the store. (ivCoeff returns 1 for a wrap-select and flags needsPeel.)
|
|
1108
|
+
for (let pass = 0; pass < 4; pass++) {
|
|
1109
|
+
let added = false
|
|
1110
|
+
const consider = (name, def) => {
|
|
1111
|
+
if (derived.has(name) || fnLocals.get(name) !== 'i32' || countSets(name) !== 1 || ivCoeff(def) !== 1) return
|
|
1112
|
+
let fk = null; for (const t of body) { const k = firstAccess(t, name); if (k) { fk = k; break } }
|
|
1113
|
+
if (fk === 'write') { derived.add(name); added = true }
|
|
1114
|
+
}
|
|
1115
|
+
const walk = (x) => { if (!isArr(x)) return; if ((x[0] === 'local.set' || x[0] === 'local.tee') && typeof x[1] === 'string' && x.length === 3) consider(x[1], x[2]); for (let i = 1; i < x.length; i++) walk(x[i]) }
|
|
1116
|
+
for (const s of body) walk(s)
|
|
1117
|
+
if (!added) break
|
|
1118
|
+
}
|
|
1119
|
+
|
|
1120
|
+
// Scan loads/stores: address `base + (IDX<<K)`, ivCoeff(IDX)=1, base invariant.
|
|
1121
|
+
let laneType = null, stride = -1
|
|
1122
|
+
const offTees = new Map() // $pe → IDX expr (from $pe = IDX<<K)
|
|
1123
|
+
const addrTees = new Map() // $ab → { base, idx }
|
|
1124
|
+
const sites = [] // { kind, base, idx, memBytes }
|
|
1125
|
+
const isInvBase = (b) => (isArr(b) && b[0] === 'global.get') || (isLocalGet(b) && !writes.has(b[1]))
|
|
1126
|
+
const matchAddr = (addr, expectStride = stride) => {
|
|
1127
|
+
let teeName = null, n = addr
|
|
1128
|
+
if (isArr(n) && n[0] === 'local.tee' && n.length === 3) { teeName = n[1]; n = n[2] }
|
|
1129
|
+
if (isLocalGet(n) && addrTees.has(n[1])) { const e = addrTees.get(n[1]); if (teeName) addrTees.set(teeName, e); return e }
|
|
1130
|
+
if (!isArr(n) || n[0] !== 'i32.add' || n.length !== 3) return null
|
|
1131
|
+
const tryOff = (off) => {
|
|
1132
|
+
let ot = null, o = off
|
|
1133
|
+
if (isArr(o) && o[0] === 'local.tee' && o.length === 3) { ot = o[1]; o = o[2] }
|
|
1134
|
+
if (isLocalGet(o) && offTees.has(o[1])) return { idx: offTees.get(o[1]) }
|
|
1135
|
+
if (isArr(o) && o[0] === 'i32.shl' && o.length === 3 && isI32Const(o[2]) && (1 << o[2][1]) === expectStride && ivCoeff(o[1]) === 1) {
|
|
1136
|
+
if (ot) offTees.set(ot, o[1])
|
|
1137
|
+
return { idx: o[1] }
|
|
1138
|
+
}
|
|
1139
|
+
return null
|
|
1140
|
+
}
|
|
1141
|
+
for (const [bi, oi] of [[1, 2], [2, 1]]) {
|
|
1142
|
+
if (!isInvBase(n[bi])) continue
|
|
1143
|
+
const om = tryOff(n[oi])
|
|
1144
|
+
if (om) { const e = { base: n[bi], idx: om.idx }; if (teeName) addrTees.set(teeName, e); return e }
|
|
1145
|
+
}
|
|
1146
|
+
return null
|
|
1147
|
+
}
|
|
1148
|
+
const scan = (node, parent, pi) => {
|
|
1149
|
+
if (!isArr(node)) return true
|
|
1150
|
+
const op = node[0]
|
|
1151
|
+
if (LOAD_OPS[op]) {
|
|
1152
|
+
let addr = node[1], memBytes = 0
|
|
1153
|
+
if (typeof addr === 'string' && addr.startsWith('offset=')) { memBytes = +addr.slice(7); addr = node[2] }
|
|
1154
|
+
const lt = LOAD_OPS[op]
|
|
1155
|
+
if (laneType == null) { if (lt !== 'f64' && lt !== 'f32') return false; laneType = lt; stride = LANE_INFO[lt].stride }
|
|
1156
|
+
else if (lt !== laneType && !(lt === 'f32' && laneType === 'f64')) return false // f32→f64 widening OK
|
|
1157
|
+
// Validate the address at the LOAD's own element stride (f64=8, widening f32=4); the index
|
|
1158
|
+
// must still be stride-1 in elements (ivCoeff===1). The f32 load is promoted in liftExprV.
|
|
1159
|
+
const m = matchAddr(addr, LANE_INFO[lt].stride)
|
|
1160
|
+
if (!m) return false
|
|
1161
|
+
sites.push({ kind: 'load', base: m.base, idx: m.idx, memBytes })
|
|
1162
|
+
return true
|
|
1163
|
+
}
|
|
1164
|
+
if (STORE_OPS[op]) {
|
|
1165
|
+
if (node.length !== 3) return false
|
|
1166
|
+
const st = STORE_OPS[op]
|
|
1167
|
+
if (laneType == null) { if (st !== 'f64' && st !== 'f32') return false; laneType = st; stride = LANE_INFO[st].stride }
|
|
1168
|
+
else if (st !== laneType) return false
|
|
1169
|
+
const m = matchAddr(node[1])
|
|
1170
|
+
if (!m) return false
|
|
1171
|
+
sites.push({ kind: 'store', base: m.base, idx: m.idx, memBytes: 0 })
|
|
1172
|
+
return scan(node[2], node, 2) // value child only
|
|
1173
|
+
}
|
|
1174
|
+
if ((op === 'local.set' || op === 'local.tee') && typeof node[1] === 'string' && node.length === 3) {
|
|
1175
|
+
const v = node[2]
|
|
1176
|
+
if (isArr(v) && v[0] === 'i32.shl' && v.length === 3 && isI32Const(v[2]) && stride > 0 && (1 << v[2][1]) === stride && ivCoeff(v[1]) === 1) offTees.set(node[1], v[1])
|
|
1177
|
+
else matchAddr(['local.tee', node[1], v])
|
|
1178
|
+
}
|
|
1179
|
+
for (let i = 1; i < node.length; i++) if (!scan(node[i], node, i)) return false
|
|
1180
|
+
return true
|
|
1181
|
+
}
|
|
1182
|
+
for (const s of body) if (!scan(s, null, -1)) return null
|
|
1183
|
+
if (!laneType || !sites.some(s => s.kind === "store") || !sites.some(s => s.kind === "load")) return null
|
|
1184
|
+
|
|
1185
|
+
// In-place / loop-carried gate: every access to a WRITTEN base must touch the
|
|
1186
|
+
// SAME element (idx + memarg). Else SIMD reads stale data vs scalar.
|
|
1187
|
+
const elemKey = (s) => `${JSON.stringify(normTee(s.idx))}@${s.memBytes / stride}`
|
|
1188
|
+
for (const st of sites) {
|
|
1189
|
+
if (st.kind !== 'store') continue
|
|
1190
|
+
for (const s of sites) if (exprEq(normTee(s.base), normTee(st.base)) && elemKey(s) !== elemKey(st)) return null
|
|
1191
|
+
}
|
|
1192
|
+
// A pure offset-0 map (every access the same element, no memarg) is tryVectorize's
|
|
1193
|
+
// job — it ran first. Nothing stencil-specific here. (Defensive; ?? order ensures it.)
|
|
1194
|
+
const k0 = elemKey(sites[0])
|
|
1195
|
+
if (sites.every(s => elemKey(s) === k0)) return null
|
|
1196
|
+
|
|
1197
|
+
// Classify locals by TYPE: i32 → addr (index/address, kept scalar), laneType
|
|
1198
|
+
// written → lane (first access must be a write), laneType unwritten → invariant.
|
|
1199
|
+
const referenced = new Set()
|
|
1200
|
+
const collectRefs = (n) => { if (!isArr(n)) return; if ((n[0] === 'local.get' || n[0] === 'local.set' || n[0] === 'local.tee') && typeof n[1] === 'string') referenced.add(n[1]); for (let i = 1; i < n.length; i++) collectRefs(n[i]) }
|
|
1201
|
+
for (const s of body) collectRefs(s)
|
|
1202
|
+
const localKind = new Map()
|
|
1203
|
+
for (const name of referenced) {
|
|
1204
|
+
if (name === incVar) continue
|
|
1205
|
+
const ty = fnLocals.get(name)
|
|
1206
|
+
if (ty === 'i32') { localKind.set(name, 'addr'); continue }
|
|
1207
|
+
if (ty === laneType) {
|
|
1208
|
+
if (writes.has(name)) {
|
|
1209
|
+
let fk = null; for (const s of body) { const k = firstAccess(s, name); if (k) { fk = k; break } }
|
|
1210
|
+
if (fk === 'read') return null // loop-carried float local
|
|
1211
|
+
localKind.set(name, 'lane')
|
|
1212
|
+
} else localKind.set(name, 'invariant')
|
|
1213
|
+
continue
|
|
1214
|
+
}
|
|
1215
|
+
if (!writes.has(name)) { localKind.set(name, 'invariant'); continue }
|
|
1216
|
+
return null // written non-i32 non-lane local
|
|
1217
|
+
}
|
|
1218
|
+
|
|
1219
|
+
// Lift through the shared lifter (addresses kept verbatim; loads → v128.load).
|
|
1220
|
+
const newLanedLocals = new Map(), extraLocals = []
|
|
1221
|
+
const ctx = { laneType, incVar, rampVar: null, rampTemp: null, widenLoads: false, localKind, newLanedLocals, extraLocals, freshIdRef, fail: false, failReason: null }
|
|
1222
|
+
const lifted = []
|
|
1223
|
+
for (const s of body) {
|
|
1224
|
+
const r = liftStmt(s, ctx)
|
|
1225
|
+
if (ctx.fail) return null
|
|
1226
|
+
if (r != null) { if (Array.isArray(r) && r[0] === '__seq__') lifted.push(...r.slice(1)); else lifted.push(r) }
|
|
1227
|
+
}
|
|
1228
|
+
if (!lifted.length) return null
|
|
1229
|
+
|
|
1230
|
+
const id = freshIdRef.next++
|
|
1231
|
+
const simdBoundName = `$__simd_bound${id}`, simdBrkLabel = `$__simd_brk${id}`, simdLoopLabel = `$__simd_loop${id}`
|
|
1232
|
+
const info = LANE_INFO[laneType], lanes = info.lanes
|
|
1233
|
+
const boundExpr = cloneNode(bound) // cloned: also lives in the scalar-tail exit guard
|
|
1234
|
+
// Overshoot-safe bound: a full lanes-wide chunk [x,x+lanes) must stay < bound for
|
|
1235
|
+
// ANY start x (stencils start at 1). `bound-(lanes-1)` — NOT `& ~(lanes-1)`, which
|
|
1236
|
+
// overshoots for a non-multiple start. SIMD reads ⊆ scalar reads ⇒ no new OOB.
|
|
1237
|
+
// A toroidal-wrap stencil additionally PEELS both boundary columns scalar: cap the SIMD at
|
|
1238
|
+
// `min(bound, …rightWrapBoundaries) - (lanes-1)` so no chunk reaches a right-wrap column x=B,
|
|
1239
|
+
// and run x=0 scalar below (where the left wrap fires) so the SIMD starts in the wrap-free interior.
|
|
1240
|
+
const simdCap = rightBs.reduce((acc, b) => ['select', cloneNode(b), acc, ['i32.lt_s', cloneNode(b), acc]], boundExpr)
|
|
1241
|
+
const boundSetup = ['local.set', simdBoundName, ['i32.sub', simdCap, ['i32.const', lanes - 1]]]
|
|
1242
|
+
const simdBlock = ['block', simdBrkLabel,
|
|
1243
|
+
['loop', simdLoopLabel,
|
|
1244
|
+
['br_if', simdBrkLabel, ['i32.eqz', ['i32.lt_s', ['local.get', incVar], ['local.get', simdBoundName]]]],
|
|
1245
|
+
...lifted,
|
|
1246
|
+
['local.set', incVar, ['i32.add', ['local.get', incVar], ['i32.const', lanes]]],
|
|
1247
|
+
['br', simdLoopLabel]]]
|
|
1248
|
+
// Left-boundary peel for a wrap stencil: run the original scalar body once for x=0 (where the wrap
|
|
1249
|
+
// takes its WRAP branch), advancing x to 1 so the SIMD starts in the wrap-free interior. Guarded so
|
|
1250
|
+
// an empty loop (x ≥ bound) is untouched. Right boundary + odd tail: the kept scalar tail (blockNode).
|
|
1251
|
+
const peelStmts = needsPeel
|
|
1252
|
+
? [['if', ['i32.lt_s', ['local.get', incVar], cloneNode(bound)],
|
|
1253
|
+
['then', ...body.map(cloneNode), cloneNode(bl.loopNode[bl.incIdx])]]]
|
|
1254
|
+
: []
|
|
1255
|
+
// LICM-hoisted $__li invariants run ahead of the SIMD block (the scalar tail's
|
|
1256
|
+
// copy inside bl.blockNode re-runs them harmlessly — pure & loop-invariant).
|
|
1257
|
+
const wrapper = ['block', ...preamble.map(cloneNode), ...peelStmts, boundSetup, simdBlock, bl.blockNode]
|
|
1258
|
+
const newLocalDecls = [['local', simdBoundName, 'i32'], ...[...newLanedLocals.values()].map(({ laneName }) => ['local', laneName, 'v128']), ...extraLocals]
|
|
1259
|
+
return { wrapper, newLocalDecls }
|
|
1260
|
+
}
|
|
1261
|
+
|
|
897
1262
|
// ---- Reduction recognizer -------------------------------------------------
|
|
898
1263
|
//
|
|
899
1264
|
// Matches inner loops of shape:
|
|
@@ -912,31 +1277,10 @@ function tryVectorize(blockNode, fnLocals, freshIdRef) {
|
|
|
912
1277
|
// Float adds are not strictly associative — vectorized reduction differs
|
|
913
1278
|
// from scalar reduction by ulps. Acceptable when bit-exact equality is not
|
|
914
1279
|
// required (which it isn't, by spec, in JS engines either).
|
|
915
|
-
function tryReduceVectorize(
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
let blockLabel = null
|
|
920
|
-
let loopNode = null
|
|
921
|
-
for (let i = 1; i < blockNode.length; i++) {
|
|
922
|
-
const c = blockNode[i]
|
|
923
|
-
if (typeof c === 'string' && c.startsWith('$') && blockLabel == null && i === 1) { blockLabel = c; continue }
|
|
924
|
-
if (isArr(c) && c[0] === 'loop') {
|
|
925
|
-
if (loopNode) return null
|
|
926
|
-
loopNode = c
|
|
927
|
-
} else if (isArr(c)) return null
|
|
928
|
-
}
|
|
929
|
-
if (!loopNode || !blockLabel) return null
|
|
930
|
-
const loopLabel = typeof loopNode[1] === 'string' && loopNode[1].startsWith('$') ? loopNode[1] : null
|
|
931
|
-
if (!loopLabel) return null
|
|
932
|
-
const endIdx = loopNode.length - 1
|
|
933
|
-
if (!(isArr(loopNode[endIdx]) && loopNode[endIdx][0] === 'br' && loopNode[endIdx][1] === loopLabel)) return null
|
|
934
|
-
const incIdx = endIdx - 1
|
|
935
|
-
const incVar = matchInc1(loopNode[incIdx])
|
|
936
|
-
if (!incVar) return null
|
|
937
|
-
const exitInfo = matchExitBrIf(loopNode[2], blockLabel)
|
|
938
|
-
if (!exitInfo) return null
|
|
939
|
-
if (exitInfo.ind !== incVar) return null
|
|
1280
|
+
function tryReduceVectorize(bl, fnLocals, freshIdRef, multiAcc = false) {
|
|
1281
|
+
// Same scaffold as tryVectorize, but no preamble: a reduction block is just the loop.
|
|
1282
|
+
if (!bl || bl.preamble.length) return null
|
|
1283
|
+
const { loopNode, incIdx, incVar } = bl
|
|
940
1284
|
|
|
941
1285
|
// Body is either a bare single-statement reduction —
|
|
942
1286
|
// (local.set $acc (OP (local.get $acc) EXPR)) add/xor/and/or
|
|
@@ -1016,6 +1360,41 @@ function tryReduceVectorize(blockNode, fnLocals, freshIdRef) {
|
|
|
1016
1360
|
exprNode = rhs[2]
|
|
1017
1361
|
} else return null
|
|
1018
1362
|
|
|
1363
|
+
// Offset-indexed reductions (matmul `s += A[ai+k]*Bt[bj+k]`): the index `ai+k`
|
|
1364
|
+
// lowers to `(i32.shl (i32.add ai i) K)`, which matchLaneAddr rejects (the IV is
|
|
1365
|
+
// not the bare shift operand). Fold the loop-invariant part into the base —
|
|
1366
|
+
// (base + (INV+i)<<K) → ((base + INV<<K) + i<<K)
|
|
1367
|
+
// so the offset is the bare IV the matcher/lifter already accept. The byte address
|
|
1368
|
+
// is unchanged, so the v128.load reads the same consecutive pair → bit-exact. INV
|
|
1369
|
+
// must be loop-invariant (not written in the loop) and IV-free (coefficient 1).
|
|
1370
|
+
{
|
|
1371
|
+
const writtenInLoop = new Set()
|
|
1372
|
+
;(function wr(n) { if (!isArr(n)) return; if ((n[0] === 'local.set' || n[0] === 'local.tee') && typeof n[1] === 'string') writtenInLoop.add(n[1]); for (let i = 1; i < n.length; i++) wr(n[i]) })(loopNode)
|
|
1373
|
+
const invFree = (n) => !isArr(n) || (!(n[0] === 'local.get' && (n[1] === incVar || writtenInLoop.has(n[1]))) && n.every((c, i) => i === 0 || invFree(c)))
|
|
1374
|
+
let folded = false
|
|
1375
|
+
const foldAddr = (n) => {
|
|
1376
|
+
if (!isArr(n)) return n
|
|
1377
|
+
if (n[0] === 'i32.add' && n.length === 3) {
|
|
1378
|
+
for (const [base, off] of [[n[1], n[2]], [n[2], n[1]]]) {
|
|
1379
|
+
if (isArr(off) && off[0] === 'i32.shl' && off.length === 3 && isArr(off[1]) && off[1][0] === 'i32.add' && off[1].length === 3) {
|
|
1380
|
+
const k = constNum(off[2]), x = off[1][1], y = off[1][2]
|
|
1381
|
+
const xIV = isLocalGet(x, incVar), yIV = isLocalGet(y, incVar)
|
|
1382
|
+
if (k != null && k >= 0 && k <= 3 && xIV !== yIV) {
|
|
1383
|
+
const inv = xIV ? y : x
|
|
1384
|
+
if (invFree(inv)) {
|
|
1385
|
+
folded = true
|
|
1386
|
+
return ['i32.add', ['i32.add', foldAddr(base), ['i32.shl', inv, ['i32.const', k]]], ['i32.shl', ['local.get', incVar], ['i32.const', k]]]
|
|
1387
|
+
}
|
|
1388
|
+
}
|
|
1389
|
+
}
|
|
1390
|
+
}
|
|
1391
|
+
}
|
|
1392
|
+
return n.map(foldAddr)
|
|
1393
|
+
}
|
|
1394
|
+
const fe = foldAddr(exprNode)
|
|
1395
|
+
if (folded) exprNode = fe
|
|
1396
|
+
}
|
|
1397
|
+
|
|
1019
1398
|
// Accumulator's declared local type must match the lane element type.
|
|
1020
1399
|
// Exception: the widening byte/short sum — i32 accumulator fed by ONE bare
|
|
1021
1400
|
// narrow load (`s += u8[i]`), whose LANE type is i8/i16 but reduces into i32.
|
|
@@ -1043,11 +1422,9 @@ function tryReduceVectorize(blockNode, fnLocals, freshIdRef) {
|
|
|
1043
1422
|
}
|
|
1044
1423
|
if (!widen && accType !== (reduceEntry.accI32 ? 'i32' : reduceEntry.accF64 ? 'f64' : reduceEntry.laneType)) return null
|
|
1045
1424
|
|
|
1046
|
-
// Bound
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
if (isArr(bound) && bound[0] === 'local.get' && typeof bound[1] === 'string') boundLocal = bound[1]
|
|
1050
|
-
else if (!isI32Const(bound)) return null
|
|
1425
|
+
// Bound must be loop-invariant: (local.get $L) or (i32.const N).
|
|
1426
|
+
const { bound, boundLocal } = bl
|
|
1427
|
+
if (!boundLocal && !isI32Const(bound)) return null
|
|
1051
1428
|
|
|
1052
1429
|
// Scan EXPR for lane-aligned loads. Stores forbidden. Re-references of
|
|
1053
1430
|
// accName forbidden (the accumulator only appears in the outer wrapper).
|
|
@@ -1100,7 +1477,7 @@ function tryReduceVectorize(blockNode, fnLocals, freshIdRef) {
|
|
|
1100
1477
|
for (const name of addrLocals.keys()) localKind.set(name, 'addr')
|
|
1101
1478
|
for (const name of offsetTees.keys()) localKind.set(name, 'addr')
|
|
1102
1479
|
|
|
1103
|
-
const ctx = { laneType, incVar, localKind, newLanedLocals: new Map(), extraLocals: [], freshIdRef, fail: false, failReason: null }
|
|
1480
|
+
const ctx = { laneType, incVar, rampVar: null, rampTemp: null, widenLoads: false, localKind, newLanedLocals: new Map(), extraLocals: [], freshIdRef, fail: false, failReason: null }
|
|
1104
1481
|
const liftedExpr = liftExprV(exprNode, ctx)
|
|
1105
1482
|
if (ctx.fail) return null
|
|
1106
1483
|
if (ctx.newLanedLocals.size > 0 || ctx.extraLocals.length > 0) return null
|
|
@@ -1108,28 +1485,55 @@ function tryReduceVectorize(blockNode, fnLocals, freshIdRef) {
|
|
|
1108
1485
|
// Synthesize SIMD prefix block + horizontal reduce + (preserved scalar tail).
|
|
1109
1486
|
const id = freshIdRef.next++
|
|
1110
1487
|
const simdBoundName = `$__simd_bound${id}`
|
|
1111
|
-
const simdAccName = `$__simd_acc${id}`
|
|
1488
|
+
const simdAccName = `$__simd_acc${id}` // accumulator 0 — the one the merge folds
|
|
1112
1489
|
const simdBrkLabel = `$__simd_brk${id}`
|
|
1113
1490
|
const simdLoopLabel = `$__simd_loop${id}`
|
|
1114
1491
|
const info = LANE_INFO[laneType]
|
|
1115
1492
|
const lanes = info.lanes
|
|
1116
1493
|
const boundExpr = boundLocal ? ['local.get', boundLocal] : bound
|
|
1117
1494
|
|
|
1495
|
+
// Multi-accumulator unroll. A reduction's loop-carried accumulator is a latency
|
|
1496
|
+
// chain — each iteration's op waits on the previous result, so a single vector
|
|
1497
|
+
// accumulator runs at FP-op latency, not throughput. N INDEPENDENT accumulators
|
|
1498
|
+
// (each summing every Nth lane-chunk, combined at the end) expose instruction-
|
|
1499
|
+
// level parallelism and hide the latency — ~2x on a dot/FIR reduction. It is
|
|
1500
|
+
// DETERMINISTIC: only the reduction's reassociation widens (8 partial sums vs 2),
|
|
1501
|
+
// the same kind the existing 2-lane fold already does, identical on every engine.
|
|
1502
|
+
// Restricted to the plain horizontal-fold FP path (not min/max-select, the
|
|
1503
|
+
// narrow-widening sums, or NaN-canon — those have their own fold shapes).
|
|
1504
|
+
const plainReduce = !reduceEntry.minmaxSelect && !widen && canonC == null
|
|
1505
|
+
const NACC = (multiAcc && plainReduce && (laneType === 'f64' || laneType === 'f32')) ? 4 : 1
|
|
1506
|
+
const accK = (k) => k === 0 ? simdAccName : `$__simd_acc${id}_${k}`
|
|
1507
|
+
const laneBytes = lanes * stride
|
|
1508
|
+
|
|
1118
1509
|
// Widening sum: the ACCUMULATOR vector is i32x4 regardless of the (narrow)
|
|
1119
1510
|
// lane type; each iteration's 16-byte load collapses via extadd_pairwise.
|
|
1120
1511
|
const accSplat = widen ? 'i32x4.splat' : info.splat
|
|
1121
1512
|
const accumOperand = widen ? widen.steps.reduce((e, s) => [s, e], liftedExpr) : liftedExpr
|
|
1122
|
-
|
|
1513
|
+
// Accumulator k reads the same lane-aligned data as acc 0, shifted by k chunks
|
|
1514
|
+
// (k·laneBytes). Acc 0 keeps the address tees (it sets them); acc k>0 reads the
|
|
1515
|
+
// tee'd address (normTee → local.get) and adds the byte offset to each load.
|
|
1516
|
+
const offsetLoads = (node, off) => !isArr(node) ? node
|
|
1517
|
+
: node[0] === 'v128.load' ? ['v128.load', ['i32.add', node[1], ['i32.const', off]]]
|
|
1518
|
+
: node.map(c => offsetLoads(c, off))
|
|
1519
|
+
const accOperandFor = (k) => k === 0 ? accumOperand : offsetLoads(normTee(accumOperand), k * laneBytes)
|
|
1520
|
+
|
|
1521
|
+
const initAcc = []
|
|
1522
|
+
for (let k = 0; k < NACC; k++) initAcc.push(['local.set', accK(k), [accSplat, reduceEntry.constNode ?? reduceEntry.identity]])
|
|
1523
|
+
const loopBody = []
|
|
1524
|
+
for (let k = 0; k < NACC; k++) loopBody.push(['local.set', accK(k), [reduceEntry.simd, ['local.get', accK(k)], accOperandFor(k)]])
|
|
1525
|
+
loopBody.push(['local.set', incVar, ['i32.add', ['local.get', incVar], ['i32.const', lanes * NACC]]])
|
|
1123
1526
|
const simdBlock = ['block', simdBrkLabel,
|
|
1124
1527
|
['loop', simdLoopLabel,
|
|
1125
1528
|
['br_if', simdBrkLabel,
|
|
1126
1529
|
['i32.eqz', ['i32.lt_s', ['local.get', incVar], ['local.get', simdBoundName]]]],
|
|
1127
|
-
|
|
1128
|
-
[reduceEntry.simd, ['local.get', simdAccName], accumOperand]],
|
|
1129
|
-
['local.set', incVar, ['i32.add', ['local.get', incVar], ['i32.const', lanes]]],
|
|
1530
|
+
...loopBody,
|
|
1130
1531
|
['br', simdLoopLabel]
|
|
1131
1532
|
]
|
|
1132
1533
|
]
|
|
1534
|
+
// Combine the N accumulators into acc 0 (lane-wise) before the horizontal fold.
|
|
1535
|
+
const combineAccs = []
|
|
1536
|
+
for (let k = 1; k < NACC; k++) combineAccs.push(['local.set', simdAccName, [reduceEntry.simd, ['local.get', simdAccName], ['local.get', accK(k)]]])
|
|
1133
1537
|
|
|
1134
1538
|
// Horizontal fold + merge into the live accumulator.
|
|
1135
1539
|
const extraDecls = []
|
|
@@ -1181,25 +1585,126 @@ function tryReduceVectorize(blockNode, fnLocals, freshIdRef) {
|
|
|
1181
1585
|
// at i=1, which `& ~(lanes-1)` masking would run one lane past the end). For
|
|
1182
1586
|
// a lane-aligned start this yields the same iteration set as masking; the
|
|
1183
1587
|
// scalar tail (original `i<bound` guard) cleans up regardless.
|
|
1184
|
-
|
|
1588
|
+
// A full N·lanes-wide step (all N accumulators) must stay in range.
|
|
1589
|
+
const boundSetup = ['local.set', simdBoundName, ['i32.sub', boundExpr, ['i32.const', lanes * NACC - 1]]]
|
|
1185
1590
|
|
|
1186
1591
|
// Narrow-widened entries seed the vector acc with a LANE-domain neutral (e.g.
|
|
1187
1592
|
// 0 for u8-max) — only neutral once real lanes fold in. Guard the whole SIMD
|
|
1188
1593
|
// prefix incl. the merge so a zero-iteration range can't clamp the live acc
|
|
1189
1594
|
// toward the identity. Full-width entries use absolute neutrals; unguarded.
|
|
1595
|
+
// (The guarded path is always NACC=1 — accI32/accF64 are non-plain reductions.)
|
|
1190
1596
|
const core = reduceEntry.accI32 || reduceEntry.accF64
|
|
1191
1597
|
? [['if', ['i32.lt_s', ['local.get', incVar], ['local.get', simdBoundName]],
|
|
1192
|
-
['then', initAcc, simdBlock, ...mergeStmts]]]
|
|
1193
|
-
: [initAcc, simdBlock, ...mergeStmts]
|
|
1194
|
-
const wrapper = ['block', boundSetup, ...core, blockNode]
|
|
1598
|
+
['then', ...initAcc, simdBlock, ...combineAccs, ...mergeStmts]]]
|
|
1599
|
+
: [...initAcc, simdBlock, ...combineAccs, ...mergeStmts]
|
|
1600
|
+
const wrapper = ['block', boundSetup, ...core, bl.blockNode]
|
|
1195
1601
|
const newLocalDecls = [
|
|
1196
1602
|
['local', simdBoundName, 'i32'],
|
|
1197
1603
|
['local', simdAccName, 'v128'],
|
|
1604
|
+
...Array.from({ length: NACC - 1 }, (_, k) => ['local', accK(k + 1), 'v128']),
|
|
1198
1605
|
...extraDecls,
|
|
1199
1606
|
]
|
|
1200
1607
|
return { wrapper, newLocalDecls }
|
|
1201
1608
|
}
|
|
1202
1609
|
|
|
1610
|
+
// Bit-exact f64 map-reduce (the direct-summation n-body force loop). A loop that
|
|
1611
|
+
// accumulates one or more f64 reductions whose per-iteration contribution is computed
|
|
1612
|
+
// INDEPENDENTLY of the accumulators. Process 2 iterations per step in f64x2 — every lane
|
|
1613
|
+
// op (sub/mul/add/div/sqrt) is IEEE-754-identical to scalar f64 (no FMA in non-relaxed
|
|
1614
|
+
// SIMD) — then accumulate each accumulator's two lane contributions IN SCALAR ORDER, so
|
|
1615
|
+
// the reduction is BIT-EXACT (unlike the reassociating tryReduceVectorize). Wins when the
|
|
1616
|
+
// per-element compute is expensive (a sqrt + reciprocal) so the 2-wide arithmetic
|
|
1617
|
+
// outweighs the serial lane-accumulation. The original block is preserved as the ≤1
|
|
1618
|
+
// scalar remainder, continuing the accumulators. Returns {wrapper, newLocalDecls} or null.
|
|
1619
|
+
function tryMapReduceVectorize(bl, fnLocals, freshIdRef) {
|
|
1620
|
+
if (!bl || bl.preamble.length) return null
|
|
1621
|
+
const { incVar, bound, boundLocal, body } = bl
|
|
1622
|
+
if (!boundLocal && !isI32Const(bound)) return null
|
|
1623
|
+
if (body.length < 2) return null
|
|
1624
|
+
|
|
1625
|
+
// Every body stmt must be `(local.set $x EXPR)`. An accumulator reads its own target
|
|
1626
|
+
// through `f64.add` (`acc = acc + EXPR`); the rest are per-iteration lane locals. (One
|
|
1627
|
+
// write per acc — duplicate writes would break the per-acc ordering.)
|
|
1628
|
+
for (const s of body) if (!(isArr(s) && s[0] === 'local.set' && typeof s[1] === 'string' && s.length === 3)) return null
|
|
1629
|
+
const accSet = new Set()
|
|
1630
|
+
for (const s of body) if (isArr(s[2]) && s[2][0] === 'f64.add' && isLocalGet(s[2][1], s[1])) accSet.add(s[1])
|
|
1631
|
+
if (!accSet.size) return null
|
|
1632
|
+
const writeCount = new Map()
|
|
1633
|
+
for (const s of body) writeCount.set(s[1], (writeCount.get(s[1]) || 0) + 1)
|
|
1634
|
+
for (const a of accSet) { if (writeCount.get(a) !== 1 || fnLocals.get(a) !== 'f64') return null }
|
|
1635
|
+
|
|
1636
|
+
// Address tees: locals that equal `ind << K`. f64 loads must be stride-8 (K=3) so one
|
|
1637
|
+
// f64x2.load (16 bytes) covers iterations j and j+1 — consecutive elements.
|
|
1638
|
+
const offsetTees = new Map()
|
|
1639
|
+
const allNames = new Set()
|
|
1640
|
+
const gather = (n) => { if (!isArr(n)) return; if ((n[0] === 'local.set' || n[0] === 'local.tee') && typeof n[1] === 'string') allNames.add(n[1]); for (let i = 1; i < n.length; i++) gather(n[i]) }
|
|
1641
|
+
for (const s of body) gather(s)
|
|
1642
|
+
for (const name of allNames) { const k = _offsetLocalStride(body, name, incVar); if (k != null) offsetTees.set(name, k) }
|
|
1643
|
+
|
|
1644
|
+
// f64x2 lift: load → f64x2.load (2 consecutive), const/invariant → splat, a lane local
|
|
1645
|
+
// → its f64x2 temp, sub/mul/add/div → f64x2.OP, sqrt → f64x2.sqrt. Anything else bails.
|
|
1646
|
+
const laneV = new Map()
|
|
1647
|
+
const newLocalDecls = []
|
|
1648
|
+
const fresh = () => { const n = `$__mr${freshIdRef.next++}`; newLocalDecls.push(['local', n, 'v128']); return n }
|
|
1649
|
+
let bad = false
|
|
1650
|
+
const lift = (e) => {
|
|
1651
|
+
if (bad || !isArr(e)) { bad = true; return null }
|
|
1652
|
+
const op = e[0]
|
|
1653
|
+
if (op === 'f64.const') return ['f64x2.splat', e]
|
|
1654
|
+
if (op === 'f64.load') {
|
|
1655
|
+
const m = matchLaneAddr(e[1], incVar, new Map(), offsetTees)
|
|
1656
|
+
if (!m || m.strideLog2 !== 3) { bad = true; return null }
|
|
1657
|
+
return ['v128.load', e[1]] // 16 bytes = 2 consecutive f64s; the f64x2 op reads them
|
|
1658
|
+
}
|
|
1659
|
+
if (op === 'local.get' && typeof e[1] === 'string') {
|
|
1660
|
+
if (e[1] === incVar || accSet.has(e[1])) { bad = true; return null } // IV-as-data / acc-dependent contribution
|
|
1661
|
+
if (laneV.has(e[1])) return ['local.get', laneV.get(e[1])]
|
|
1662
|
+
if (writeCount.has(e[1])) { bad = true; return null } // a body local used BEFORE its set this iteration → loop-carried, not a fresh lane
|
|
1663
|
+
return ['f64x2.splat', e] // genuine loop-invariant scalar (xi, …)
|
|
1664
|
+
}
|
|
1665
|
+
if ((op === 'f64.add' || op === 'f64.sub' || op === 'f64.mul' || op === 'f64.div') && e.length === 3)
|
|
1666
|
+
return [op.replace('f64.', 'f64x2.'), lift(e[1]), lift(e[2])]
|
|
1667
|
+
if (op === 'f64.sqrt' && e.length === 2) return ['f64x2.sqrt', lift(e[1])]
|
|
1668
|
+
bad = true; return null
|
|
1669
|
+
}
|
|
1670
|
+
|
|
1671
|
+
// Lifted body: setup lanes → f64x2 temps (in order, so the offset tee in the first load
|
|
1672
|
+
// is set before later loads read it); each accumulator → a temp + two in-order adds.
|
|
1673
|
+
const lifted = []
|
|
1674
|
+
for (const s of body) {
|
|
1675
|
+
if (accSet.has(s[1])) {
|
|
1676
|
+
const cV = fresh()
|
|
1677
|
+
const v = lift(s[2][2])
|
|
1678
|
+
if (bad) return null
|
|
1679
|
+
lifted.push(['local.set', cV, v],
|
|
1680
|
+
['local.set', s[1], ['f64.add', ['local.get', s[1]], ['f64x2.extract_lane', 0, ['local.get', cV]]]],
|
|
1681
|
+
['local.set', s[1], ['f64.add', ['local.get', s[1]], ['f64x2.extract_lane', 1, ['local.get', cV]]]])
|
|
1682
|
+
} else {
|
|
1683
|
+
const tv = fresh()
|
|
1684
|
+
laneV.set(s[1], tv)
|
|
1685
|
+
const v = lift(s[2])
|
|
1686
|
+
if (bad) return null
|
|
1687
|
+
lifted.push(['local.set', tv, v])
|
|
1688
|
+
}
|
|
1689
|
+
}
|
|
1690
|
+
if (bad || !lifted.length) return null
|
|
1691
|
+
|
|
1692
|
+
// SIMD prefix over the even prefix [0, bound & ~1); the original block is the ≤1 scalar
|
|
1693
|
+
// remainder (continues j and the accumulators). IV advances by 2.
|
|
1694
|
+
const id = freshIdRef.next++
|
|
1695
|
+
const simdBoundName = `$__mrb${id}`, simdBrk = `$__mrbrk${id}`, simdLoop = `$__mrl${id}`
|
|
1696
|
+
const boundExpr = boundLocal ? ['local.get', boundLocal] : ['i32.const', constNum(bound)]
|
|
1697
|
+
const simdBlock = ['block', simdBrk,
|
|
1698
|
+
['loop', simdLoop,
|
|
1699
|
+
['br_if', simdBrk, ['i32.eqz', ['i32.lt_s', ['local.get', incVar], ['local.get', simdBoundName]]]],
|
|
1700
|
+
...lifted,
|
|
1701
|
+
['local.set', incVar, ['i32.add', ['local.get', incVar], ['i32.const', 2]]],
|
|
1702
|
+
['br', simdLoop]]]
|
|
1703
|
+
const boundSetup = ['local.set', simdBoundName, ['i32.and', boundExpr, ['i32.const', -2]]]
|
|
1704
|
+
const wrapper = ['block', boundSetup, simdBlock, bl.blockNode]
|
|
1705
|
+
return { wrapper, newLocalDecls: [['local', simdBoundName, 'i32'], ...newLocalDecls] }
|
|
1706
|
+
}
|
|
1707
|
+
|
|
1203
1708
|
// Scalar locals that are ALWAYS computed as `(i32.add base (i32.shl ind K))`
|
|
1204
1709
|
// or aliased to such an address are "address tees", not lane data. They stay
|
|
1205
1710
|
// scalar i32 in the lifted body.
|
|
@@ -1259,13 +1764,30 @@ function liftCanon(coreV, C, ctx, info) {
|
|
|
1259
1764
|
['v128.bitselect', splatC, g, [laneNe, g, g]]]
|
|
1260
1765
|
}
|
|
1261
1766
|
|
|
1767
|
+
// --why-not-simd diagnostics. `_whyNotActive` is armed only for the duration of a
|
|
1768
|
+
// vectorizeLaneLocal call made with the flag on (cleared on exit — never leaks into
|
|
1769
|
+
// codegen, which never reads it). `_whyNotReason` captures the FIRST (deepest) lift
|
|
1770
|
+
// bail for the block currently under the recognizer chain; the walk reads it after.
|
|
1771
|
+
let _whyNotActive = false
|
|
1772
|
+
let _whyNotReason = null
|
|
1773
|
+
|
|
1774
|
+
// Mark a lift bail and record its reason. First-write-wins: the innermost failing op
|
|
1775
|
+
// sets ctx.failReason; outer frames see ctx.fail already set and return without
|
|
1776
|
+
// overwriting, so the reason names the actual blocking op, not a wrapper.
|
|
1777
|
+
const liftFail = (ctx, reason) => {
|
|
1778
|
+
ctx.fail = true
|
|
1779
|
+
if (ctx.failReason == null) ctx.failReason = reason
|
|
1780
|
+
if (_whyNotActive && _whyNotReason == null) _whyNotReason = reason
|
|
1781
|
+
return null
|
|
1782
|
+
}
|
|
1783
|
+
|
|
1262
1784
|
/** Lift a statement. Returns lifted stmt, or null to skip, or ['__seq__', ...] for multiple. */
|
|
1263
1785
|
function liftStmt(stmt, ctx) {
|
|
1264
1786
|
if (!isArr(stmt)) {
|
|
1265
1787
|
// Bare strings like "drop" — produced by stack-form WAT. We unwrap value-blocks
|
|
1266
1788
|
// separately so an isolated "drop" should not appear here, but tolerate it.
|
|
1267
1789
|
if (stmt === 'drop') return null
|
|
1268
|
-
ctx
|
|
1790
|
+
return liftFail(ctx, 'non-array statement')
|
|
1269
1791
|
}
|
|
1270
1792
|
const op = stmt[0]
|
|
1271
1793
|
|
|
@@ -1282,7 +1804,7 @@ function liftStmt(stmt, ctx) {
|
|
|
1282
1804
|
if (ctx.fail) return null
|
|
1283
1805
|
return ['local.set', laneName, v]
|
|
1284
1806
|
}
|
|
1285
|
-
ctx.
|
|
1807
|
+
return liftFail(ctx, `local.set ${name}: loop-carried or unclassified local`)
|
|
1286
1808
|
}
|
|
1287
1809
|
|
|
1288
1810
|
if (STORE_OPS[op]) {
|
|
@@ -1292,7 +1814,7 @@ function liftStmt(stmt, ctx) {
|
|
|
1292
1814
|
if (ctx.fail) return null
|
|
1293
1815
|
// Handle memarg if present (last positional after addr/val): unlikely in
|
|
1294
1816
|
// pre-watr IR for this shape; bail if more than 3 children.
|
|
1295
|
-
if (stmt.length !== 3)
|
|
1817
|
+
if (stmt.length !== 3) return liftFail(ctx, `${op} with memarg`)
|
|
1296
1818
|
return [simdStore, addr, val]
|
|
1297
1819
|
}
|
|
1298
1820
|
|
|
@@ -1319,19 +1841,86 @@ function liftStmt(stmt, ctx) {
|
|
|
1319
1841
|
return out
|
|
1320
1842
|
}
|
|
1321
1843
|
|
|
1844
|
+
// Standalone conditional store: `if (COND) { …inter; store(addr,A) } [else { …inter; store(addr,B) }]`.
|
|
1845
|
+
// Both arms end in a store to the SAME address; a missing else keeps the current value. Speculatively
|
|
1846
|
+
// lift both arms (intermediate sets become lane locals; masked lanes are discarded — lane-pure ops
|
|
1847
|
+
// are trap-free) and emit ONE store of `bitselect(A, B, mask(COND))`. Unlocks per-pixel conditional
|
|
1848
|
+
// maps like lorenz's i32x4 trail fade (`if (p & 0xffffff) px[i] = fade(p)`).
|
|
1849
|
+
if (op === 'if' && isArr(stmt[2]) && stmt[2][0] === 'then') {
|
|
1850
|
+
const armOf = (arm) => {
|
|
1851
|
+
let body = arm.slice(1)
|
|
1852
|
+
if (body.length === 1 && isArr(body[0]) && body[0][0] === 'block') { // unwrap a single block arm
|
|
1853
|
+
const b = body[0]; let i = 1
|
|
1854
|
+
if (typeof b[i] === 'string' && b[i].startsWith('$')) i++
|
|
1855
|
+
if (isArr(b[i]) && b[i][0] === 'result') i++
|
|
1856
|
+
body = b.slice(i)
|
|
1857
|
+
}
|
|
1858
|
+
const last = body[body.length - 1]
|
|
1859
|
+
if (!isArr(last) || !STORE_OPS[last[0]] || last.length !== 3) return null
|
|
1860
|
+
return { inter: body.slice(0, -1), addr: last[1], val: last[2], store: last[0] }
|
|
1861
|
+
}
|
|
1862
|
+
const thenA = armOf(stmt[2]), elseA = (isArr(stmt[3]) && stmt[3][0] === 'else') ? armOf(stmt[3]) : null
|
|
1863
|
+
if (!thenA || (isArr(stmt[3]) && !elseA)) return liftFail(ctx, 'if-store: arm is not a conditional store')
|
|
1864
|
+
if (elseA && (JSON.stringify(thenA.addr) !== JSON.stringify(elseA.addr) || thenA.store !== elseA.store)) return liftFail(ctx, 'if-store: arms store differently')
|
|
1865
|
+
// mask from COND: a lane comparison, or (i32) a truthy test `lift(cond) != 0`.
|
|
1866
|
+
let cond = stmt[1]
|
|
1867
|
+
if (isArr(cond) && cond[0] === 'i32.ne' && isI32Const(cond[2]) && cond[2][1] === 0) cond = cond[1]
|
|
1868
|
+
const cmp = isArr(cond) && cond.length === 3 ? LANE_COMPARE[ctx.laneType]?.[cond[0]] : null
|
|
1869
|
+
let mask
|
|
1870
|
+
if (cmp) { const ca = liftExprV(cond[1], ctx); if (ctx.fail) return null; const cb = liftExprV(cond[2], ctx); if (ctx.fail) return null; mask = [cmp, ca, cb] }
|
|
1871
|
+
else if (ctx.laneType === 'i32') { const lc = liftExprV(cond, ctx); if (ctx.fail) return null; mask = ['i32x4.ne', lc, ['i32x4.splat', ['i32.const', 0]]] }
|
|
1872
|
+
else return liftFail(ctx, 'if-store: non-comparison condition')
|
|
1873
|
+
const out = ['__seq__']
|
|
1874
|
+
const liftInter = (arm) => { for (const s of arm.inter) { const l = liftStmt(s, ctx); if (ctx.fail) return false; if (l != null) { if (Array.isArray(l) && l[0] === '__seq__') out.push(...l.slice(1)); else out.push(l) } } return true }
|
|
1875
|
+
if (!liftInter(thenA)) return null
|
|
1876
|
+
if (elseA && !liftInter(elseA)) return null
|
|
1877
|
+
const thenVal = liftExprV(thenA.val, ctx); if (ctx.fail) return null
|
|
1878
|
+
const elseVal = elseA ? liftExprV(elseA.val, ctx) : ['v128.load', thenA.addr] // no else ⇒ keep current value
|
|
1879
|
+
if (ctx.fail) return null
|
|
1880
|
+
const mtmp = `$__mask${ctx.freshIdRef.next++}`
|
|
1881
|
+
ctx.extraLocals.push(['local', mtmp, 'v128'])
|
|
1882
|
+
out.push(['local.set', mtmp, mask], ['v128.store', thenA.addr, ['v128.bitselect', thenVal, elseVal, ['local.get', mtmp]]])
|
|
1883
|
+
return out
|
|
1884
|
+
}
|
|
1885
|
+
|
|
1322
1886
|
// Standalone expression-as-statement (e.g. a load that gets dropped) — bail.
|
|
1323
|
-
ctx
|
|
1887
|
+
return liftFail(ctx, `standalone ${op} statement`)
|
|
1324
1888
|
}
|
|
1325
1889
|
|
|
1326
1890
|
/** Lift a value expression into v128 context. */
|
|
1327
1891
|
function liftExprV(expr, ctx) {
|
|
1328
|
-
if (!isArr(expr))
|
|
1892
|
+
if (!isArr(expr)) return liftFail(ctx, 'non-expression operand')
|
|
1329
1893
|
const op = expr[0]
|
|
1330
1894
|
const info = LANE_INFO[ctx.laneType]
|
|
1331
1895
|
|
|
1896
|
+
// Widening byte-map: a narrow UNSIGNED load feeding i32-lane arithmetic. Load
|
|
1897
|
+
// the 4 elements as a partial vector and zero-extend to i32x4. Only the
|
|
1898
|
+
// widening recognizer sets ctx.widenLoads; tryVectorize ties the lane type to
|
|
1899
|
+
// the load width and never reaches here with a narrow load under i32 lanes.
|
|
1900
|
+
if (ctx.widenLoads && ctx.laneType === 'i32') {
|
|
1901
|
+
if (op === 'i32.load8_u')
|
|
1902
|
+
return ['i32x4.extend_low_i16x8_u', ['i16x8.extend_low_i8x16_u', ['v128.load32_zero', expr[1]]]]
|
|
1903
|
+
if (op === 'i32.load16_u')
|
|
1904
|
+
return ['i32x4.extend_low_i16x8_u', ['v128.load64_zero', expr[1]]]
|
|
1905
|
+
}
|
|
1906
|
+
|
|
1907
|
+
// Widening f32→f64 load: a Float32Array read promoted to f64 (`f64.promote_f32(f32.load …)`,
|
|
1908
|
+
// e.g. schrodinger's f32 potential `V[idx]` inside an f64 stencil). Load the 2 f32 lanes
|
|
1909
|
+
// (load64_zero = 8 bytes = V[idx],V[idx+1]) and promote to f64x2 — the consecutive pair the
|
|
1910
|
+
// two f64 lanes need. Only in f64-lane context; bit-exact (same promote the scalar does).
|
|
1911
|
+
if (op === 'f64.promote_f32' && ctx.laneType === 'f64' && isArr(expr[1]) && expr[1][0] === 'f32.load') {
|
|
1912
|
+
const ld = expr[1]
|
|
1913
|
+
const addr = typeof ld[1] === 'string' && ld[1].startsWith('offset=') ? ['v128.load64_zero', ld[1], ld[2]] : ['v128.load64_zero', ld[1]]
|
|
1914
|
+
return ['f64x2.promote_low_f32x4', addr]
|
|
1915
|
+
}
|
|
1916
|
+
|
|
1332
1917
|
// Loads → v128.load (preserving address, including any local.tee).
|
|
1333
1918
|
if (LOAD_OPS[op]) {
|
|
1334
|
-
if (LOAD_OPS[op] !== ctx.laneType)
|
|
1919
|
+
if (LOAD_OPS[op] !== ctx.laneType) return liftFail(ctx, `${op}: load type ≠ lane type ${ctx.laneType}`)
|
|
1920
|
+
// memarg form `(T.load offset=N addr)` — the stencil neighbour `a[i+1]` jz folds
|
|
1921
|
+
// onto `a[i]`'s address tee. `v128.load offset=N` reads the N-byte-shifted vector,
|
|
1922
|
+
// i.e. the (a[i+1], a[i+2]) pair — exactly the δ-shifted lane data. Preserve it.
|
|
1923
|
+
if (typeof expr[1] === 'string' && expr[1].startsWith('offset=')) return ['v128.load', expr[1], expr[2]]
|
|
1335
1924
|
return ['v128.load', expr[1]]
|
|
1336
1925
|
}
|
|
1337
1926
|
|
|
@@ -1343,6 +1932,14 @@ function liftExprV(expr, ctx) {
|
|
|
1343
1932
|
// local.get
|
|
1344
1933
|
if (op === 'local.get' && typeof expr[1] === 'string') {
|
|
1345
1934
|
const name = expr[1]
|
|
1935
|
+
// Induction variable used AS DATA (ramp-map) → splat to a ramp vector
|
|
1936
|
+
// [i, i+1, … i+LANES-1]. Only set by tryRampMap (i32 lanes); other
|
|
1937
|
+
// recognizers leave ctx.rampVar undefined, so the IV stays address-only.
|
|
1938
|
+
if (name === ctx.rampVar) {
|
|
1939
|
+
// The ramp [i, i+1, i+2, i+3] is materialized once per iteration into
|
|
1940
|
+
// ctx.rampTemp (set at the top of the lifted body); every use reads it.
|
|
1941
|
+
return ['local.get', ctx.rampTemp]
|
|
1942
|
+
}
|
|
1346
1943
|
const kind = ctx.localKind.get(name)
|
|
1347
1944
|
if (kind === 'lane') {
|
|
1348
1945
|
const { laneName } = getOrAllocLanedLocal(name, ctx)
|
|
@@ -1352,9 +1949,9 @@ function liftExprV(expr, ctx) {
|
|
|
1352
1949
|
return [info.splat, ['local.get', name]]
|
|
1353
1950
|
}
|
|
1354
1951
|
if (kind === 'addr' || name === ctx.incVar) {
|
|
1355
|
-
ctx
|
|
1952
|
+
return liftFail(ctx, `${name}: address/induction var used as lane data`)
|
|
1356
1953
|
}
|
|
1357
|
-
ctx
|
|
1954
|
+
return liftFail(ctx, `${name}: unclassified local in value position`)
|
|
1358
1955
|
}
|
|
1359
1956
|
|
|
1360
1957
|
// Loop-invariant global (e.g. a hoistConstantPool'd const, or any global the
|
|
@@ -1364,6 +1961,15 @@ function liftExprV(expr, ctx) {
|
|
|
1364
1961
|
return [info.splat, expr]
|
|
1365
1962
|
}
|
|
1366
1963
|
|
|
1964
|
+
// `f64(invariant i32)` — e.g. `x[i] / N` with N an i32 global/invariant: the convert is a
|
|
1965
|
+
// loop-invariant scalar, so compute once and splat (== scalar-then-splat, bit-exact). Unblocks
|
|
1966
|
+
// pure-f64 maps that scale/divide by an integer count (rfft cepstrum `cep[i] = x[i] / N`).
|
|
1967
|
+
if ((op === 'f64.convert_i32_s' || op === 'f64.convert_i32_u') && expr.length === 2) {
|
|
1968
|
+
const inner = expr[1]
|
|
1969
|
+
const inv = isArr(inner) && (inner[0] === 'global.get' || (inner[0] === 'local.get' && ctx.localKind.get(inner[1]) === 'invariant'))
|
|
1970
|
+
if (inv) return [info.splat, expr]
|
|
1971
|
+
}
|
|
1972
|
+
|
|
1367
1973
|
// NaN-canonicalization wrapper (float lanes only; integer lanes never carry
|
|
1368
1974
|
// it). Both the flattened `select` form and the un-flattened `block` form
|
|
1369
1975
|
// lift to a per-lane v128.bitselect — canonical value in NaN lanes, X
|
|
@@ -1371,13 +1977,13 @@ function liftExprV(expr, ctx) {
|
|
|
1371
1977
|
if (ctx.laneType === 'f64' || ctx.laneType === 'f32') {
|
|
1372
1978
|
if (op === 'select') {
|
|
1373
1979
|
const m = matchCanonSelect(expr, ctx.laneType)
|
|
1374
|
-
if (!m)
|
|
1980
|
+
if (!m) return liftFail(ctx, 'non-canonical select (not a NaN-canon idiom)')
|
|
1375
1981
|
const coreV = liftExprV(m.val, ctx)
|
|
1376
1982
|
return ctx.fail ? null : liftCanon(coreV, m.C, ctx, info)
|
|
1377
1983
|
}
|
|
1378
1984
|
if (op === 'block') {
|
|
1379
1985
|
const m = matchCanonBlock(expr, ctx.laneType)
|
|
1380
|
-
if (!m)
|
|
1986
|
+
if (!m) return liftFail(ctx, 'non-canonical value-block')
|
|
1381
1987
|
const coreV = liftExprV(m.core, ctx)
|
|
1382
1988
|
return ctx.fail ? null : liftCanon(coreV, m.C, ctx, info)
|
|
1383
1989
|
}
|
|
@@ -1392,14 +1998,14 @@ function liftExprV(expr, ctx) {
|
|
|
1392
1998
|
// evaluates X,Y before its 3rd operand, but any address `local.tee` lives in
|
|
1393
1999
|
// COND and must run before the branches read it (matching scalar order).
|
|
1394
2000
|
if (op === 'if') {
|
|
1395
|
-
if (!isArr(expr[1]) || expr[1][0] !== 'result' || expr[1][1] !== ctx.laneType)
|
|
2001
|
+
if (!isArr(expr[1]) || expr[1][0] !== 'result' || expr[1][1] !== ctx.laneType) return liftFail(ctx, 'conditional without lane-typed result')
|
|
1396
2002
|
const thenN = expr[3], elseN = expr[4]
|
|
1397
|
-
if (!isArr(thenN) || thenN[0] !== 'then' || thenN.length !== 2)
|
|
1398
|
-
if (!isArr(elseN) || elseN[0] !== 'else' || elseN.length !== 2)
|
|
2003
|
+
if (!isArr(thenN) || thenN[0] !== 'then' || thenN.length !== 2) return liftFail(ctx, 'malformed conditional then-branch')
|
|
2004
|
+
if (!isArr(elseN) || elseN[0] !== 'else' || elseN.length !== 2) return liftFail(ctx, 'malformed conditional else-branch')
|
|
1399
2005
|
let cond = expr[2]
|
|
1400
2006
|
if (isArr(cond) && cond[0] === 'i32.ne' && isI32Const(cond[2]) && cond[2][1] === 0) cond = cond[1] // strip `!= 0`
|
|
1401
2007
|
const cmpSimd = isArr(cond) && cond.length === 3 ? LANE_COMPARE[ctx.laneType]?.[cond[0]] : null
|
|
1402
|
-
if (!cmpSimd)
|
|
2008
|
+
if (!cmpSimd) return liftFail(ctx, `${isArr(cond) ? cond[0] : 'condition'}: not a lane-vectorizable comparison`)
|
|
1403
2009
|
const ca = liftExprV(cond[1], ctx); if (ctx.fail) return null
|
|
1404
2010
|
const cb = liftExprV(cond[2], ctx); if (ctx.fail) return null
|
|
1405
2011
|
const x = liftExprV(thenN[1], ctx); if (ctx.fail) return null
|
|
@@ -1421,7 +2027,7 @@ function liftExprV(expr, ctx) {
|
|
|
1421
2027
|
// Second operand stays scalar i32 — must be const or invariant local.
|
|
1422
2028
|
const b = expr[2]
|
|
1423
2029
|
if (!isI32Const(b) && !(isArr(b) && b[0] === 'local.get' && ctx.localKind.get(b[1]) === 'invariant')) {
|
|
1424
|
-
ctx
|
|
2030
|
+
return liftFail(ctx, `${op}: shift amount not a constant or loop-invariant`)
|
|
1425
2031
|
}
|
|
1426
2032
|
return [entry.simd, a, b]
|
|
1427
2033
|
}
|
|
@@ -1433,7 +2039,7 @@ function liftExprV(expr, ctx) {
|
|
|
1433
2039
|
return [entry.simd, a, b]
|
|
1434
2040
|
}
|
|
1435
2041
|
|
|
1436
|
-
ctx
|
|
2042
|
+
return liftFail(ctx, `${op}: no lane-pure SIMD mapping for ${ctx.laneType}`)
|
|
1437
2043
|
}
|
|
1438
2044
|
|
|
1439
2045
|
// ---- Induction-variable strength reduction --------------------------------
|
|
@@ -1467,25 +2073,12 @@ function matchAffineAddr(node, ind) {
|
|
|
1467
2073
|
* `br_if` to the *block* label, i.e. an early break, is fine: the loop is exiting). Runs
|
|
1468
2074
|
* only where the vectorizer runs (speed levels), so it never grows the size-tuned build.
|
|
1469
2075
|
*/
|
|
1470
|
-
function tryStrengthReduceIV(
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
if (isArr(c) && c[0] === 'loop') { if (loopNode) return null; loopNode = c }
|
|
1477
|
-
else if (isArr(c)) return null
|
|
1478
|
-
}
|
|
1479
|
-
if (!loopNode || !blockLabel) return null
|
|
1480
|
-
const loopLabel = typeof loopNode[1] === 'string' && loopNode[1].startsWith('$') ? loopNode[1] : null
|
|
1481
|
-
if (!loopLabel) return null
|
|
1482
|
-
const endIdx = loopNode.length - 1
|
|
1483
|
-
if (!(isArr(loopNode[endIdx]) && loopNode[endIdx][0] === 'br' && loopNode[endIdx][1] === loopLabel)) return null
|
|
1484
|
-
const incIdx = endIdx - 1
|
|
1485
|
-
const incVar = matchInc1(loopNode[incIdx])
|
|
1486
|
-
if (!incVar) return null
|
|
1487
|
-
const exitInfo = matchExitBrIf(loopNode[2], blockLabel)
|
|
1488
|
-
if (!exitInfo || exitInfo.ind !== incVar) return null
|
|
2076
|
+
function tryStrengthReduceIV(bl, fnLocals, freshIdRef) {
|
|
2077
|
+
// Bound-shape-agnostic: strength-reduces address arithmetic, not the loop bound,
|
|
2078
|
+
// so it accepts any scaffold (no local-or-const bound check). No preamble (was
|
|
2079
|
+
// allowPreamble:false).
|
|
2080
|
+
if (!bl || bl.preamble.length) return null
|
|
2081
|
+
const { loopNode, loopLabel, incIdx, incVar } = bl
|
|
1489
2082
|
|
|
1490
2083
|
// Scan the body (stmts between the exit br_if and the bottom increment): collect
|
|
1491
2084
|
// affine-address sites, track every written local, and bail on a loop-label branch.
|
|
@@ -1529,7 +2122,7 @@ function tryStrengthReduceIV(blockNode, fnLocals, freshIdRef) {
|
|
|
1529
2122
|
for (const s of g.sites) s.parent[s.idx] = ['local.get', p]
|
|
1530
2123
|
}
|
|
1531
2124
|
loopNode.splice(incIdx + 1, 0, ...bumps) // after the induction increment, before the br
|
|
1532
|
-
return { wrapper: ['block', ...preInits, blockNode], newLocalDecls }
|
|
2125
|
+
return { wrapper: ['block', ...preInits, bl.blockNode], newLocalDecls }
|
|
1533
2126
|
}
|
|
1534
2127
|
|
|
1535
2128
|
// ---- memory.copy / memory.fill loop idioms ---------------------------------
|
|
@@ -1574,27 +2167,12 @@ const MEMOP_STORES = {
|
|
|
1574
2167
|
* - In-bounds for the same reason the lane vectorizer's wide loads are: the
|
|
1575
2168
|
* moved window is exactly the byte range the scalar iterations touch.
|
|
1576
2169
|
*/
|
|
1577
|
-
function tryMemCopyFill(
|
|
1578
|
-
if (!
|
|
1579
|
-
|
|
1580
|
-
for (let i = 1; i < blockNode.length; i++) {
|
|
1581
|
-
const c = blockNode[i]
|
|
1582
|
-
if (typeof c === 'string' && c.startsWith('$') && blockLabel == null && i === 1) { blockLabel = c; continue }
|
|
1583
|
-
if (isArr(c) && c[0] === 'loop') { if (loopNode) return null; loopNode = c }
|
|
1584
|
-
else if (isArr(c)) return null
|
|
1585
|
-
}
|
|
1586
|
-
if (!loopNode || !blockLabel) return null
|
|
1587
|
-
const loopLabel = typeof loopNode[1] === 'string' && loopNode[1].startsWith('$') ? loopNode[1] : null
|
|
1588
|
-
if (!loopLabel) return null
|
|
2170
|
+
function tryMemCopyFill(bl, fnLocals, freshIdRef) {
|
|
2171
|
+
if (!bl || bl.preamble.length) return null // no-preamble policy (was allowPreamble:false)
|
|
2172
|
+
const { loopNode, incVar, bound } = bl
|
|
1589
2173
|
// Shape: [loop, $l, boundExit, (set,)? store, inc, br] — 1- or 2-statement body.
|
|
1590
2174
|
if (loopNode.length !== 6 && loopNode.length !== 7) return null
|
|
1591
|
-
const
|
|
1592
|
-
if (!(isArr(loopNode[endIdx]) && loopNode[endIdx][0] === 'br' && loopNode[endIdx][1] === loopLabel)) return null
|
|
1593
|
-
const incVar = matchInc1(loopNode[endIdx - 1])
|
|
1594
|
-
if (!incVar) return null
|
|
1595
|
-
const exitInfo = matchExitBrIf(loopNode[2], blockLabel)
|
|
1596
|
-
if (!exitInfo || exitInfo.ind !== incVar) return null
|
|
1597
|
-
const bound = exitInfo.bound
|
|
2175
|
+
// Bound: const or an invariant local that is not the IV itself.
|
|
1598
2176
|
if (!(isI32Const(bound) || (isArr(bound) && bound[0] === 'local.get' && typeof bound[1] === 'string' && bound[1] !== incVar))) return null
|
|
1599
2177
|
|
|
1600
2178
|
// Body shapes (emit produces the two-statement form with a temp + shared
|
|
@@ -1679,7 +2257,7 @@ function tryMemCopyFill(blockNode, fnLocals, freshIdRef) {
|
|
|
1679
2257
|
['then',
|
|
1680
2258
|
['memory.copy', ['local.get', dstA], ['local.get', srcA], ['local.get', lenB]],
|
|
1681
2259
|
...finish],
|
|
1682
|
-
['else', blockNode]]]
|
|
2260
|
+
['else', bl.blockNode]]]
|
|
1683
2261
|
}
|
|
1684
2262
|
|
|
1685
2263
|
const wrapper = ['block',
|
|
@@ -1737,25 +2315,13 @@ function matchByteCompare(node, ind) {
|
|
|
1737
2315
|
* reads too. (charCodeAt over a jz string is out of scope — it lowers to per-char
|
|
1738
2316
|
* bounds/SSO/heap/decode branches, not a flat byte load.)
|
|
1739
2317
|
*/
|
|
1740
|
-
function tryByteScan(
|
|
1741
|
-
if (!
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
if (isArr(c) && c[0] === 'loop') { if (loopNode) return null; loopNode = c }
|
|
1747
|
-
else if (isArr(c)) return null
|
|
1748
|
-
}
|
|
1749
|
-
if (!loopNode || !blockLabel) return null
|
|
1750
|
-
const loopLabel = typeof loopNode[1] === 'string' && loopNode[1].startsWith('$') ? loopNode[1] : null
|
|
1751
|
-
if (!loopLabel) return null
|
|
1752
|
-
// Exact shape: [loop, $l, boundExit, matchExit, inc, br] — nothing else.
|
|
2318
|
+
function tryByteScan(bl, fnLocals, freshIdRef) {
|
|
2319
|
+
if (!bl || bl.preamble.length) return null
|
|
2320
|
+
const { blockLabel, loopNode, incVar, exitInfo } = bl
|
|
2321
|
+
// Exact shape: [loop, $l, boundExit, matchExit, inc, br] — nothing else. The
|
|
2322
|
+
// scaffold already verified boundExit/inc/back-branch; only the extra match-exit
|
|
2323
|
+
// br_if at loopNode[3] and the strict length remain.
|
|
1753
2324
|
if (loopNode.length !== 6) return null
|
|
1754
|
-
if (!(isArr(loopNode[5]) && loopNode[5][0] === 'br' && loopNode[5][1] === loopLabel)) return null
|
|
1755
|
-
const incVar = matchInc1(loopNode[4])
|
|
1756
|
-
if (!incVar) return null
|
|
1757
|
-
const exitInfo = matchExitBrIf(loopNode[2], blockLabel) // (br_if $b (eqz (i < bound)))
|
|
1758
|
-
if (!exitInfo || exitInfo.ind !== incVar) return null
|
|
1759
2325
|
const matchExit = loopNode[3]
|
|
1760
2326
|
if (!(isArr(matchExit) && matchExit[0] === 'br_if' && matchExit[1] === blockLabel && matchExit.length === 3)) return null
|
|
1761
2327
|
const bc = matchByteCompare(matchExit[2], incVar)
|
|
@@ -1819,32 +2385,2216 @@ function tryByteScan(blockNode, fnLocals, freshIdRef) {
|
|
|
1819
2385
|
|
|
1820
2386
|
const cloneNode = (n) => Array.isArray(n) ? n.map(cloneNode) : n
|
|
1821
2387
|
|
|
1822
|
-
// ----
|
|
2388
|
+
// ---- Channel-reduction recognizer (RGBA box-filter accumulation) -----------
|
|
2389
|
+
//
|
|
2390
|
+
// The image-convolution hot shape: 4 adjacent-byte accumulators summed over a
|
|
2391
|
+
// window, then divided + stored — `for k: sr+=src[p]; sg+=src[p+1]; …` (blur,
|
|
2392
|
+
// box filters, separable convolutions). The 4 channels are 4 i32x4 lanes, the
|
|
2393
|
+
// 4-byte load is one widening v128.load32_zero. We vectorize ONLY the inner
|
|
2394
|
+
// accumulation (integer add → associative/exact → bit-identical) and extract the
|
|
2395
|
+
// lane sums back to the scalars; the edge-clamp address math and the per-pixel
|
|
2396
|
+
// divide+store stay scalar, untouched. So no float-reduction reordering, no
|
|
2397
|
+
// lane-juggled divide — just the dense inner loop lifted, safely.
|
|
2398
|
+
//
|
|
2399
|
+
// Operates on the OUTER pixel-loop block: its body is [exit, 4×(acc=0), …setup,
|
|
2400
|
+
// (block (loop INNER)), …uses-of-acc, inc, br]. INNER's body accumulates the 4
|
|
2401
|
+
// channels off a shared base address.
|
|
1823
2402
|
|
|
1824
|
-
|
|
1825
|
-
|
|
1826
|
-
|
|
1827
|
-
|
|
1828
|
-
|
|
1829
|
-
|
|
1830
|
-
const
|
|
1831
|
-
if (
|
|
2403
|
+
// Match `(local.set $ACC (i32.add (local.get $ACC) (i32.load8_u ADDR)))` where
|
|
2404
|
+
// ADDR is `(local.get $base)` or `(local.tee $base EXPR)` at byte offset `off`.
|
|
2405
|
+
// Returns { acc, base, off, teeExpr? } or null.
|
|
2406
|
+
function matchChannelAccum(stmt, off) {
|
|
2407
|
+
if (!isArr(stmt) || stmt[0] !== 'local.set' || stmt.length !== 3) return null
|
|
2408
|
+
const acc = stmt[1]
|
|
2409
|
+
const add = stmt[2]
|
|
2410
|
+
if (!isArr(add) || add[0] !== 'i32.add' || add.length !== 3) return null
|
|
2411
|
+
if (!isLocalGet(add[1], acc)) return null
|
|
2412
|
+
const load = add[2]
|
|
2413
|
+
if (!isArr(load) || load[0] !== 'i32.load8_u') return null
|
|
2414
|
+
// memarg offset: bare load → off 0; `i32.load8_u offset=N addr` → off N.
|
|
2415
|
+
let addr = load[1], loadOff = 0
|
|
2416
|
+
if (typeof load[1] === 'string' && /^offset=/.test(load[1])) { loadOff = +load[1].slice(7); addr = load[2] }
|
|
2417
|
+
if (loadOff !== off) return null
|
|
2418
|
+
if (isArr(addr) && addr[0] === 'local.tee' && addr.length === 3) return { acc, base: addr[1], off, teeExpr: addr[2] }
|
|
2419
|
+
if (isLocalGet(addr)) return { acc, base: addr[1], off }
|
|
2420
|
+
return null
|
|
2421
|
+
}
|
|
1832
2422
|
|
|
1833
|
-
|
|
1834
|
-
|
|
1835
|
-
|
|
1836
|
-
|
|
1837
|
-
|
|
1838
|
-
|
|
1839
|
-
|
|
1840
|
-
|
|
2423
|
+
// Recognize, inside the inner loop body, four consecutive channel accumulations
|
|
2424
|
+
// `acc0+=base[0]; acc1+=base[1]; acc2+=base[2]; acc3+=base[3]` over ONE shared
|
|
2425
|
+
// base. Returns { accs:[a0..a3], baseLocal, teeExpr, idx } (idx = position of the
|
|
2426
|
+
// first accum stmt in `body`) or null.
|
|
2427
|
+
function matchChannelGroup(body) {
|
|
2428
|
+
for (let i = 0; i + 3 < body.length; i++) {
|
|
2429
|
+
const m0 = matchChannelAccum(body[i], 0)
|
|
2430
|
+
if (!m0 || m0.teeExpr == null) continue // first load carries the address tee
|
|
2431
|
+
const ms = [m0]
|
|
2432
|
+
let ok = true
|
|
2433
|
+
for (let c = 1; c < 4; c++) {
|
|
2434
|
+
const m = matchChannelAccum(body[i + c], c)
|
|
2435
|
+
if (!m || m.base !== m0.base || m.teeExpr != null) { ok = false; break }
|
|
2436
|
+
ms.push(m)
|
|
1841
2437
|
}
|
|
2438
|
+
if (!ok) continue
|
|
2439
|
+
const accs = ms.map(m => m.acc)
|
|
2440
|
+
if (new Set(accs).size !== 4) continue // four distinct accumulators
|
|
2441
|
+
return { accs, baseLocal: m0.base, teeExpr: m0.teeExpr, idx: i }
|
|
1842
2442
|
}
|
|
2443
|
+
return null
|
|
2444
|
+
}
|
|
1843
2445
|
|
|
1844
|
-
|
|
1845
|
-
|
|
1846
|
-
|
|
1847
|
-
|
|
2446
|
+
// ---- Byte-map recognizer (ramp + widening loads) ---------------------------
|
|
2447
|
+
//
|
|
2448
|
+
// Vectorize `for (i = 0; i < N; i++) out[i] = NARROW(f_i32(…))` where the i32
|
|
2449
|
+
// value is built from the induction variable used AS DATA (an i32x4 RAMP
|
|
2450
|
+
// `[i, i+1, i+2, i+3]`) and/or WIDENED narrow loads (`u8[i]` zero-extended to
|
|
2451
|
+
// i32x4). tryVectorize can't express either: it derives the lane type from an
|
|
2452
|
+
// input load and ties the compute width to it, so a byte LUT-free map whose
|
|
2453
|
+
// arithmetic overflows a byte (mul, shifts) — or that has no load at all —
|
|
2454
|
+
// falls through to here, where everything lifts to i32x4 and narrow-stores.
|
|
2455
|
+
// • pure ramp (no loads, i8 store) → 16-wide pack (bytebeat)
|
|
2456
|
+
// • widening u8 loads → 4-wide (alpha blend, brightness/color/threshold)
|
|
2457
|
+
//
|
|
2458
|
+
// Matches the post-strength-reduction shape (the IV strength-reducer runs
|
|
2459
|
+
// before this pass): the loop carries the logical IV (`i`, in the exit test,
|
|
2460
|
+
// += 1) plus one or more strided output pointers (`p += C`). Each increment is
|
|
2461
|
+
// scaled by LANES; the store narrows i32x4 back to the element width.
|
|
2462
|
+
//
|
|
2463
|
+
// Narrowing is truncation-exact for ANY i32 value (matching scalar store8/16):
|
|
2464
|
+
// `i8x16.shuffle` selects the low byte of each lane — never saturates — so no
|
|
2465
|
+
// value-range assumption is needed.
|
|
2466
|
+
function tryRampMap(blockNode, fnLocals, freshIdRef) {
|
|
2467
|
+
if (!isArr(blockNode) || blockNode[0] !== 'block') return null
|
|
2468
|
+
// Envelope: (block $brk (loop $L …)) — identical to tryVectorize.
|
|
2469
|
+
let blockLabel = null, loopNode = null
|
|
2470
|
+
for (let i = 1; i < blockNode.length; i++) {
|
|
2471
|
+
const c = blockNode[i]
|
|
2472
|
+
if (typeof c === 'string' && c.startsWith('$') && blockLabel == null && i === 1) { blockLabel = c; continue }
|
|
2473
|
+
if (isArr(c) && c[0] === 'loop') { if (loopNode) return null; loopNode = c }
|
|
2474
|
+
else if (isArr(c)) return null
|
|
2475
|
+
}
|
|
2476
|
+
if (!loopNode || !blockLabel) return null
|
|
2477
|
+
const loopLabel = typeof loopNode[1] === 'string' && loopNode[1].startsWith('$') ? loopNode[1] : null
|
|
2478
|
+
if (!loopLabel) return null
|
|
2479
|
+
|
|
2480
|
+
// End = (br $L); preceded by a run of trailing `x += C` increments.
|
|
2481
|
+
const endIdx = loopNode.length - 1
|
|
2482
|
+
if (!(isArr(loopNode[endIdx]) && loopNode[endIdx][0] === 'br' && loopNode[endIdx][1] === loopLabel)) return null
|
|
2483
|
+
const increments = []
|
|
2484
|
+
let bodyEnd = endIdx - 1
|
|
2485
|
+
while (bodyEnd >= 2) {
|
|
2486
|
+
const inc = matchIncN(loopNode[bodyEnd])
|
|
2487
|
+
if (!inc) break
|
|
2488
|
+
increments.unshift(inc)
|
|
2489
|
+
bodyEnd--
|
|
2490
|
+
}
|
|
2491
|
+
if (!increments.length) return null
|
|
2492
|
+
|
|
2493
|
+
// First stmt = exit guard → logical IV + bound. IV must advance by exactly 1.
|
|
2494
|
+
const exitInfo = matchExitBrIf(loopNode[2], blockLabel)
|
|
2495
|
+
if (!exitInfo) return null
|
|
2496
|
+
const ivName = exitInfo.ind
|
|
2497
|
+
const ivInc = increments.find(x => x.name === ivName)
|
|
2498
|
+
if (!ivInc || ivInc.c !== 1) return null
|
|
2499
|
+
let bound = exitInfo.bound, boundLocal = null
|
|
2500
|
+
if (isArr(bound) && bound[0] === 'local.get' && typeof bound[1] === 'string') boundLocal = bound[1]
|
|
2501
|
+
else if (!isI32Const(bound)) return null
|
|
2502
|
+
|
|
2503
|
+
const body = []
|
|
2504
|
+
for (let i = 3; i <= bodyEnd; i++) body.push(loopNode[i])
|
|
2505
|
+
if (!body.length) return null
|
|
2506
|
+
if (body.some(hasGlobalSet)) return null
|
|
2507
|
+
|
|
2508
|
+
// Find exactly one store. Its address is the inline lane address
|
|
2509
|
+
// `base + (i << K)` — the IV strength-reducer runs AFTER this pass, so the
|
|
2510
|
+
// pointer is still expressed in terms of the IV. We keep the address verbatim
|
|
2511
|
+
// (scalar i32) and advance the IV by LANES, so `base + (i<<K)` lands on the
|
|
2512
|
+
// next group's first element each SIMD step — for any element width.
|
|
2513
|
+
let storeStmt = null, storeIdx = -1
|
|
2514
|
+
for (let i = 0; i < body.length; i++) {
|
|
2515
|
+
const s = body[i]
|
|
2516
|
+
if (isArr(s) && STORE_OPS[s[0]]) {
|
|
2517
|
+
if (storeStmt) return null
|
|
2518
|
+
storeStmt = s; storeIdx = i
|
|
2519
|
+
}
|
|
2520
|
+
}
|
|
2521
|
+
if (!storeStmt) return null
|
|
2522
|
+
const storeOp = storeStmt[0]
|
|
2523
|
+
if (storeStmt.length !== 3) return null
|
|
2524
|
+
const elemLog2 = { 'i32.store8': 0, 'i32.store': 2 }[storeOp]
|
|
2525
|
+
if (elemLog2 === undefined) return null
|
|
2526
|
+
if (increments.some(x => x.name !== ivName)) return null
|
|
2527
|
+
|
|
2528
|
+
// CSE'd lane offsets: a local written ONLY as `i << K` (or bare `i`) is the
|
|
2529
|
+
// shared offset the IV stage threads across base pointers (src[i], dst[i],
|
|
2530
|
+
// out[i] all reuse one `(local.tee $p (local.get i))`). Resolve them so the
|
|
2531
|
+
// load/store address matchers accept the `(local.get $p)` reuses.
|
|
2532
|
+
const offsetTees = new Map()
|
|
2533
|
+
const allNames = new Set()
|
|
2534
|
+
const gatherNames = (n) => { if (!isArr(n)) return; if ((n[0] === 'local.set' || n[0] === 'local.tee') && typeof n[1] === 'string') allNames.add(n[1]); for (let i = 1; i < n.length; i++) gatherNames(n[i]) }
|
|
2535
|
+
for (const s of body) gatherNames(s)
|
|
2536
|
+
for (const name of allNames) { const k = _offsetLocalStride(body, name, ivName); if (k != null) offsetTees.set(name, k) }
|
|
2537
|
+
|
|
2538
|
+
const storeAddr = storeStmt[1]
|
|
2539
|
+
const addrM = matchLaneAddr(storeAddr, ivName, new Map(), offsetTees)
|
|
2540
|
+
if (!addrM || addrM.strideLog2 !== elemLog2) return null
|
|
2541
|
+
|
|
2542
|
+
// Memory loads turn this into a widening byte-map: out[i] = narrow(f(widen(a[i])…)).
|
|
2543
|
+
// Only the u8 shape is supported — every load must be a narrow UNSIGNED u8 load
|
|
2544
|
+
// of the same 4 elements the store writes (base + i; the IV strength-reducer
|
|
2545
|
+
// runs later). Full-width i32 maps are tryVectorize's job; mixed widths bail.
|
|
2546
|
+
let hasLoads = false, loadsOk = true
|
|
2547
|
+
const checkLoad = (n) => {
|
|
2548
|
+
if (!isArr(n)) return
|
|
2549
|
+
if (LOAD_OPS[n[0]]) {
|
|
2550
|
+
hasLoads = true
|
|
2551
|
+
if (storeOp !== 'i32.store8' || n[0] !== 'i32.load8_u') { loadsOk = false; return }
|
|
2552
|
+
const m = matchLaneAddr(n[1], ivName, new Map(), offsetTees)
|
|
2553
|
+
if (!m || m.strideLog2 !== 0) loadsOk = false
|
|
2554
|
+
return // address validated; the IV-strided subtree is not data
|
|
2555
|
+
}
|
|
2556
|
+
for (let i = 1; i < n.length; i++) checkLoad(n[i])
|
|
2557
|
+
}
|
|
2558
|
+
for (const s of body) checkLoad(s)
|
|
2559
|
+
if (!loadsOk) return null
|
|
2560
|
+
|
|
2561
|
+
// Every other body stmt must be `(local.set $lane EXPR)` — straight-line lane
|
|
2562
|
+
// locals feeding the store. Classify locals for the lift.
|
|
2563
|
+
const writes = new Set()
|
|
2564
|
+
for (const s of body) collectWrites(s, writes)
|
|
2565
|
+
if (boundLocal && writes.has(boundLocal)) return null
|
|
2566
|
+
const referenced = new Set()
|
|
2567
|
+
const collectRefs = (n) => {
|
|
2568
|
+
if (!isArr(n)) return
|
|
2569
|
+
if ((n[0] === 'local.get' || n[0] === 'local.set' || n[0] === 'local.tee') && typeof n[1] === 'string') referenced.add(n[1])
|
|
2570
|
+
for (let i = 1; i < n.length; i++) collectRefs(n[i])
|
|
2571
|
+
}
|
|
2572
|
+
for (const s of body) collectRefs(s)
|
|
2573
|
+
|
|
2574
|
+
const localKind = new Map()
|
|
2575
|
+
for (const name of referenced) {
|
|
2576
|
+
if (name === ivName) continue
|
|
2577
|
+
if (writes.has(name)) {
|
|
2578
|
+
let firstKind = null
|
|
2579
|
+
for (const s of body) { const kAcc = firstAccess(s, name); if (kAcc) { firstKind = kAcc; break } }
|
|
2580
|
+
if (firstKind === 'read') return null // loop-carried → reduction/stencil, not a pure map
|
|
2581
|
+
localKind.set(name, 'lane')
|
|
2582
|
+
} else {
|
|
2583
|
+
localKind.set(name, 'invariant')
|
|
2584
|
+
}
|
|
2585
|
+
}
|
|
2586
|
+
|
|
2587
|
+
// Lift. lane type is always i32 (the ramp and all narrow stores compute in i32).
|
|
2588
|
+
const newLanedLocals = new Map()
|
|
2589
|
+
const extraLocals = []
|
|
2590
|
+
const freshV128 = (tag) => { const n = `$__${tag}${freshIdRef.next++}`; extraLocals.push(['local', n, 'v128']); return n }
|
|
2591
|
+
const ctx = { laneType: 'i32', incVar: ivName, rampVar: ivName, rampTemp: null, widenLoads: true, localKind, newLanedLocals, extraLocals, freshIdRef, fail: false, failReason: null }
|
|
2592
|
+
|
|
2593
|
+
// A byte store fed by one value expression (inline, or via a single lane-local
|
|
2594
|
+
// temp `tw = EXPR; store(addr, tw)`) carries no loop-carried state, so we can
|
|
2595
|
+
// run the lane group 4× (16 samples) per iteration off four offset ramps and
|
|
2596
|
+
// pack the low bytes into ONE i8x16 v128.store — amortizing store + loop
|
|
2597
|
+
// overhead the way clang/zig's 16-wide NEON does. wideValueExpr is the
|
|
2598
|
+
// expression to lift per offset; null (i32 stores, multi-stmt bodies, or
|
|
2599
|
+
// widening loads — whose addresses would need per-offset advancing) → 4-wide.
|
|
2600
|
+
// The single byte-value expression feeding the store — inline, or via one lane-local
|
|
2601
|
+
// temp `tw = EXPR; store(addr, tw)`. Shared by both 16-wide paths below.
|
|
2602
|
+
const byteValueExpr = (() => {
|
|
2603
|
+
if (storeOp !== 'i32.store8') return null
|
|
2604
|
+
if (body.length === 1 && storeIdx === 0) return storeStmt[2]
|
|
2605
|
+
if (body.length === 2 && storeIdx === 1) {
|
|
2606
|
+
const set = body[0], sv = storeStmt[2]
|
|
2607
|
+
if (isArr(set) && set[0] === 'local.set' && set.length === 3 &&
|
|
2608
|
+
isLocalGet(sv, set[1]) && localKind.get(set[1]) === 'lane') return set[2]
|
|
2609
|
+
}
|
|
2610
|
+
return null
|
|
2611
|
+
})()
|
|
2612
|
+
// With u8 loads, the byte map can go 16-wide in i16x8 (the alpha-blend shape: out[i] =
|
|
2613
|
+
// (src[i]*A + dst[i]*B + bias) >> s) — load 16, extend_low/high, the affine arithmetic
|
|
2614
|
+
// in i16x8, narrow_u, store 16 — exactly clang's NEON. Sound ONLY when every
|
|
2615
|
+
// intermediate provably fits u16 ([0,65535], so i16x8 mod-2^16 never wraps and shr_u ==
|
|
2616
|
+
// the scalar shr_s on a non-negative value) and the result fits a byte ([0,255], so
|
|
2617
|
+
// narrow_u never saturates). `byteValueRange` returns [min,max] or null when any node
|
|
2618
|
+
// is unanalyzable, can go negative, or exceeds u16. An invariant local of unknown
|
|
2619
|
+
// magnitude (local.get) → null → falls back to the bit-exact 4-wide path.
|
|
2620
|
+
const byteValueRange = (e) => {
|
|
2621
|
+
if (!isArr(e)) return null
|
|
2622
|
+
const op = e[0]
|
|
2623
|
+
let r
|
|
2624
|
+
if (op === 'i32.const') { const v = constNum(e); r = [v, v] }
|
|
2625
|
+
else if (op === 'i32.load8_u') r = [0, 255]
|
|
2626
|
+
else if (op === 'i32.add') { const a = byteValueRange(e[1]), b = byteValueRange(e[2]); if (!a || !b) return null; r = [a[0] + b[0], a[1] + b[1]] }
|
|
2627
|
+
else if (op === 'i32.sub') { const a = byteValueRange(e[1]), b = byteValueRange(e[2]); if (!a || !b) return null; r = [a[0] - b[1], a[1] - b[0]] }
|
|
2628
|
+
else if (op === 'i32.mul') { const a = byteValueRange(e[1]), b = byteValueRange(e[2]); if (!a || !b) return null; const p = [a[0] * b[0], a[0] * b[1], a[1] * b[0], a[1] * b[1]]; r = [Math.min(...p), Math.max(...p)] }
|
|
2629
|
+
else if ((op === 'i32.shr_u' || op === 'i32.shr_s') && isI32Const(e[2])) { const a = byteValueRange(e[1]); if (!a || a[0] < 0) return null; const s = constNum(e[2]); if (s < 0 || s > 16) return null; r = [a[0] >> s, a[1] >> s] }
|
|
2630
|
+
else if (op === 'i32.and' && isI32Const(e[2])) { const a = byteValueRange(e[1]); if (!a) return null; const m = constNum(e[2]); if (m < 0) return null; r = [0, Math.min(a[1], m)] }
|
|
2631
|
+
else return null
|
|
2632
|
+
return (r[0] < 0 || r[1] > 65535) ? null : r
|
|
2633
|
+
}
|
|
2634
|
+
const wideValueExpr = (!hasLoads && byteValueExpr) ? byteValueExpr : null // pure ramp → 16-wide pack
|
|
2635
|
+
const widenRange = (hasLoads && byteValueExpr) ? byteValueRange(byteValueExpr) : null
|
|
2636
|
+
const WIDEN16 = widenRange != null && widenRange[1] <= 255 // result fits a byte ⇒ narrow_u exact
|
|
2637
|
+
const WIDE16 = wideValueExpr != null
|
|
2638
|
+
const LANES = (WIDE16 || WIDEN16) ? 16 : 4
|
|
2639
|
+
const ramp = (off) => ['i32x4.add', ['i32x4.splat', ['local.get', ivName]],
|
|
2640
|
+
['v128.const', 'i32x4', String(off), String(off + 1), String(off + 2), String(off + 3)]]
|
|
2641
|
+
|
|
2642
|
+
let lifted
|
|
2643
|
+
if (WIDEN16) {
|
|
2644
|
+
// 16-wide widening byte-map: load each u8 input once (v128.load of 16 bytes),
|
|
2645
|
+
// extend_low/high to two i16x8 halves, run the affine map in i16x8 on each half,
|
|
2646
|
+
// narrow_u the two result halves to one i8x16, store 16. Each load is hoisted to a
|
|
2647
|
+
// temp (so extend_low + extend_high share one load); the loads run in source order,
|
|
2648
|
+
// so the offset `local.tee` in the first one is set before later loads read it.
|
|
2649
|
+
const loadTemps = new Map()
|
|
2650
|
+
const loadSets = []
|
|
2651
|
+
const collectLoads = (e) => {
|
|
2652
|
+
if (!isArr(e)) return
|
|
2653
|
+
if (e[0] === 'i32.load8_u') {
|
|
2654
|
+
const k = JSON.stringify(e[1])
|
|
2655
|
+
if (!loadTemps.has(k)) { const t = freshV128('win'); loadTemps.set(k, t); loadSets.push(['local.set', t, ['v128.load', e[1]]]) }
|
|
2656
|
+
} else for (let i = 1; i < e.length; i++) collectLoads(e[i])
|
|
2657
|
+
}
|
|
2658
|
+
collectLoads(byteValueExpr)
|
|
2659
|
+
const liftW = (e, half) => {
|
|
2660
|
+
const op = e[0]
|
|
2661
|
+
if (op === 'i32.const') return ['i16x8.splat', e]
|
|
2662
|
+
if (op === 'i32.load8_u') return [`i16x8.extend_${half}_i8x16_u`, ['local.get', loadTemps.get(JSON.stringify(e[1]))]]
|
|
2663
|
+
if (op === 'i32.add') return ['i16x8.add', liftW(e[1], half), liftW(e[2], half)]
|
|
2664
|
+
if (op === 'i32.sub') return ['i16x8.sub', liftW(e[1], half), liftW(e[2], half)]
|
|
2665
|
+
if (op === 'i32.mul') return ['i16x8.mul', liftW(e[1], half), liftW(e[2], half)]
|
|
2666
|
+
if (op === 'i32.shr_u' || op === 'i32.shr_s') return ['i16x8.shr_u', liftW(e[1], half), e[2]]
|
|
2667
|
+
if (op === 'i32.and') return ['v128.and', liftW(e[1], half), ['i16x8.splat', e[2]]]
|
|
2668
|
+
return null // byteValueRange already proved every op is one of the above
|
|
2669
|
+
}
|
|
2670
|
+
lifted = [...loadSets,
|
|
2671
|
+
['v128.store', storeAddr, ['i8x16.narrow_i16x8_u', liftW(byteValueExpr, 'low'), liftW(byteValueExpr, 'high')]]]
|
|
2672
|
+
} else if (WIDE16) {
|
|
2673
|
+
lifted = []
|
|
2674
|
+
const vv = []
|
|
2675
|
+
for (let j = 0; j < 4; j++) {
|
|
2676
|
+
const rt = freshV128('ramp')
|
|
2677
|
+
lifted.push(['local.set', rt, ramp(j * 4)])
|
|
2678
|
+
ctx.rampTemp = rt
|
|
2679
|
+
const v = liftExprV(wideValueExpr, ctx)
|
|
2680
|
+
if (ctx.fail) return null
|
|
2681
|
+
const vn = freshV128('rampv')
|
|
2682
|
+
lifted.push(['local.set', vn, v])
|
|
2683
|
+
vv.push(vn)
|
|
2684
|
+
}
|
|
2685
|
+
// Pack the low byte of all 16 i32 lanes (4 vectors) into one i8x16, in order.
|
|
2686
|
+
const g = (n) => ['local.get', n]
|
|
2687
|
+
const sh = (a, b, idx) => ['i8x16.shuffle', ...idx.map(String), a, b]
|
|
2688
|
+
const lo = freshV128('ramplo'), hi = freshV128('ramphi')
|
|
2689
|
+
lifted.push(['local.set', lo, sh(g(vv[0]), g(vv[1]), [0, 4, 8, 12, 16, 20, 24, 28, 0, 0, 0, 0, 0, 0, 0, 0])])
|
|
2690
|
+
lifted.push(['local.set', hi, sh(g(vv[2]), g(vv[3]), [0, 4, 8, 12, 16, 20, 24, 28, 0, 0, 0, 0, 0, 0, 0, 0])])
|
|
2691
|
+
lifted.push(['v128.store', storeAddr, sh(g(lo), g(hi), [0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23])])
|
|
2692
|
+
} else {
|
|
2693
|
+
ctx.rampTemp = freshV128('ramp')
|
|
2694
|
+
// ramp = [i, i+1, i+2, i+3], computed once per SIMD iteration.
|
|
2695
|
+
lifted = [['local.set', ctx.rampTemp, ramp(0)]]
|
|
2696
|
+
for (let i = 0; i < body.length; i++) {
|
|
2697
|
+
if (i === storeIdx) {
|
|
2698
|
+
const vval = liftExprV(storeStmt[2], ctx)
|
|
2699
|
+
if (ctx.fail) return null
|
|
2700
|
+
lifted.push(buildRampStore(storeOp, storeAddr, vval, ctx))
|
|
2701
|
+
} else {
|
|
2702
|
+
const r = liftStmt(body[i], ctx)
|
|
2703
|
+
if (ctx.fail) return null
|
|
2704
|
+
if (r != null) { if (Array.isArray(r) && r[0] === '__seq__') lifted.push(...r.slice(1)); else lifted.push(r) }
|
|
2705
|
+
}
|
|
2706
|
+
}
|
|
2707
|
+
}
|
|
2708
|
+
if (!lifted.length) return null
|
|
2709
|
+
|
|
2710
|
+
const id = freshIdRef.next++
|
|
2711
|
+
const simdBoundName = `$__simd_bound${id}`
|
|
2712
|
+
const simdBrkLabel = `$__simd_brk${id}`
|
|
2713
|
+
const simdLoopLabel = `$__simd_loop${id}`
|
|
2714
|
+
const boundExpr = boundLocal ? ['local.get', boundLocal] : bound
|
|
2715
|
+
|
|
2716
|
+
const scaledIncs = increments.map(({ name, c }) =>
|
|
2717
|
+
['local.set', name, ['i32.add', ['local.get', name], ['i32.const', c * LANES]]])
|
|
2718
|
+
|
|
2719
|
+
const simdBlock = ['block', simdBrkLabel,
|
|
2720
|
+
['loop', simdLoopLabel,
|
|
2721
|
+
['br_if', simdBrkLabel, ['i32.eqz', ['i32.lt_s', ['local.get', ivName], ['local.get', simdBoundName]]]],
|
|
2722
|
+
...lifted,
|
|
2723
|
+
...scaledIncs,
|
|
2724
|
+
['br', simdLoopLabel]]]
|
|
2725
|
+
const boundSetup = ['local.set', simdBoundName, ['i32.and', boundExpr, ['i32.const', -LANES]]]
|
|
2726
|
+
const wrapper = ['block', boundSetup, simdBlock, blockNode]
|
|
2727
|
+
const newLocalDecls = [
|
|
2728
|
+
['local', simdBoundName, 'i32'],
|
|
2729
|
+
...[...newLanedLocals.values()].map(({ laneName }) => ['local', laneName, 'v128']),
|
|
2730
|
+
...extraLocals,
|
|
2731
|
+
]
|
|
2732
|
+
return { wrapper, newLocalDecls }
|
|
2733
|
+
}
|
|
2734
|
+
|
|
2735
|
+
// Build the store for a ramp-map iteration: i32x4 `vval` → element width of
|
|
2736
|
+
// `storeOp` at scalar address `addr`. i32.store is the full vector; i32.store8
|
|
2737
|
+
// truncates (low byte of each lane) via i8x16.shuffle — exactly matching scalar
|
|
2738
|
+
// store8, with no value-range assumption (shuffle selects, never saturates).
|
|
2739
|
+
function buildRampStore(storeOp, addr, vval, ctx) {
|
|
2740
|
+
if (storeOp === 'i32.store') return ['v128.store', addr, vval] // 4 i32 lanes → 16 bytes
|
|
2741
|
+
// i32.store8: hoist vval to a temp so the shuffle reads it once; low byte of
|
|
2742
|
+
// each of 4 lanes → bytes 0..3 → one i32.store (4 bytes). Shuffle lane indices
|
|
2743
|
+
// are string tokens for watr's binary encoder.
|
|
2744
|
+
const tmp = `$__rampv${ctx.freshIdRef.next++}`
|
|
2745
|
+
ctx.extraLocals.push(['local', tmp, 'v128'])
|
|
2746
|
+
const g = ['local.get', tmp]
|
|
2747
|
+
const packed = ['i8x16.shuffle', ...[0, 4, 8, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0].map(String), g, g]
|
|
2748
|
+
return ['block', ['local.set', tmp, vval], ['i32.store', addr, ['i32x4.extract_lane', 0, packed]]]
|
|
2749
|
+
}
|
|
2750
|
+
|
|
2751
|
+
// Pivot-stride analysis for the multi-pixel lift. The lift reads 16 source bytes
|
|
2752
|
+
// (4 RGBA pixels) with ONE v128.load at the address for output pixel `pivot`, so the
|
|
2753
|
+
// 4 outputs are correct ONLY if consecutive output pixels read consecutive source
|
|
2754
|
+
// pixels — the load address must advance by EXACTLY 4 bytes per pivot step. Build, in
|
|
2755
|
+
// program order, each local's value-delta per unit-pivot increment (`pivot` → 1, every
|
|
2756
|
+
// other local → its assigned expr's delta, unknown/outer locals → 0 = pivot-invariant).
|
|
2757
|
+
// `delta(e)` returns that constant byte-delta, or null when the dependence is non-
|
|
2758
|
+
// constant (e.g. x*k) or uses an op we don't model — both of which must bail.
|
|
2759
|
+
// Index arithmetic is often f64-lowered (JS number `*`): `(yi*ww + x)` becomes
|
|
2760
|
+
// `(f64(yi)*ww + f64(x)) |0` = trunc_sat with a NaN-guard `select`. We model those
|
|
2761
|
+
// passthrough/arith ops too, so a runtime-dimension vblur (x carried through f64)
|
|
2762
|
+
// analyses the same as a literal-dimension one (x in plain i32). The select is the
|
|
2763
|
+
// `|0` coercion (value branch when the index is finite — always so for an integer
|
|
2764
|
+
// array index); we take the value branch's delta. Multiplies need a compile-time
|
|
2765
|
+
// constant factor on the pivot-bearing side (else the stride isn't constant).
|
|
2766
|
+
function buildPivotCoeff(loopNode, pivot) {
|
|
2767
|
+
const coeff = new Map([[pivot, 1]])
|
|
2768
|
+
const cval = (n) => isI32Const(n) ? constNum(n) : (isArr(n) && n[0] === 'f64.const' && n[1] != null ? Number(n[1]) : null)
|
|
2769
|
+
const delta = (e) => {
|
|
2770
|
+
if (isI32Const(e)) return 0
|
|
2771
|
+
if (isLocalGet(e)) return coeff.has(e[1]) ? coeff.get(e[1]) : 0
|
|
2772
|
+
if (!isArr(e)) return null
|
|
2773
|
+
const [op, a, b] = e
|
|
2774
|
+
switch (op) {
|
|
2775
|
+
case 'i32.add': case 'f64.add': { const x = delta(a), y = delta(b); return x == null || y == null ? null : x + y }
|
|
2776
|
+
case 'i32.sub': case 'f64.sub': { const x = delta(a), y = delta(b); return x == null || y == null ? null : x - y }
|
|
2777
|
+
case 'i32.shl': { const c = cval(b), x = delta(a); return c == null || x == null ? null : x << c }
|
|
2778
|
+
case 'i32.mul': case 'f64.mul': {
|
|
2779
|
+
const ca = cval(a), cb = cval(b), x = delta(a), y = delta(b)
|
|
2780
|
+
if (x == null || y == null) return null
|
|
2781
|
+
if (ca != null) return ca * y // const × expr
|
|
2782
|
+
if (cb != null) return cb * x // expr × const
|
|
2783
|
+
return x === 0 && y === 0 ? 0 : null // var × var: pivot-dependent ⇒ non-constant
|
|
2784
|
+
}
|
|
2785
|
+
// value-preserving conversions: passthrough. `local.tee $v EXPR` yields EXPR.
|
|
2786
|
+
case 'f64.convert_i32_s': case 'f64.convert_i32_u': case 'i64.trunc_sat_f64_s':
|
|
2787
|
+
case 'i64.trunc_sat_f64_u': case 'i32.wrap_i64': case 'i64.extend_i32_s':
|
|
2788
|
+
return delta(a)
|
|
2789
|
+
case 'local.tee': return delta(b)
|
|
2790
|
+
// the `(expr)|0` NaN/Inf-guard `select(value, 0, finite?)`: take the value branch
|
|
2791
|
+
// (the index is a finite integer in any real blur). Only when the else-branch is
|
|
2792
|
+
// the const-0 guard — a general ternary index whose arms differ in stride bails.
|
|
2793
|
+
case 'select': return isI32Const(e[2]) && constNum(e[2]) === 0 ? delta(a) : null
|
|
2794
|
+
default: return null // any other op (and/or/div/…) ⇒ unanalyzable
|
|
2795
|
+
}
|
|
2796
|
+
}
|
|
2797
|
+
const visit = (n) => {
|
|
2798
|
+
if (!isArr(n)) return
|
|
2799
|
+
if ((n[0] === 'local.set' || n[0] === 'local.tee') && typeof n[1] === 'string') { coeff.set(n[1], delta(n[2])); visit(n[2]); return }
|
|
2800
|
+
n.forEach(visit)
|
|
2801
|
+
}
|
|
2802
|
+
visit(loopNode)
|
|
2803
|
+
return delta
|
|
2804
|
+
}
|
|
2805
|
+
|
|
2806
|
+
// Vectorize the inner accumulation of an RGBA box-filter pixel loop. See the
|
|
2807
|
+
// matchChannelGroup header. Returns { wrapper, newLocalDecls } or null.
|
|
2808
|
+
// Multi-pixel SIMD lift of a clamp-free RGBA box-filter interior (produced by the
|
|
2809
|
+
// clamp-peel pass). Processes 4 output pixels per iteration: at tap k the 4 outputs
|
|
2810
|
+
// read 4 CONSECUTIVE source pixels — one 16-byte v128.load whose lanes map DIRECTLY
|
|
2811
|
+
// to the 4 accumulators (no shuffle). Sums (≤ win·255 < 2^16) accumulate in two
|
|
2812
|
+
// i16x8 vectors; the divide+store reuses the scalar store templates per pixel.
|
|
2813
|
+
// Validated bit-exact + 4.34× vs scalar. Bails (→ tryChannelReduce 1-pixel lift)
|
|
2814
|
+
// unless the body is exactly: clamp-free `xi=x+k; p=(row+xi)<<2; acc_c += u8[base+c]`
|
|
2815
|
+
// over a unit-stride x, then a 4-byte RGBA store `dst[ab1+c] = f(acc_c)`. r≥127 →
|
|
2816
|
+
// i16 overflow → bail. Leaves a scalar remainder loop for the ≤3 trailing pixels.
|
|
2817
|
+
function tryBlurMultiPixel(blockNode, fnLocals, freshIdRef) {
|
|
2818
|
+
if (!isArr(blockNode) || blockNode[0] !== 'block') return null
|
|
2819
|
+
let blockLabel = null, loopNode = null
|
|
2820
|
+
for (let i = 1; i < blockNode.length; i++) {
|
|
2821
|
+
const c = blockNode[i]
|
|
2822
|
+
if (typeof c === 'string' && c.startsWith('$') && blockLabel == null && i === 1) { blockLabel = c; continue }
|
|
2823
|
+
if (isArr(c) && c[0] === 'loop') { if (loopNode) return null; loopNode = c }
|
|
2824
|
+
}
|
|
2825
|
+
if (!loopNode || !blockLabel) return null
|
|
2826
|
+
const loopLabel = typeof loopNode[1] === 'string' && loopNode[1].startsWith('$') ? loopNode[1] : null
|
|
2827
|
+
if (!loopLabel) return null
|
|
2828
|
+
const endIdx = loopNode.length - 1
|
|
2829
|
+
if (!(isArr(loopNode[endIdx]) && loopNode[endIdx][0] === 'br' && loopNode[endIdx][1] === loopLabel)) return null
|
|
2830
|
+
// exit guard: br_if $brk (i32.eqz (i32.lt_s x BOUND))
|
|
2831
|
+
const exit = loopNode[2]
|
|
2832
|
+
if (!(isArr(exit) && exit[0] === 'br_if' && isArr(exit[2]) && exit[2][0] === 'i32.eqz'
|
|
2833
|
+
&& isArr(exit[2][1]) && exit[2][1][0] === 'i32.lt_s' && isLocalGet(exit[2][1][1]))) return null
|
|
2834
|
+
const pivot = exit[2][1][1][1] // pixel IV $x
|
|
2835
|
+
const bound = exit[2][1][2] // interior bound (expr)
|
|
2836
|
+
const bodyEnd = endIdx - 1
|
|
2837
|
+
// increment must be `x = x + 1`
|
|
2838
|
+
const inc = loopNode[bodyEnd]
|
|
2839
|
+
if (!(isArr(inc) && inc[0] === 'local.set' && inc[1] === pivot && isArr(inc[2]) && inc[2][0] === 'i32.add'
|
|
2840
|
+
&& isLocalGet(inc[2][1], pivot) && isI32Const(inc[2][2]) && constNum(inc[2][2]) === 1)) return null
|
|
2841
|
+
|
|
2842
|
+
// four `acc_c = 0` inits
|
|
2843
|
+
let initIdx = -1, accInits = null
|
|
2844
|
+
for (let i = 3; i + 3 <= bodyEnd; i++) {
|
|
2845
|
+
const z = []
|
|
2846
|
+
for (let c = 0; c < 4; c++) { const s = loopNode[i + c]; if (isArr(s) && s[0] === 'local.set' && s.length === 3 && isI32Const(s[2]) && constNum(s[2]) === 0) z.push(s[1]); else break }
|
|
2847
|
+
if (z.length === 4 && new Set(z).size === 4) { initIdx = i; accInits = z; break }
|
|
2848
|
+
}
|
|
2849
|
+
if (initIdx < 0) return null
|
|
2850
|
+
// the tap accumulation loop
|
|
2851
|
+
let innerIdx = -1, innerBlock = null
|
|
2852
|
+
for (let i = initIdx + 4; i <= bodyEnd; i++) {
|
|
2853
|
+
const s = loopNode[i]
|
|
2854
|
+
if (isArr(s) && s[0] === 'block' && s.slice(1).some(c => isArr(c) && c[0] === 'loop')) { innerIdx = i; innerBlock = s; break }
|
|
2855
|
+
if (isArr(s) && s[0] === 'loop') { innerIdx = i; innerBlock = ['block', s]; break }
|
|
2856
|
+
}
|
|
2857
|
+
if (!innerBlock) return null
|
|
2858
|
+
const innerLoop = innerBlock.find(c => isArr(c) && c[0] === 'loop')
|
|
2859
|
+
if (!innerLoop) return null
|
|
2860
|
+
const ilEnd = innerLoop.length - 1
|
|
2861
|
+
if (!(isArr(innerLoop[ilEnd]) && innerLoop[ilEnd][0] === 'br')) return null
|
|
2862
|
+
const innerBody = innerLoop.slice(3, ilEnd - 1)
|
|
2863
|
+
const grp = matchChannelGroup(innerBody)
|
|
2864
|
+
if (!grp) return null
|
|
2865
|
+
if (grp.accs.slice().sort().join('\x00') !== accInits.slice().sort().join('\x00')) return null
|
|
2866
|
+
// CLAMP-FREE: nothing before the channel group may be an `if` (the edge clamp).
|
|
2867
|
+
for (let i = 0; i < grp.idx; i++) if (isArr(innerBody[i]) && innerBody[i][0] === 'if') return null
|
|
2868
|
+
// The tap loop must be the i32 form `k <= r`; extract the radius r (reused in the
|
|
2869
|
+
// runtime overflow guard below). f64-typed tap loops (f64 dims) bail to scalar.
|
|
2870
|
+
const ic = innerLoop[2]
|
|
2871
|
+
if (!(isArr(ic) && ic[0] === 'br_if' && isArr(ic[2]) && ic[2][0] === 'i32.eqz'
|
|
2872
|
+
&& isArr(ic[2][1]) && ic[2][1][0] === 'i32.le_s' && isLocalGet(ic[2][1][1]))) return null
|
|
2873
|
+
const rBound = ic[2][1][2]
|
|
2874
|
+
if (!(isLocalGet(rBound) || isI32Const(rBound))) return null
|
|
2875
|
+
|
|
2876
|
+
// the RGBA store: 4 consecutive i32.store8 at `ab1 + c`, ab1 tee'd in the first.
|
|
2877
|
+
let storeIdx = -1
|
|
2878
|
+
for (let i = innerIdx + 1; i + 3 <= bodyEnd; i++) {
|
|
2879
|
+
const s0 = loopNode[i]
|
|
2880
|
+
if (isArr(s0) && s0[0] === 'i32.store8' && s0.length === 3 && isArr(s0[1]) && s0[1][0] === 'local.tee') { storeIdx = i; break }
|
|
2881
|
+
}
|
|
2882
|
+
if (storeIdx < 0) return null
|
|
2883
|
+
const s0 = loopNode[storeIdx]
|
|
2884
|
+
const ab1 = s0[1][1], dstExpr = s0[1][2] // ab1 local, its address expr
|
|
2885
|
+
const storeVals = [s0[2]]
|
|
2886
|
+
for (let c = 1; c < 4; c++) {
|
|
2887
|
+
const s = loopNode[storeIdx + c]
|
|
2888
|
+
if (!(isArr(s) && s[0] === 'i32.store8' && s[1] === `offset=${c}` && isLocalGet(s[2], ab1))) return null
|
|
2889
|
+
storeVals.push(s[3])
|
|
2890
|
+
}
|
|
2891
|
+
// each store value must read its accumulator (the divided sum)
|
|
2892
|
+
for (let c = 0; c < 4; c++) if (!readsVar(storeVals[c], accInits[c])) return null
|
|
2893
|
+
|
|
2894
|
+
// ── Soundness guards for the 4-pixels-per-load lift ──────────────────────────
|
|
2895
|
+
// Scope the stride analysis to THIS pixel loop. A peeled interior is one segment ⇒
|
|
2896
|
+
// one inner tap loop, so there is no sibling same-named-`p` ambiguity, and the scope
|
|
2897
|
+
// still captures the LICM preamble where the x-dependent index term is hoisted out of
|
|
2898
|
+
// the tap loop (e.g. `__li = f64(x)` — x is invariant w.r.t. the tap IV k).
|
|
2899
|
+
const pivotDelta = buildPivotCoeff(loopNode, pivot)
|
|
2900
|
+
// (a) Source unit-stride: the v128.load address (array base + byte index) must
|
|
2901
|
+
// advance by exactly 4 bytes (one RGBA pixel) per output-pixel step, else the
|
|
2902
|
+
// 16-byte load spans the wrong source pixels. Bails on strided indices
|
|
2903
|
+
// ((xi*2+row)<<2) and non-constant ones ((yi*W + x + k*x)<<2 → x*(1+k)).
|
|
2904
|
+
if (pivotDelta(grp.teeExpr) !== 4) return null
|
|
2905
|
+
// (b) No dropped pivot-dependent setup: statements before the inits (loopNode[3..
|
|
2906
|
+
// initIdx)) are NOT carried into the 4-pixel loop, so a per-pixel value computed
|
|
2907
|
+
// there (e.g. `x2 = x*2`) would go stale. Bail if any such statement is pivot-
|
|
2908
|
+
// dependent. (Statements between the inits and the inner loop ARE carried.)
|
|
2909
|
+
for (let i = 3; i < initIdx; i++) {
|
|
2910
|
+
let bad = false
|
|
2911
|
+
const chk = (n) => { if (!isArr(n)) return; if ((n[0] === 'local.set' || n[0] === 'local.tee') && typeof n[1] === 'string' && pivotDelta(n[2]) !== 0) bad = true; n.forEach(chk) }
|
|
2912
|
+
chk(loopNode[i])
|
|
2913
|
+
if (bad) return null
|
|
2914
|
+
}
|
|
2915
|
+
// (c) Store values must read only the accumulators (set per-pixel by the epilogue)
|
|
2916
|
+
// and pivot-invariants — never a per-pixel local like the pivot itself, since the
|
|
2917
|
+
// store template is reused verbatim for all 4 pixels (`dst[o]=(sr+x)&255` would use
|
|
2918
|
+
// the group-base x for pixels 1-3).
|
|
2919
|
+
const perPixel = new Set()
|
|
2920
|
+
const collectSet = (n) => { if (!isArr(n)) return; if ((n[0] === 'local.set' || n[0] === 'local.tee') && typeof n[1] === 'string') perPixel.add(n[1]); n.forEach(collectSet) }
|
|
2921
|
+
collectSet(loopNode)
|
|
2922
|
+
for (const a of grp.accs) perPixel.delete(a) // accumulators are repopulated per pixel
|
|
2923
|
+
const readsPerPixel = (n) => isArr(n) ? ((n[0] === 'local.get' && perPixel.has(n[1])) || n.some(readsPerPixel)) : false
|
|
2924
|
+
for (let c = 0; c < 4; c++) if (readsPerPixel(storeVals[c])) return null
|
|
2925
|
+
|
|
2926
|
+
// ── Lift ───────────────────────────────────────────────────────────────────
|
|
2927
|
+
const id = freshIdRef.next++
|
|
2928
|
+
const accLo = `$__bmplo${id}`, accHi = `$__bmphi${id}`, b4 = `$__bmpb${id}`
|
|
2929
|
+
const newLocalDecls = [['local', accLo, 'v128'], ['local', accHi, 'v128'], ['local', b4, 'i32']]
|
|
2930
|
+
const Z = ['v128.const', 'i64x2', '0', '0']
|
|
2931
|
+
// inner body: 4 channel adds → 2 i16 widening adds off the same tee'd base.
|
|
2932
|
+
const newInner = []
|
|
2933
|
+
for (let i = 0; i < innerBody.length; i++) {
|
|
2934
|
+
if (i === grp.idx) {
|
|
2935
|
+
newInner.push(['local.set', accLo, ['i16x8.add', ['local.get', accLo], ['i16x8.extend_low_i8x16_u', ['v128.load', ['local.tee', grp.baseLocal, grp.teeExpr]]]]])
|
|
2936
|
+
newInner.push(['local.set', accHi, ['i16x8.add', ['local.get', accHi], ['i16x8.extend_high_i8x16_u', ['v128.load', ['local.get', grp.baseLocal]]]]])
|
|
2937
|
+
} else if (i > grp.idx && i <= grp.idx + 3) continue
|
|
2938
|
+
else newInner.push(innerBody[i])
|
|
2939
|
+
}
|
|
2940
|
+
const newInnerLoop = [...innerLoop.slice(0, 3), ...newInner, ...innerLoop.slice(ilEnd - 1)]
|
|
2941
|
+
const newInnerBlock = innerBlock.map(c => c === innerLoop ? newInnerLoop : c)
|
|
2942
|
+
// store epilogue: the o=(row+x)<<2 setup, then ab1, then per pixel j set acc_c to
|
|
2943
|
+
// its lane and reuse the scalar store template at byte offset j*4+c.
|
|
2944
|
+
const preStore = loopNode.slice(innerIdx + 1, storeIdx) // `o = (row+x)<<2` etc.
|
|
2945
|
+
const epilogue = [...preStore, ['local.set', ab1, dstExpr]]
|
|
2946
|
+
for (let j = 0; j < 4; j++) {
|
|
2947
|
+
const vec = j < 2 ? accLo : accHi, base = (j % 2) * 4
|
|
2948
|
+
// lane base+c summed source byte at offset c → grp.accs[c] (the accumulator
|
|
2949
|
+
// tied to read-offset c), NOT accInits[c] (zero-init order); see tryChannelReduce.
|
|
2950
|
+
for (let c = 0; c < 4; c++) epilogue.push(['local.set', grp.accs[c], ['i16x8.extract_lane_u', String(base + c), ['local.get', vec]]])
|
|
2951
|
+
for (let c = 0; c < 4; c++) epilogue.push(['i32.store8', `offset=${j * 4 + c}`, ['local.get', ab1], storeVals[c]])
|
|
2952
|
+
}
|
|
2953
|
+
// 4-pixel loop body: splat inits, the tap-IV init, the lifted inner loop, the
|
|
2954
|
+
// lifted store, x += 4.
|
|
2955
|
+
const v4label = `$__bmpx${id}`, v4brk = `$__bmpe${id}`
|
|
2956
|
+
const fourBody = [
|
|
2957
|
+
['br_if', v4brk, ['i32.eqz', ['i32.lt_s', ['local.get', pivot], ['local.get', b4]]]],
|
|
2958
|
+
['local.set', accLo, Z], ['local.set', accHi, Z],
|
|
2959
|
+
...loopNode.slice(initIdx + 4, innerIdx),
|
|
2960
|
+
newInnerBlock,
|
|
2961
|
+
...epilogue,
|
|
2962
|
+
['local.set', pivot, ['i32.add', ['local.get', pivot], ['i32.const', 4]]],
|
|
2963
|
+
['br', v4label],
|
|
2964
|
+
]
|
|
2965
|
+
// Deep-clone the 4-pixel loop: it reuses sub-trees (inner body, store templates,
|
|
2966
|
+
// dst expr) that also live in the scalar remainder below — sharing would let a
|
|
2967
|
+
// later pass mutating one corrupt the other.
|
|
2968
|
+
const dc = (n) => isArr(n) ? n.map(dc) : n
|
|
2969
|
+
const fourLoop = dc(['block', v4brk, ['loop', v4label, ...fourBody]])
|
|
2970
|
+
// b4 = x + (r<128 ? ((bound-x) & ~3) : 0): the 4-aligned end of the interior from
|
|
2971
|
+
// the entry x. The r<128 guard is the i16-overflow check (win·255 < 2^16 ⇔ r<128);
|
|
2972
|
+
// when r≥128 the 4-pixel loop runs zero iterations and the scalar remainder covers
|
|
2973
|
+
// the whole interior — sound for any runtime r, no duplicated loop body.
|
|
2974
|
+
const b4set = ['local.set', b4, ['i32.add', ['local.get', pivot],
|
|
2975
|
+
['select', ['i32.and', ['i32.sub', dc(bound), ['local.get', pivot]], ['i32.const', -4]], ['i32.const', 0],
|
|
2976
|
+
['i32.lt_s', dc(rBound), ['i32.const', 128]]]]]
|
|
2977
|
+
// The interior block may carry a LICM-hoisted invariant preamble (e.g. `k=-r`'s
|
|
2978
|
+
// value) that the lifted body reads but which the original sets only on block
|
|
2979
|
+
// entry — run a clone of it first so the 4-pixel loop sees it. Then b4 setup, the
|
|
2980
|
+
// 4-pixel loop, then the ORIGINAL block unchanged (the ≤3-pixel scalar remainder;
|
|
2981
|
+
// x continues from b4).
|
|
2982
|
+
const loopPos = blockNode.indexOf(loopNode)
|
|
2983
|
+
const preamble = blockNode.slice(2, loopPos).map(dc)
|
|
2984
|
+
const wrapper = ['block', ...preamble, b4set, fourLoop, blockNode]
|
|
2985
|
+
return { wrapper, newLocalDecls }
|
|
2986
|
+
}
|
|
2987
|
+
|
|
2988
|
+
function tryChannelReduce(blockNode, fnLocals, freshIdRef) {
|
|
2989
|
+
if (!isArr(blockNode) || blockNode[0] !== 'block') return null
|
|
2990
|
+
// Outer pixel loop: (block $brk [invariant preamble…] (loop $L …)). When the
|
|
2991
|
+
// pixel loop is nested in an outer row loop, LICM hoists the invariant edge-clamp
|
|
2992
|
+
// bound (`w-1`) into this block ahead of the loop; allow any such non-loop
|
|
2993
|
+
// preamble — it is preserved verbatim by the wrapper rebuild, which rewrites only
|
|
2994
|
+
// the loop. Exactly one loop is still required (a second → bail).
|
|
2995
|
+
let blockLabel = null, loopNode = null
|
|
2996
|
+
for (let i = 1; i < blockNode.length; i++) {
|
|
2997
|
+
const c = blockNode[i]
|
|
2998
|
+
if (typeof c === 'string' && c.startsWith('$') && blockLabel == null && i === 1) { blockLabel = c; continue }
|
|
2999
|
+
if (isArr(c) && c[0] === 'loop') { if (loopNode) return null; loopNode = c }
|
|
3000
|
+
}
|
|
3001
|
+
if (!loopNode || !blockLabel) return null
|
|
3002
|
+
const loopLabel = typeof loopNode[1] === 'string' && loopNode[1].startsWith('$') ? loopNode[1] : null
|
|
3003
|
+
if (!loopLabel) return null
|
|
3004
|
+
const endIdx = loopNode.length - 1
|
|
3005
|
+
if (!(isArr(loopNode[endIdx]) && loopNode[endIdx][0] === 'br' && loopNode[endIdx][1] === loopLabel)) return null
|
|
3006
|
+
|
|
3007
|
+
// Pixel-loop body (between the exit guard and the back-branch). Find: four
|
|
3008
|
+
// `acc_c = 0` inits and the inner accumulation loop that sums into them.
|
|
3009
|
+
const bodyStart = 3, bodyEnd = endIdx - 1 // [exit] at 2, [inc?][br] at the end
|
|
3010
|
+
// The four zero-inits must be consecutive `(local.set $acc (i32.const 0))`.
|
|
3011
|
+
let initIdx = -1, accInits = null
|
|
3012
|
+
for (let i = bodyStart; i + 3 <= bodyEnd; i++) {
|
|
3013
|
+
const z = []
|
|
3014
|
+
for (let c = 0; c < 4; c++) {
|
|
3015
|
+
const s = loopNode[i + c]
|
|
3016
|
+
if (isArr(s) && s[0] === 'local.set' && s.length === 3 && isI32Const(s[2]) && constNum(s[2]) === 0 && typeof s[1] === 'string') z.push(s[1])
|
|
3017
|
+
else break
|
|
3018
|
+
}
|
|
3019
|
+
if (z.length === 4 && new Set(z).size === 4) { initIdx = i; accInits = z; break }
|
|
3020
|
+
}
|
|
3021
|
+
if (initIdx < 0) return null
|
|
3022
|
+
|
|
3023
|
+
// The inner accumulation loop is the (block (loop)) appearing after the inits.
|
|
3024
|
+
let innerIdx = -1, innerBlock = null
|
|
3025
|
+
for (let i = initIdx + 4; i <= bodyEnd; i++) {
|
|
3026
|
+
const s = loopNode[i]
|
|
3027
|
+
if (isArr(s) && s[0] === 'block' && s.slice(1).some(c => isArr(c) && c[0] === 'loop')) { innerIdx = i; innerBlock = s; break }
|
|
3028
|
+
if (isArr(s) && s[0] === 'loop') { innerIdx = i; innerBlock = ['block', s]; break }
|
|
3029
|
+
}
|
|
3030
|
+
if (!innerBlock) return null
|
|
3031
|
+
// Locate the loop within the inner block and its body.
|
|
3032
|
+
const innerLoop = innerBlock.find(c => isArr(c) && c[0] === 'loop')
|
|
3033
|
+
if (!innerLoop) return null
|
|
3034
|
+
const ilEnd = innerLoop.length - 1
|
|
3035
|
+
if (!(isArr(innerLoop[ilEnd]) && innerLoop[ilEnd][0] === 'br')) return null
|
|
3036
|
+
const innerBody = innerLoop.slice(3, ilEnd - 1) // between exit guard and the (k+=1)(br)
|
|
3037
|
+
|
|
3038
|
+
const grp = matchChannelGroup(innerBody)
|
|
3039
|
+
if (!grp) return null
|
|
3040
|
+
// The four accumulators summed must be exactly the four that were zero-inited
|
|
3041
|
+
// (same set, in any order).
|
|
3042
|
+
if (grp.accs.slice().sort().join('\x00') !== accInits.slice().sort().join('\x00')) return null
|
|
3043
|
+
|
|
3044
|
+
// ── Lift. accv (v128) holds the four channel sums.
|
|
3045
|
+
const id = freshIdRef.next++
|
|
3046
|
+
const accv = `$__chv${id}`
|
|
3047
|
+
const newLocalDecls = [['local', accv, 'v128']]
|
|
3048
|
+
// Zero-init → one splat.
|
|
3049
|
+
const zeroInit = ['local.set', accv, ['v128.const', 'i32x4', '0', '0', '0', '0']]
|
|
3050
|
+
// Inner accumulation: keep the address-producing stmts, replace the 4 channel
|
|
3051
|
+
// adds with one widening add off the shared base. The base address is the
|
|
3052
|
+
// tee'd expr of the first load (re-tee'd so any later `local.get base` still
|
|
3053
|
+
// resolves — though the other channel loads are gone).
|
|
3054
|
+
const widen = ['i32x4.extend_low_i16x8_u', ['i16x8.extend_low_i8x16_u',
|
|
3055
|
+
['v128.load32_zero', ['local.tee', grp.baseLocal, grp.teeExpr]]]]
|
|
3056
|
+
const newInner = []
|
|
3057
|
+
for (let i = 0; i < innerBody.length; i++) {
|
|
3058
|
+
if (i === grp.idx) newInner.push(['local.set', accv, ['i32x4.add', ['local.get', accv], widen]])
|
|
3059
|
+
else if (i > grp.idx && i <= grp.idx + 3) continue // drop the other 3 channel adds
|
|
3060
|
+
else newInner.push(innerBody[i])
|
|
3061
|
+
}
|
|
3062
|
+
// Rebuild the inner loop with the lifted body.
|
|
3063
|
+
const newInnerLoop = [...innerLoop.slice(0, 3), ...newInner, ...innerLoop.slice(ilEnd - 1)]
|
|
3064
|
+
const newInnerBlock = innerBlock.map(c => c === innerLoop ? newInnerLoop : c)
|
|
3065
|
+
// After the inner loop, extract the four lane sums back to the scalar
|
|
3066
|
+
// accumulators the divide+store code reads. Lane c summed source byte
|
|
3067
|
+
// `base+c`, so it must land in grp.accs[c] — the accumulator matchChannelGroup
|
|
3068
|
+
// tied to read-offset c — NOT accInits[c] (zero-init order). They coincide only
|
|
3069
|
+
// when the program inits/sums in offset order (every real blur); a channel-
|
|
3070
|
+
// permuted program (sums offset 0 into a var stored at a later offset) needs the
|
|
3071
|
+
// offset-order mapping or the lanes mis-map.
|
|
3072
|
+
const extracts = grp.accs.map((acc, c) => ['local.set', acc, ['i32x4.extract_lane', c, ['local.get', accv]]])
|
|
3073
|
+
|
|
3074
|
+
// Reassemble the pixel-loop body: replace the 4 inits with the splat, the inner
|
|
3075
|
+
// block with the lifted one, and insert the extracts right after it.
|
|
3076
|
+
const newLoop = [...loopNode]
|
|
3077
|
+
newLoop.splice(innerIdx, 1, newInnerBlock, ...extracts) // inner block → lifted + extracts
|
|
3078
|
+
newLoop.splice(initIdx, 4, zeroInit) // 4 inits → 1 splat (do last; lower index)
|
|
3079
|
+
const wrapper = blockNode.map(c => c === loopNode ? newLoop : c)
|
|
3080
|
+
return { wrapper, newLocalDecls }
|
|
3081
|
+
}
|
|
3082
|
+
|
|
3083
|
+
/**
|
|
3084
|
+
* Divergent escape-time vectorizer — 2-wide f64x2, bit-exact with scalar f64.
|
|
3085
|
+
*
|
|
3086
|
+
* Recognizes the fractal escape-time nest (mandelbrot / julia / burning-ship / …):
|
|
3087
|
+
*
|
|
3088
|
+
* for (qx = 0; qx < W; qx++) { // outer pixel loop; ≥1 i32 pixel IVs
|
|
3089
|
+
* cx = <expr in qx> // per-pixel coordinate(s)
|
|
3090
|
+
* x = 0; y = 0; it = 0 // loop-carried f64 + i32 counter
|
|
3091
|
+
* while (it < MAXIT) { // inner escape loop (it-limited)
|
|
3092
|
+
* <f64 updates to x,y (+ temps); one `if (|z|² > T) break`, ANY position>
|
|
3093
|
+
* it++
|
|
3094
|
+
* }
|
|
3095
|
+
* <epilogue: store(it) OR smooth-colour(x,y,it) → store>
|
|
3096
|
+
* }
|
|
3097
|
+
*
|
|
3098
|
+
* Two adjacent pixels run in f64x2 lockstep with a per-lane active mask. A lane is
|
|
3099
|
+
* frozen (v128.bitselect) the instant it would escape or reach MAXIT, so its
|
|
3100
|
+
* it/x/y stay bit-identical to the scalar loop — f64x2 add/sub/mul/abs are IEEE-
|
|
3101
|
+
* identical to scalar f64 (no FMA fusion). The body is emitted statement-by-
|
|
3102
|
+
* statement in source order, so the escape mask lands exactly where the scalar
|
|
3103
|
+
* `break` is (before OR after the z-update), and the per-lane state freezes at the
|
|
3104
|
+
* matching point. iter is kept as f64x2 (an i32 `it` is exact in f64), so the
|
|
3105
|
+
* limit compare and the (possibly fractional) colour math both stay bitwise-exact.
|
|
3106
|
+
*
|
|
3107
|
+
* Loop-carried vars (first body access is a READ) freeze via bitselect; within-
|
|
3108
|
+
* iteration temps (first access a WRITE — inlined squares, `xt`) recompute raw.
|
|
3109
|
+
* The colour epilogue runs scalar, twice — once per lane — reading each lane's
|
|
3110
|
+
* extracted x/y/it, with the pixel IVs advanced by +0/+1. Odd widths and extra
|
|
3111
|
+
* lanes fall through to the original scalar pixel loop (the exact tail).
|
|
3112
|
+
*/
|
|
3113
|
+
const CMP_NEG = { // comparison → its logical negation (active lanes are finite → NaN-free)
|
|
3114
|
+
'f64.gt': 'f64.le', 'f64.ge': 'f64.lt', 'f64.lt': 'f64.ge', 'f64.le': 'f64.gt', 'f64.eq': 'f64.ne', 'f64.ne': 'f64.eq',
|
|
3115
|
+
'i32.lt_s': 'i32.ge_s', 'i32.ge_s': 'i32.lt_s', 'i32.gt_s': 'i32.le_s', 'i32.le_s': 'i32.gt_s',
|
|
3116
|
+
'i32.lt_u': 'i32.ge_u', 'i32.ge_u': 'i32.lt_u', 'i32.gt_u': 'i32.le_u', 'i32.le_u': 'i32.gt_u',
|
|
3117
|
+
}
|
|
3118
|
+
const CMP_LANE = { // f64/i32 scalar compare → f64x2 lane compare (iter is f64x2; z-compares are f64)
|
|
3119
|
+
'f64.gt': 'f64x2.gt', 'f64.ge': 'f64x2.ge', 'f64.lt': 'f64x2.lt', 'f64.le': 'f64x2.le', 'f64.eq': 'f64x2.eq', 'f64.ne': 'f64x2.ne',
|
|
3120
|
+
'i32.lt_s': 'f64x2.lt', 'i32.le_s': 'f64x2.le', 'i32.ge_s': 'f64x2.ge', 'i32.gt_s': 'f64x2.gt',
|
|
3121
|
+
'i32.lt_u': 'f64x2.lt', 'i32.le_u': 'f64x2.le', 'i32.ge_u': 'f64x2.ge', 'i32.gt_u': 'f64x2.gt',
|
|
3122
|
+
}
|
|
3123
|
+
const readsVar = (n, v) => isArr(n) && ((n[0] === 'local.get' && n[1] === v) || n.some(c => readsVar(c, v)))
|
|
3124
|
+
const writesName = (n, name) => isArr(n) && (((n[0] === 'local.set' || n[0] === 'local.tee' || n[0] === 'global.set') && n[1] === name) || n.some(c => writesName(c, name)))
|
|
3125
|
+
// Pixel induction variables may be i32 (const-bound loops) or f64 (param-bound loops,
|
|
3126
|
+
// e.g. `for (x=0; x<width; ++x)` with f64 `width`). Match `v += 1` and `v < bound` for both.
|
|
3127
|
+
const matchPixelInc = (stmt) => {
|
|
3128
|
+
if (!isArr(stmt) || stmt[0] !== 'local.set' || stmt.length !== 3) return null
|
|
3129
|
+
const x = stmt[1], v = stmt[2]
|
|
3130
|
+
if (!isArr(v) || v.length !== 3 || !isLocalGet(v[1], x)) return null
|
|
3131
|
+
if (v[0] === 'i32.add' && constNum(v[2]) === 1) return { name: x, type: 'i32' }
|
|
3132
|
+
if (v[0] === 'f64.add' && isArr(v[2]) && v[2][0] === 'f64.const' && Number(v[2][1]) === 1) return { name: x, type: 'f64' }
|
|
3133
|
+
return null
|
|
3134
|
+
}
|
|
3135
|
+
const matchPixelExit = (stmt, label) => {
|
|
3136
|
+
if (!isArr(stmt) || stmt[0] !== 'br_if' || stmt[1] !== label) return null
|
|
3137
|
+
const cond = stmt[2]
|
|
3138
|
+
if (!isArr(cond) || cond[0] !== 'i32.eqz') return null
|
|
3139
|
+
const cmp = cond[1]
|
|
3140
|
+
if (!isArr(cmp) || !isLocalGet(cmp[1])) return null
|
|
3141
|
+
if (cmp[0] === 'i32.lt_s' || cmp[0] === 'i32.lt_u') return { ind: cmp[1][1], bound: cmp[2], cmpOp: cmp[0], type: 'i32' }
|
|
3142
|
+
if (cmp[0] === 'f64.lt') return { ind: cmp[1][1], bound: cmp[2], cmpOp: cmp[0], type: 'f64' }
|
|
3143
|
+
return null
|
|
3144
|
+
}
|
|
3145
|
+
|
|
3146
|
+
/**
|
|
3147
|
+
* Match the OUTER per-pixel loop scaffold shared by tryDivergentEscapeVectorize
|
|
3148
|
+
* (inner escape loop) and tryPerPixelColor (straight-line body):
|
|
3149
|
+
* (block $o [preamble: pure local.set…]
|
|
3150
|
+
* (loop $l (br_if $o (i32.eqz (IV < WIDTH))) OBODY… (pxIV += 1)… (br $l)))
|
|
3151
|
+
* One-or-more trailing `v += 1` are pixel induction vars (the bound IV plus any
|
|
3152
|
+
* parallel counters like `j`); the exit bounds one of them by an invariant width.
|
|
3153
|
+
*
|
|
3154
|
+
* Returns the shared FACTS, or null:
|
|
3155
|
+
* { oLabel, loopNode, preamble, pixelIVs, pivStart, pxVar, widthBound, pivType,
|
|
3156
|
+
* obody } — obody = loopNode.slice(3, pivStart), the per-pixel work between
|
|
3157
|
+
* the exit guard and the IV bumps. Both consumers branch on `obody` afterward.
|
|
3158
|
+
* The bound is re-evaluated for the SIMD guard, so it must be invariant + pure:
|
|
3159
|
+
* a constant, or a local/global the loop nest never writes (`writesName`).
|
|
3160
|
+
*/
|
|
3161
|
+
function matchOuterPixelLoop(blockNode) {
|
|
3162
|
+
if (!isArr(blockNode) || blockNode[0] !== 'block') return null
|
|
3163
|
+
const oLabel = (typeof blockNode[1] === 'string' && blockNode[1].startsWith('$')) ? blockNode[1] : null
|
|
3164
|
+
if (!oLabel) return null
|
|
3165
|
+
let loopNode = null, loopPos = -1
|
|
3166
|
+
for (let i = 2; i < blockNode.length; i++) {
|
|
3167
|
+
const c = blockNode[i]
|
|
3168
|
+
if (!isArr(c)) return null
|
|
3169
|
+
if (c[0] === 'loop') { if (loopNode) return null; loopNode = c; loopPos = i }
|
|
3170
|
+
else if (loopNode) return null // statement after the loop → bail
|
|
3171
|
+
else if (c[0] !== 'local.set') return null // preamble must be pure local.set
|
|
3172
|
+
}
|
|
3173
|
+
if (!loopNode) return null
|
|
3174
|
+
const preamble = blockNode.slice(2, loopPos)
|
|
3175
|
+
const llabel = (typeof loopNode[1] === 'string' && loopNode[1].startsWith('$')) ? loopNode[1] : null
|
|
3176
|
+
if (!llabel) return null
|
|
3177
|
+
const oEnd = loopNode.length - 1
|
|
3178
|
+
if (!(isArr(loopNode[oEnd]) && loopNode[oEnd][0] === 'br' && loopNode[oEnd][1] === llabel)) return null
|
|
3179
|
+
const pixelIVs = [] // [{ name, type }]
|
|
3180
|
+
let pivStart = oEnd
|
|
3181
|
+
for (let i = oEnd - 1; i >= 3; i--) {
|
|
3182
|
+
const m = matchPixelInc(loopNode[i])
|
|
3183
|
+
if (!m) break
|
|
3184
|
+
pixelIVs.unshift(m); pivStart = i
|
|
3185
|
+
}
|
|
3186
|
+
if (!pixelIVs.length) return null
|
|
3187
|
+
const oExit = matchPixelExit(loopNode[2], oLabel)
|
|
3188
|
+
const pxIV = oExit && pixelIVs.find(p => p.name === oExit.ind && p.type === oExit.type)
|
|
3189
|
+
if (!pxIV) return null
|
|
3190
|
+
const widthBound = oExit.bound
|
|
3191
|
+
const pivType = new Map(pixelIVs.map(p => [p.name, p.type]))
|
|
3192
|
+
if (isI32Const(widthBound)) { /* ok */ }
|
|
3193
|
+
else if (isArr(widthBound) && (widthBound[0] === 'local.get' || widthBound[0] === 'global.get')) {
|
|
3194
|
+
if (writesName(loopNode, widthBound[1])) return null
|
|
3195
|
+
} else return null
|
|
3196
|
+
const obody = loopNode.slice(3, pivStart) // between exit guard and the pixel-IV bumps
|
|
3197
|
+
return { oLabel, loopNode, preamble, pixelIVs, pivStart, pxVar: oExit.ind, widthBound, pivType, obody, oExit }
|
|
3198
|
+
}
|
|
3199
|
+
|
|
3200
|
+
function tryDivergentEscapeVectorize(blockNode, fnLocals, freshIdRef) {
|
|
3201
|
+
// Outer per-pixel scaffold (+ LICM preamble feeding both SIMD path and tail).
|
|
3202
|
+
const outer = matchOuterPixelLoop(blockNode)
|
|
3203
|
+
if (!outer) return null
|
|
3204
|
+
const { oLabel, loopNode, preamble, pixelIVs, pxVar, widthBound, pivType, obody, oExit } = outer
|
|
3205
|
+
|
|
3206
|
+
let innerIdx = -1, innerBlock = null
|
|
3207
|
+
for (let i = 0; i < obody.length; i++) {
|
|
3208
|
+
const s = obody[i]
|
|
3209
|
+
if (isArr(s) && s[0] === 'block' && s.slice(1).some(c => isArr(c) && c[0] === 'loop')) {
|
|
3210
|
+
if (innerBlock) return null
|
|
3211
|
+
innerBlock = s; innerIdx = i
|
|
3212
|
+
}
|
|
3213
|
+
}
|
|
3214
|
+
if (!innerBlock) return null
|
|
3215
|
+
const iLabel = (typeof innerBlock[1] === 'string' && innerBlock[1].startsWith('$')) ? innerBlock[1] : null
|
|
3216
|
+
if (!iLabel || innerBlock.length < 3) return null
|
|
3217
|
+
const innerLoop = innerBlock[innerBlock.length - 1]
|
|
3218
|
+
if (!isArr(innerLoop) || innerLoop[0] !== 'loop') return null
|
|
3219
|
+
const ilLabel = (typeof innerLoop[1] === 'string' && innerLoop[1].startsWith('$')) ? innerLoop[1] : null
|
|
3220
|
+
if (!ilLabel) return null
|
|
3221
|
+
// The inner block may also carry LICM-hoisted invariants (e.g. BAILOUT) before the loop.
|
|
3222
|
+
// They must be pixel-pair-invariant (read only outer-invariants) so one copy feeds both
|
|
3223
|
+
// lanes; hoist them ahead of the SIMD loop.
|
|
3224
|
+
const innerPre = innerBlock.slice(2, innerBlock.length - 1)
|
|
3225
|
+
for (const s of innerPre) {
|
|
3226
|
+
if (!isArr(s) || s[0] !== 'local.set' || s.length !== 3) return null
|
|
3227
|
+
const inv = (n) => !isArr(n) || ((n[0] === 'local.get' || n[0] === 'global.get') ? !writesName(loopNode, n[1]) : n.every((c, i) => i === 0 || inv(c)))
|
|
3228
|
+
if (!inv(s[2])) return null
|
|
3229
|
+
}
|
|
3230
|
+
|
|
3231
|
+
// ---- inner escape loop: a top while-cond + body (f64 updates + one mid-break) + it++.
|
|
3232
|
+
// The two continue/break conditions are an `it < MAXIT` limit and a `|z|² vs T` escape,
|
|
3233
|
+
// in EITHER order (mandelbrot/ship: limit at top; example-mandelbrot: escape at top). ----
|
|
3234
|
+
const iEnd = innerLoop.length - 1
|
|
3235
|
+
if (!(isArr(innerLoop[iEnd]) && innerLoop[iEnd][0] === 'br' && innerLoop[iEnd][1] === ilLabel)) return null
|
|
3236
|
+
const itVar = matchInc1(innerLoop[iEnd - 1])
|
|
3237
|
+
if (!itVar || fnLocals.get(itVar) !== 'i32') return null
|
|
3238
|
+
|
|
3239
|
+
// A break statement (br_if, or `if BC (then (br iLabel))`) → its break-when condition.
|
|
3240
|
+
// breakCond: does this statement break the inner loop? Returns { cond, assigns } where
|
|
3241
|
+
// `assigns` is a list of { name, val } local.set stmts extracted from the then-block
|
|
3242
|
+
// BEFORE the final br. Existing single-br form → assigns=[]. Multi-stmt then form (Newton:
|
|
3243
|
+
// `if(cond)(then (local.set $root K)(br $iLabel))`) → assigns=[{name,val}].
|
|
3244
|
+
const breakCond = (s) => {
|
|
3245
|
+
if (!isArr(s)) return null
|
|
3246
|
+
if (s[0] === 'br_if' && s[1] === iLabel) return { cond: s[2], assigns: [] }
|
|
3247
|
+
if (s[0] !== 'if' || s.length !== 3 || !isArr(s[2]) || s[2][0] !== 'then') return null
|
|
3248
|
+
const then = s[2]
|
|
3249
|
+
// then-block must end with (br $iLabel); all preceding stmts must be local.set
|
|
3250
|
+
if (!isArr(then[then.length - 1]) || then[then.length - 1][0] !== 'br' || then[then.length - 1][1] !== iLabel) return null
|
|
3251
|
+
const assigns = []
|
|
3252
|
+
for (let k = 1; k < then.length - 1; k++) {
|
|
3253
|
+
const st = then[k]
|
|
3254
|
+
if (!isArr(st) || st[0] !== 'local.set' || st.length !== 3) return null
|
|
3255
|
+
assigns.push({ name: st[1], val: st[2] })
|
|
3256
|
+
}
|
|
3257
|
+
return { cond: s[1], assigns }
|
|
3258
|
+
}
|
|
3259
|
+
// keep (continue) condition vs the break-when condition. A while-guard `(i32.eqz CONT)`
|
|
3260
|
+
// keeps on CONT directly; a mid-break `if (X) break` keeps on ¬X. We must NOT lower ¬X by
|
|
3261
|
+
// flipping the comparison (f64.gt→f64.le): for NaN, scalar `NOT(NaN>T)` is TRUE but
|
|
3262
|
+
// `NaN<=T` is FALSE — they disagree, so an orbit that reaches NaN without escaping would
|
|
3263
|
+
// wrongly deactivate. Instead keep the DIRECT comparison and negate the lane mask with
|
|
3264
|
+
// v128.not (NaN-exact: gt is false on NaN, ¬false = keep, matching scalar).
|
|
3265
|
+
const keepOf = (bc) => {
|
|
3266
|
+
if (!isArr(bc)) return null
|
|
3267
|
+
if (bc[0] === 'i32.eqz' && isArr(bc[1]) && CMP_LANE[bc[1][0]]) return { cmp: bc[1], negate: false }
|
|
3268
|
+
if (CMP_NEG[bc[0]]) return { cmp: bc, negate: true }
|
|
3269
|
+
return null
|
|
3270
|
+
}
|
|
3271
|
+
const topBcR = breakCond(innerLoop[2])
|
|
3272
|
+
const topBc = topBcR?.cond // raw condition expr (backward-compat name for compound-top checks)
|
|
3273
|
+
let keepTop = topBcR && keepOf(topBcR.cond)
|
|
3274
|
+
const ibody = innerLoop.slice(3, iEnd - 1)
|
|
3275
|
+
let midIdx = -1, keepMid = null, compoundTop = false
|
|
3276
|
+
// Compound while-guard `while (it<MAX && |z|²<T)` → `eqz(and(A,B))`: two keep conditions, both
|
|
3277
|
+
// at the top (continue while A AND B). The Julia set is written this way. Split A,B into the two
|
|
3278
|
+
// keeps; the body is then pure updates (no mid-break).
|
|
3279
|
+
if (!keepTop && isArr(topBc) && topBc[0] === 'i32.eqz' && isArr(topBc[1]) && topBc[1][0] === 'i32.and'
|
|
3280
|
+
&& isArr(topBc[1][1]) && CMP_LANE[topBc[1][1][0]] && isArr(topBc[1][2]) && CMP_LANE[topBc[1][2][0]]) {
|
|
3281
|
+
keepTop = { cmp: topBc[1][1], negate: false }
|
|
3282
|
+
keepMid = { cmp: topBc[1][2], negate: false }
|
|
3283
|
+
compoundTop = true
|
|
3284
|
+
}
|
|
3285
|
+
if (!keepTop) return null
|
|
3286
|
+
// Collect mid-breaks. The existing single-break path requires exactly one.
|
|
3287
|
+
// The new multi-break path (Newton-style convergence) allows several outcome breaks
|
|
3288
|
+
// — each `if(COND)(then (local.set $X K)(br $iLabel))` — when the top guard is the
|
|
3289
|
+
// sole limit condition and ALL mid-breaks are escape-kind with i32 outcome assigns.
|
|
3290
|
+
const midBreaks = [] // [{ idx, keep, assigns }]
|
|
3291
|
+
if (compoundTop) {
|
|
3292
|
+
for (let i = 0; i < ibody.length; i++)
|
|
3293
|
+
if (!(isArr(ibody[i]) && ibody[i][0] === 'local.set' && ibody[i].length === 3 && fnLocals.get(ibody[i][1]) === 'f64')) return null
|
|
3294
|
+
} else {
|
|
3295
|
+
for (let i = 0; i < ibody.length; i++) {
|
|
3296
|
+
const bcR = breakCond(ibody[i])
|
|
3297
|
+
if (bcR) {
|
|
3298
|
+
const k = keepOf(bcR.cond)
|
|
3299
|
+
if (!k) return null
|
|
3300
|
+
midBreaks.push({ idx: i, keep: k, assigns: bcR.assigns })
|
|
3301
|
+
} else if (!(isArr(ibody[i]) && ibody[i][0] === 'local.set' && ibody[i].length === 3 && fnLocals.get(ibody[i][1]) === 'f64')) return null
|
|
3302
|
+
}
|
|
3303
|
+
if (midBreaks.length === 0) return null
|
|
3304
|
+
if (midBreaks.length === 1) {
|
|
3305
|
+
// single-break: preserve exact existing behaviour
|
|
3306
|
+
midIdx = midBreaks[0].idx; keepMid = midBreaks[0].keep
|
|
3307
|
+
}
|
|
3308
|
+
// multi-break: handled below after classification
|
|
3309
|
+
}
|
|
3310
|
+
|
|
3311
|
+
// classify f64 vars written across [top-guard, …body…]: loop-carried (first access a READ)
|
|
3312
|
+
// vs within-iteration temp (first access a WRITE — including squares tee'd in the guard).
|
|
3313
|
+
const seq = [innerLoop[2], ...ibody]
|
|
3314
|
+
const written = new Set()
|
|
3315
|
+
const collectW = (n) => { if (!isArr(n)) return; if ((n[0] === 'local.set' || n[0] === 'local.tee') && fnLocals.get(n[1]) === 'f64') written.add(n[1]); for (let i = 1; i < n.length; i++) collectW(n[i]) }
|
|
3316
|
+
seq.forEach(collectW)
|
|
3317
|
+
const carried = new Set(), temp = new Set(), seen = new Set()
|
|
3318
|
+
const classify = (n) => {
|
|
3319
|
+
if (!isArr(n)) return
|
|
3320
|
+
if (n[0] === 'local.get') { const v = n[1]; if (written.has(v) && !seen.has(v)) { seen.add(v); carried.add(v) }; return }
|
|
3321
|
+
if (n[0] === 'local.set' || n[0] === 'local.tee') { classify(n[2]); const v = n[1]; if (written.has(v) && !seen.has(v)) { seen.add(v); temp.add(v) }; return }
|
|
3322
|
+
for (let i = 1; i < n.length; i++) classify(n[i])
|
|
3323
|
+
}
|
|
3324
|
+
seq.forEach(classify)
|
|
3325
|
+
|
|
3326
|
+
// hoist tees out of the keep conditions into explicit temp computations
|
|
3327
|
+
const tees = []
|
|
3328
|
+
const detee = (n) => isArr(n) ? (n[0] === 'local.tee' ? (tees.push({ tgt: n[1], expr: detee(n[2]) }), ['local.get', n[1]]) : n.map(detee)) : n
|
|
3329
|
+
const keepTopC = { cmp: [keepTop.cmp[0], detee(keepTop.cmp[1]), detee(keepTop.cmp[2])], negate: keepTop.negate }
|
|
3330
|
+
const teeTop = tees.length
|
|
3331
|
+
const keepMidC = keepMid ? { cmp: [keepMid.cmp[0], detee(keepMid.cmp[1]), detee(keepMid.cmp[2])], negate: keepMid.negate } : null
|
|
3332
|
+
|
|
3333
|
+
// each keep is an it-limit (mentions `it` directly or via convert) or a z-escape; need one of each
|
|
3334
|
+
const isIt = (n) => isLocalGet(n, itVar) || (isArr(n) && n[0] === 'f64.convert_i32_s' && isLocalGet(n[1], itVar))
|
|
3335
|
+
const kindOf = (k) => (isIt(k.cmp[1]) || isIt(k.cmp[2])) ? 'limit' : 'escape'
|
|
3336
|
+
const kTop = kindOf(keepTopC)
|
|
3337
|
+
const limitKeep = keepTopC // initialised here; may be reassigned below for single-break
|
|
3338
|
+
let kMid = null, boundExpr, boundI32, itLeft
|
|
3339
|
+
if (midBreaks.length === 1) {
|
|
3340
|
+
kMid = kindOf(keepMidC)
|
|
3341
|
+
if (kTop === kMid) return null
|
|
3342
|
+
const lk = kTop === 'limit' ? keepTopC : keepMidC
|
|
3343
|
+
itLeft = isIt(lk.cmp[1])
|
|
3344
|
+
boundExpr = itLeft ? lk.cmp[2] : lk.cmp[1]
|
|
3345
|
+
boundI32 = lk.cmp[0].startsWith('i32.')
|
|
3346
|
+
} else if (midBreaks.length > 1) {
|
|
3347
|
+
// Multi-break path: top must be the sole limit; all mid-breaks must be escape-kind.
|
|
3348
|
+
if (kTop !== 'limit') return null
|
|
3349
|
+
for (const mb of midBreaks) if (kindOf(mb.keep) !== 'escape') return null
|
|
3350
|
+
itLeft = isIt(keepTopC.cmp[1])
|
|
3351
|
+
boundExpr = itLeft ? keepTopC.cmp[2] : keepTopC.cmp[1]
|
|
3352
|
+
boundI32 = keepTopC.cmp[0].startsWith('i32.')
|
|
3353
|
+
}
|
|
3354
|
+
// limit bound must be loop-invariant (splatted once per pair): no carried/temp ref, not written
|
|
3355
|
+
for (const v of [...carried, ...temp]) if (readsVar(boundExpr, v)) return null
|
|
3356
|
+
const boundInvariant = (n) => !isArr(n) || ((n[0] !== 'local.get' && n[0] !== 'global.get') || !writesName(loopNode, n[1])) && n.every((c, i) => i === 0 || boundInvariant(c))
|
|
3357
|
+
if (!boundInvariant(boundExpr)) return null
|
|
3358
|
+
|
|
3359
|
+
// c-vars: f64 locals read in the lifted exprs but never written there (per-pixel/invariant inputs).
|
|
3360
|
+
// Also allow loop-invariant global.get (e.g. module constants like R3 in newton) — splatted inline.
|
|
3361
|
+
const cVars = new Set()
|
|
3362
|
+
const liftable = (n) => {
|
|
3363
|
+
if (!isArr(n)) return false
|
|
3364
|
+
if (n[0] === 'local.get') {
|
|
3365
|
+
const v = n[1]
|
|
3366
|
+
if (v !== itVar && !carried.has(v) && !temp.has(v)) { if (fnLocals.get(v) !== 'f64') return false; cVars.add(v) }
|
|
3367
|
+
return true
|
|
3368
|
+
}
|
|
3369
|
+
if (n[0] === 'f64.const') return true
|
|
3370
|
+
if (n[0] === 'global.get') return !writesName(loopNode, n[1]) // loop-invariant global → splat
|
|
3371
|
+
if (LANE_PURE.f64.has(n[0])) { for (let i = 1; i < n.length; i++) if (!liftable(n[i])) return false; return true }
|
|
3372
|
+
return false
|
|
3373
|
+
}
|
|
3374
|
+
const midIdxSet = new Set(midBreaks.map(mb => mb.idx))
|
|
3375
|
+
for (const t of tees) if (!liftable(t.expr)) return null
|
|
3376
|
+
for (let i = 0; i < ibody.length; i++) if (!midIdxSet.has(i) && !liftable(ibody[i][2])) return null
|
|
3377
|
+
if (midBreaks.length === 1) {
|
|
3378
|
+
const escapeKeep = kTop === 'escape' ? keepTopC : keepMidC
|
|
3379
|
+
if (!liftable(escapeKeep.cmp[1]) || !liftable(escapeKeep.cmp[2])) return null
|
|
3380
|
+
} else {
|
|
3381
|
+
// multi-break: each break's escape condition arguments must be liftable
|
|
3382
|
+
for (const mb of midBreaks) {
|
|
3383
|
+
const mbC = { cmp: [mb.keep.cmp[0], detee(mb.keep.cmp[1]), detee(mb.keep.cmp[2])], negate: mb.keep.negate }
|
|
3384
|
+
mb.keepC = mbC // store deteed keepC on mb for use in emit
|
|
3385
|
+
if (!liftable(mbC.cmp[1]) || !liftable(mbC.cmp[2])) return null
|
|
3386
|
+
}
|
|
3387
|
+
}
|
|
3388
|
+
|
|
3389
|
+
// ---- pre-inner inits + epilogue ----
|
|
3390
|
+
// i32 outcome variables assigned by multi-break outcome stmts (e.g. $root in Newton).
|
|
3391
|
+
// Their pre-inner `local.set $root 0` default-init is valid and must not be rejected.
|
|
3392
|
+
const outcomeVarSetEarly = new Set(midBreaks.flatMap(mb => mb.assigns.map(a => a.name)))
|
|
3393
|
+
const carriedInit = new Map() // carried var → its f64.const seed (before the loop)
|
|
3394
|
+
const perPxInit = new Map() // c-var → its per-pixel init expr
|
|
3395
|
+
for (let i = 0; i < innerIdx; i++) {
|
|
3396
|
+
const s = obody[i]
|
|
3397
|
+
if (!isArr(s) || s[0] !== 'local.set' || s.length !== 3) return null
|
|
3398
|
+
const tgt = s[1]
|
|
3399
|
+
if (carried.has(tgt)) {
|
|
3400
|
+
// z₀ seed: a constant (mandelbrot/burning-ship z₀=0) OR a per-pixel expr (Julia set, where
|
|
3401
|
+
// z₀ = the pixel and c is constant — the dual of mandelbrot). liftCLane handles both at emit.
|
|
3402
|
+
carriedInit.set(tgt, s[2])
|
|
3403
|
+
} else if (temp.has(tgt)) { /* recomputed each iteration — init ignored */ }
|
|
3404
|
+
else if (tgt === itVar) { if (constNum(s[2]) !== 0) return null }
|
|
3405
|
+
else if (cVars.has(tgt)) perPxInit.set(tgt, s[2])
|
|
3406
|
+
else if (outcomeVarSetEarly.has(tgt)) { /* i32 outcome var default init — ignored, handled per-lane */ }
|
|
3407
|
+
else return null
|
|
3408
|
+
}
|
|
3409
|
+
for (const c of carried) if (!carriedInit.has(c)) return null
|
|
3410
|
+
const epilogue = obody.slice(innerIdx + 1)
|
|
3411
|
+
// The epilogue runs scalar per lane; it may only read carried/it/pixel-IV/invariant
|
|
3412
|
+
// values (each statement's reads, before that statement's writes). A read of an
|
|
3413
|
+
// inner-loop temp or a per-pixel c-var has no post-loop per-lane value → bail.
|
|
3414
|
+
{
|
|
3415
|
+
const written = new Set()
|
|
3416
|
+
for (const s of epilogue) {
|
|
3417
|
+
const r = new Set(); (function rd(n){ if(!isArr(n)) return; if(n[0]==='local.get') r.add(n[1]); else for(const c of n) rd(c) })(s)
|
|
3418
|
+
for (const v of r) if (!written.has(v) && (temp.has(v) || perPxInit.has(v))) return null
|
|
3419
|
+
;(function wr(n){ if(!isArr(n)) return; if((n[0]==='local.set'||n[0]==='local.tee')&&typeof n[1]==='string') written.add(n[1]); for(const c of n.slice(2)) wr(c) })(s)
|
|
3420
|
+
}
|
|
3421
|
+
if (!epilogue.length) return null
|
|
3422
|
+
}
|
|
3423
|
+
|
|
3424
|
+
// ============================ emit ============================
|
|
3425
|
+
const id = freshIdRef.next++
|
|
3426
|
+
const nm = (s) => `$__esc${id}_${s}`
|
|
3427
|
+
const shadow = new Map() // carried/temp f64 var → v128 shadow
|
|
3428
|
+
for (const v of [...carried, ...temp]) shadow.set(v, nm('z' + v.replace(/\W/g, '')))
|
|
3429
|
+
const cLane = new Map()
|
|
3430
|
+
for (const cv of cVars) cLane.set(cv, nm('c' + cv.replace(/\W/g, '')))
|
|
3431
|
+
const iterV = nm('iter'), activeV = nm('act'), maxitLane = nm('lim')
|
|
3432
|
+
const newLocalDecls = []
|
|
3433
|
+
for (const n of [...shadow.values(), ...cLane.values()]) newLocalDecls.push(['local', n, 'v128'])
|
|
3434
|
+
|
|
3435
|
+
const lift = (n) => {
|
|
3436
|
+
if (n[0] === 'local.get') return ['local.get', shadow.has(n[1]) ? shadow.get(n[1]) : cLane.get(n[1])]
|
|
3437
|
+
if (n[0] === 'f64.const') return ['f64x2.splat', n]
|
|
3438
|
+
if (n[0] === 'global.get') return ['f64x2.splat', n] // loop-invariant module global (e.g. R3)
|
|
3439
|
+
return [LANE_PURE.f64.get(n[0]).simd, ...n.slice(1).map(lift)]
|
|
3440
|
+
}
|
|
3441
|
+
const bump = (n, k) => k === 0 ? n // substitute every pixel-IV with (IV + k), in its type
|
|
3442
|
+
: (isArr(n) && n[0] === 'local.get' && pivType.has(n[1]))
|
|
3443
|
+
? [pivType.get(n[1]) + '.add', n, [pivType.get(n[1]) + '.const', k]]
|
|
3444
|
+
: (isArr(n) ? n.map(c => bump(c, k)) : n)
|
|
3445
|
+
|
|
3446
|
+
// Build a per-pixel c-var's two lanes by lifting its init to f64x2: the pixel IV becomes the
|
|
3447
|
+
// ramp [v, v+1], a dependency on another c-var resolves to that c-var's already-built lane
|
|
3448
|
+
// (so chains like `bail = 4 + cx*cx` get the right per-lane value — NOT the px=0 value from a
|
|
3449
|
+
// bump that can't follow `cx`), and everything else (grid constants) splats. Returns null if
|
|
3450
|
+
// the init reads inner-loop state or an op we can't lift, in which case we decline.
|
|
3451
|
+
const liftCLane = (n) => {
|
|
3452
|
+
if (!isArr(n)) return null
|
|
3453
|
+
if (n[0] === 'local.get' || n[0] === 'global.get') {
|
|
3454
|
+
const v = n[1]
|
|
3455
|
+
if (n[0] === 'local.get') {
|
|
3456
|
+
if (cLane.has(v)) return ['local.get', cLane.get(v)] // another c-var's lane
|
|
3457
|
+
if (carried.has(v) || temp.has(v)) return null // inner-loop state — must not appear
|
|
3458
|
+
if (pivType.get(v) === 'f64') return ['f64x2.replace_lane', 1, ['f64x2.splat', n], ['f64.add', n, ['f64.const', 1]]]
|
|
3459
|
+
}
|
|
3460
|
+
if (writesName(loopNode, v)) return null // a per-pixel value we can't ramp → decline
|
|
3461
|
+
return ['f64x2.splat', n] // loop-invariant local/global
|
|
3462
|
+
}
|
|
3463
|
+
if (n[0] === 'f64.const') return ['f64x2.splat', n]
|
|
3464
|
+
if (n[0] === 'f64.convert_i32_s' && isArr(n[1]) && n[1][0] === 'local.get' && pivType.get(n[1][1]) === 'i32') {
|
|
3465
|
+
const piv = n[1]
|
|
3466
|
+
return ['f64x2.replace_lane', 1, ['f64x2.splat', ['f64.convert_i32_s', piv]], ['f64.convert_i32_s', ['i32.add', piv, ['i32.const', 1]]]]
|
|
3467
|
+
}
|
|
3468
|
+
if (LANE_PURE.f64.has(n[0])) { const ks = n.slice(1).map(liftCLane); return ks.some(k => k === null) ? null : [LANE_PURE.f64.get(n[0]).simd, ...ks] }
|
|
3469
|
+
return null
|
|
3470
|
+
}
|
|
3471
|
+
// c-lanes in dependency order: invariants splat first, then per-pixel inits in source (obody) order
|
|
3472
|
+
const cOrder = [...cVars].filter(cv => !perPxInit.has(cv))
|
|
3473
|
+
for (let i = 0; i < innerIdx; i++) { const s = obody[i]; if (isArr(s) && s[0] === 'local.set' && perPxInit.has(s[1])) cOrder.push(s[1]) }
|
|
3474
|
+
const pre = []
|
|
3475
|
+
for (const cv of cOrder) {
|
|
3476
|
+
if (perPxInit.has(cv)) {
|
|
3477
|
+
const lane = liftCLane(perPxInit.get(cv))
|
|
3478
|
+
if (!lane) return null
|
|
3479
|
+
pre.push(['local.set', cLane.get(cv), lane])
|
|
3480
|
+
} else pre.push(['local.set', cLane.get(cv), ['f64x2.splat', ['local.get', cv]]])
|
|
3481
|
+
}
|
|
3482
|
+
// Seed each carried lane in dependency order. Seeds reading only non-carried values (z₀ = a
|
|
3483
|
+
// const, or a per-pixel ramp for the Julia set) build their lanes via liftCLane. Seeds reading
|
|
3484
|
+
// a carried var — the cached squares `zx2 = zx*zx` in Julia's compound guard — lift through the
|
|
3485
|
+
// already-built shadow lanes instead.
|
|
3486
|
+
const seedDeps = (e) => [...carried].some(c => readsVar(e, c))
|
|
3487
|
+
const depLiftable = (n) => !isArr(n) ? false
|
|
3488
|
+
: n[0] === 'local.get' ? shadow.has(n[1])
|
|
3489
|
+
: n[0] === 'f64.const' ? true
|
|
3490
|
+
: (LANE_PURE.f64.has(n[0]) && n.slice(1).every(depLiftable))
|
|
3491
|
+
for (const v of carried) if (!seedDeps(carriedInit.get(v))) {
|
|
3492
|
+
const lane = liftCLane(carriedInit.get(v))
|
|
3493
|
+
if (!lane) return null
|
|
3494
|
+
pre.push(['local.set', shadow.get(v), lane])
|
|
3495
|
+
}
|
|
3496
|
+
for (const v of carried) if (seedDeps(carriedInit.get(v))) {
|
|
3497
|
+
if (!depLiftable(carriedInit.get(v))) return null
|
|
3498
|
+
pre.push(['local.set', shadow.get(v), lift(carriedInit.get(v))])
|
|
3499
|
+
}
|
|
3500
|
+
const teeStmts = (range) => range.map(t => ['local.set', shadow.get(t.tgt), lift(t.expr)])
|
|
3501
|
+
const sIn = nm('ib'), sIl = nm('il'), sOut = nm('ob'), sOl = nm('ol')
|
|
3502
|
+
|
|
3503
|
+
// FAST PATH — the common escape-time shape `while (it<MAX){ …updates…; if (|z|²>T) break }`:
|
|
3504
|
+
// break the pair the instant the FIRST lane escapes (no per-iteration freeze/mask), keep `it`
|
|
3505
|
+
// a scalar i32, raw f64x2 updates. A short scalar tail then finishes whichever lane lagged (≈0
|
|
3506
|
+
// iterations when the pair is coherent, which adjacent pixels overwhelmingly are). Inside-set
|
|
3507
|
+
// pixels (both lanes to MAX) run clean 2× with zero mask overhead — exactly where the masked
|
|
3508
|
+
// loop bled its speedup. Other shapes (escape-at-top, body after the break) take the masked path.
|
|
3509
|
+
// escape-at-MID: `…updates…; if (|z|²>T) break` (burning-ship). escape-at-TOP: the escape gates
|
|
3510
|
+
// the loop head — a `while (|z|²<T)` (mandelbrot) or the second half of a compound `while (it<MAX
|
|
3511
|
+
// && |z|²<T)` (Julia). Both take the fast path; only the tail's `it` resume point differs.
|
|
3512
|
+
const escAtMid = !compoundTop && kMid === 'escape' && midIdx === ibody.length - 1
|
|
3513
|
+
const escAtTop = compoundTop || kTop === 'escape'
|
|
3514
|
+
const fastPath = escAtMid || escAtTop
|
|
3515
|
+
let simdInner, epiLane, postLoop = []
|
|
3516
|
+
|
|
3517
|
+
if (fastPath) {
|
|
3518
|
+
const shIt = nm('shit'), escF = nm('escf')
|
|
3519
|
+
newLocalDecls.push(['local', shIt, 'i32'], ['local', escF, 'i32'])
|
|
3520
|
+
pre.push(['local.set', itVar, ['i32.const', 0]], ['local.set', escF, ['i32.const', 0]])
|
|
3521
|
+
const limOf = (keepC) => keepC.negate ? [CMP_NEG[keepC.cmp[0]], keepC.cmp[1], keepC.cmp[2]] : keepC.cmp
|
|
3522
|
+
// emit a keep at its source position: a LIMIT (it is shared) → a scalar i32 guard; an ESCAPE
|
|
3523
|
+
// → an any_true break. `escF` records whether the loop exited via an escape (vs the limit) —
|
|
3524
|
+
// they can BOTH land at it=MAX (escape-at-top fires before the limit-at-mid's final update),
|
|
3525
|
+
// so `it < MAX` alone can't tell them apart; the tail must run only on an escape exit.
|
|
3526
|
+
const fastKeep = (keepC, kind, teeRange) => {
|
|
3527
|
+
const out = teeStmts(teeRange)
|
|
3528
|
+
if (kind === 'limit') out.push(['br_if', sIn, ['i32.eqz', limOf(keepC)]])
|
|
3529
|
+
else {
|
|
3530
|
+
const escL = [CMP_LANE[keepC.cmp[0]], lift(keepC.cmp[1]), lift(keepC.cmp[2])]
|
|
3531
|
+
out.push(['local.set', escF, ['i32.const', 1]])
|
|
3532
|
+
out.push(['br_if', sIn, ['v128.any_true', keepC.negate ? escL : ['v128.not', escL]]])
|
|
3533
|
+
out.push(['local.set', escF, ['i32.const', 0]])
|
|
3534
|
+
}
|
|
3535
|
+
return out
|
|
3536
|
+
}
|
|
3537
|
+
const sbody = [...fastKeep(keepTopC, kTop, tees.slice(0, teeTop))]
|
|
3538
|
+
if (compoundTop) sbody.push(...fastKeep(keepMidC, kMid, tees.slice(teeTop))) // second top keep
|
|
3539
|
+
for (let i = 0; i < ibody.length; i++) {
|
|
3540
|
+
if (!compoundTop && i === midIdx) sbody.push(...fastKeep(keepMidC, kMid, tees.slice(teeTop)))
|
|
3541
|
+
else sbody.push(['local.set', shadow.get(ibody[i][1]), lift(ibody[i][2])]) // raw update, no freeze
|
|
3542
|
+
}
|
|
3543
|
+
sbody.push(['local.set', itVar, ['i32.add', ['local.get', itVar], ['i32.const', 1]]])
|
|
3544
|
+
simdInner = ['block', sIn, ['loop', sIl, ...sbody, ['br', sIl]]]
|
|
3545
|
+
postLoop = [['local.set', shIt, ['local.get', itVar]]] // capture the break `it` AFTER the block exits
|
|
3546
|
+
|
|
3547
|
+
// a fresh-labelled copy of the inner block, to finish a lagging lane's remaining iterations
|
|
3548
|
+
let copyN = 0
|
|
3549
|
+
const relabelInner = () => {
|
|
3550
|
+
const ni = nm('tb' + copyN), nl = nm('tl' + copyN); copyN++
|
|
3551
|
+
const rl = (n) => !isArr(n) ? n
|
|
3552
|
+
: ((n[0] === 'block' || n[0] === 'loop' || n[0] === 'br' || n[0] === 'br_if') && (n[1] === iLabel || n[1] === ilLabel))
|
|
3553
|
+
? [n[0], n[1] === iLabel ? ni : nl, ...n.slice(2).map(rl)]
|
|
3554
|
+
: n.map(rl)
|
|
3555
|
+
return rl(innerBlock)
|
|
3556
|
+
}
|
|
3557
|
+
const escKeepC = compoundTop ? keepMidC : (kTop === 'escape' ? keepTopC : keepMidC)
|
|
3558
|
+
const escTees = (kTop === 'escape' ? tees.slice(0, teeTop) : tees.slice(teeTop)).map(t => ['local.set', t.tgt, t.expr])
|
|
3559
|
+
const notEsc = escKeepC.negate ? ['i32.eqz', escKeepC.cmp] : escKeepC.cmp
|
|
3560
|
+
epiLane = (k) => {
|
|
3561
|
+
const out = []
|
|
3562
|
+
// Extract carried AND temp lanes: the escape compare may read a temp (the optimizer
|
|
3563
|
+
// copy-propagates `x = xt` so `x*x` becomes `xt*xt`); the skip test below needs it.
|
|
3564
|
+
for (const v of [...carried, ...temp]) out.push(['local.set', v, ['f64x2.extract_lane', k, ['local.get', shadow.get(v)]]])
|
|
3565
|
+
for (let i = 0; i < innerIdx; i++) { const s = obody[i]; if (isArr(s) && s[0] === 'local.set' && perPxInit.has(s[1])) out.push(bump(s, k)) }
|
|
3566
|
+
out.push(['local.set', itVar, ['local.get', shIt]])
|
|
3567
|
+
out.push(['if', ['local.get', escF], ['then', // exited via escape (not the limit)…
|
|
3568
|
+
...escTees,
|
|
3569
|
+
['if', notEsc, ['then', // …and THIS lane hadn't escaped → finish it
|
|
3570
|
+
// escape-at-MID already ran this iteration's update, so step past it before resuming;
|
|
3571
|
+
// escape-at-TOP tests before the update, so resume at the same it.
|
|
3572
|
+
...(escAtMid ? [['local.set', itVar, ['i32.add', ['local.get', itVar], ['i32.const', 1]]]] : []),
|
|
3573
|
+
relabelInner()]]]])
|
|
3574
|
+
for (const s of epilogue) out.push(bump(s, k))
|
|
3575
|
+
return out
|
|
3576
|
+
}
|
|
3577
|
+
} else if (midBreaks.length > 1) {
|
|
3578
|
+
// ---- MULTI-OUTCOME masked path (Newton-style convergence loops) ----
|
|
3579
|
+
// Each mid-break `if(COND)(then (local.set $X K)(br))` converges a subset of lanes per
|
|
3580
|
+
// iteration. We track which lanes are still running in `activeV` and per-lane outcomes
|
|
3581
|
+
// in i32x4 shadow vectors (one per distinct i32 outcome variable). The break mask for
|
|
3582
|
+
// each convergence condition is: v128.and(activeV, breakMaskOf(keepC)).
|
|
3583
|
+
// Bit-exactness: i32x4 layout has 4 lanes of 32 bits. f64x2 has 2 lanes of 64 bits.
|
|
3584
|
+
// For f64 lane k, i32 lane 2*k carries the outcome (lower 32 bits of the f64 lane).
|
|
3585
|
+
// i32x4.splat(K) fills all 4 i32 lanes with K; v128.bitselect selects at bit level, so
|
|
3586
|
+
// for a converged f64 lane (bits 64k…64k+63 = -1 in the mask) both i32 halves get K.
|
|
3587
|
+
// i32x4.extract_lane(2*k, outcomeVec) recovers the scalar outcome per lane. ✓
|
|
3588
|
+
|
|
3589
|
+
// Collect distinct i32 outcome variables across all mid-breaks.
|
|
3590
|
+
const outcomeVars = [] // distinct i32 local names that get assigned in outcome breaks
|
|
3591
|
+
const outcomeVarSet = new Set()
|
|
3592
|
+
for (const mb of midBreaks) {
|
|
3593
|
+
for (const a of mb.assigns) {
|
|
3594
|
+
// itVar is tracked by iterV (f64x2) directly — not as an i32x4 outcome shadow.
|
|
3595
|
+
if (a.name !== itVar && !outcomeVarSet.has(a.name)) { outcomeVarSet.add(a.name); outcomeVars.push(a.name) }
|
|
3596
|
+
}
|
|
3597
|
+
}
|
|
3598
|
+
// Each outcome var gets an i32x4 shadow vector (initial value 0 = "no convergence" default).
|
|
3599
|
+
const outcomeVec = new Map()
|
|
3600
|
+
for (const v of outcomeVars) { const sv = nm('out' + v.replace(/\W/g, '')); outcomeVec.set(v, sv); newLocalDecls.push(['local', sv, 'v128']) }
|
|
3601
|
+
|
|
3602
|
+
newLocalDecls.push(['local', iterV, 'v128'], ['local', activeV, 'v128'], ['local', maxitLane, 'v128'])
|
|
3603
|
+
pre.push(['local.set', iterV, ['f64x2.splat', ['f64.const', 0]]])
|
|
3604
|
+
pre.push(['local.set', activeV, ['v128.const', 'i32x4', '-1', '-1', '-1', '-1']])
|
|
3605
|
+
pre.push(['local.set', maxitLane, ['f64x2.splat', boundI32 ? ['f64.convert_i32_s', boundExpr] : boundExpr]])
|
|
3606
|
+
for (const sv of outcomeVec.values()) pre.push(['local.set', sv, ['v128.const', 'i32x4', '0', '0', '0', '0']])
|
|
3607
|
+
|
|
3608
|
+
// keepMask for the limit top-guard (iter < MAXIT)
|
|
3609
|
+
const keepMask = (keep, isLimit) => {
|
|
3610
|
+
const m = isLimit
|
|
3611
|
+
? [CMP_LANE[keep.cmp[0]], itLeft ? ['local.get', iterV] : ['local.get', maxitLane], itLeft ? ['local.get', maxitLane] : ['local.get', iterV]]
|
|
3612
|
+
: [CMP_LANE[keep.cmp[0]], lift(keep.cmp[1]), lift(keep.cmp[2])]
|
|
3613
|
+
return keep.negate ? ['v128.not', m] : m
|
|
3614
|
+
}
|
|
3615
|
+
// breakMaskOf: v128 where bits = -1 for lanes whose break condition is NOW true
|
|
3616
|
+
// (opposite of keepMask: keep.negate=true → break is the positive compare).
|
|
3617
|
+
const breakMaskOf = (keepC) => {
|
|
3618
|
+
const m = [CMP_LANE[keepC.cmp[0]], lift(keepC.cmp[1]), lift(keepC.cmp[2])]
|
|
3619
|
+
return keepC.negate ? m : ['v128.not', m]
|
|
3620
|
+
}
|
|
3621
|
+
|
|
3622
|
+
const sbody = [
|
|
3623
|
+
...teeStmts(tees.slice(0, teeTop)),
|
|
3624
|
+
['local.set', activeV, ['v128.and', ['local.get', activeV], keepMask(keepTopC, true)]],
|
|
3625
|
+
['br_if', sIn, ['i32.eqz', ['v128.any_true', ['local.get', activeV]]]],
|
|
3626
|
+
]
|
|
3627
|
+
for (let i = 0; i < ibody.length; i++) {
|
|
3628
|
+
const mb = midBreaks.find(m => m.idx === i)
|
|
3629
|
+
if (mb) {
|
|
3630
|
+
// Convergence break: compute which lanes converge on THIS step.
|
|
3631
|
+
const bm = breakMaskOf(mb.keepC)
|
|
3632
|
+
const convV = nm('conv' + i)
|
|
3633
|
+
newLocalDecls.push(['local', convV, 'v128'])
|
|
3634
|
+
sbody.push(['local.set', convV, ['v128.and', ['local.get', activeV], bm]])
|
|
3635
|
+
// Apply each outcome assignment via bitselect on converged-this-step mask.
|
|
3636
|
+
for (const a of mb.assigns) {
|
|
3637
|
+
if (a.name === itVar) {
|
|
3638
|
+
// Assignment to the iteration counter (e.g. it=MAXIT before break): freeze iterV.
|
|
3639
|
+
sbody.push(['local.set', iterV, ['v128.bitselect', ['local.get', maxitLane], ['local.get', iterV], ['local.get', convV]]])
|
|
3640
|
+
} else if (outcomeVec.has(a.name)) {
|
|
3641
|
+
// i32 outcome variable (e.g. root=1/2/3): bitselect into its i32x4 shadow.
|
|
3642
|
+
// The scalar value K must be an i32 constant — splat it across all i32x4 lanes.
|
|
3643
|
+
if (!isArr(a.val) || a.val[0] !== 'i32.const') return null
|
|
3644
|
+
const sv = outcomeVec.get(a.name)
|
|
3645
|
+
sbody.push(['local.set', sv, ['v128.bitselect', ['i32x4.splat', a.val], ['local.get', sv], ['local.get', convV]]])
|
|
3646
|
+
} else return null // unexpected assign target
|
|
3647
|
+
}
|
|
3648
|
+
// Remove newly-converged lanes from active set.
|
|
3649
|
+
sbody.push(['local.set', activeV, ['v128.andnot', ['local.get', activeV], ['local.get', convV]]])
|
|
3650
|
+
} else {
|
|
3651
|
+
const v = ibody[i][1]
|
|
3652
|
+
if (carried.has(v)) sbody.push(['local.set', shadow.get(v), ['v128.bitselect', lift(ibody[i][2]), ['local.get', shadow.get(v)], ['local.get', activeV]]])
|
|
3653
|
+
else sbody.push(['local.set', shadow.get(v), lift(ibody[i][2])])
|
|
3654
|
+
}
|
|
3655
|
+
}
|
|
3656
|
+
sbody.push(['local.set', iterV, ['v128.bitselect', ['f64x2.add', ['local.get', iterV], ['f64x2.splat', ['f64.const', 1]]], ['local.get', iterV], ['local.get', activeV]]])
|
|
3657
|
+
simdInner = ['block', sIn, ['loop', sIl, ...sbody, ['br', sIl]]]
|
|
3658
|
+
const carriedInEpi = [...carried].filter(v => epilogue.some(s => readsVar(s, v)))
|
|
3659
|
+
epiLane = (k) => {
|
|
3660
|
+
const out = []
|
|
3661
|
+
for (const v of carriedInEpi) out.push(['local.set', v, ['f64x2.extract_lane', k, ['local.get', shadow.get(v)]]])
|
|
3662
|
+
out.push(['local.set', itVar, ['i32.trunc_f64_s', ['f64x2.extract_lane', k, ['local.get', iterV]]]])
|
|
3663
|
+
// Extract per-lane outcome variables from their i32x4 shadows (i32 lane = 2*k).
|
|
3664
|
+
for (const [v, sv] of outcomeVec) out.push(['local.set', v, ['i32x4.extract_lane', 2 * k, ['local.get', sv]]])
|
|
3665
|
+
for (const s of epilogue) out.push(bump(s, k))
|
|
3666
|
+
return out
|
|
3667
|
+
}
|
|
3668
|
+
} else {
|
|
3669
|
+
newLocalDecls.push(['local', iterV, 'v128'], ['local', activeV, 'v128'], ['local', maxitLane, 'v128'])
|
|
3670
|
+
pre.push(['local.set', iterV, ['f64x2.splat', ['f64.const', 0]]])
|
|
3671
|
+
pre.push(['local.set', activeV, ['v128.const', 'i32x4', '-1', '-1', '-1', '-1']])
|
|
3672
|
+
pre.push(['local.set', maxitLane, ['f64x2.splat', boundI32 ? ['f64.convert_i32_s', boundExpr] : boundExpr]])
|
|
3673
|
+
// AND a keep into the active mask: limit compares iter (f64x2) to the bound; escape lifts its
|
|
3674
|
+
// z-comparison; a mid-break's ¬X is the DIRECT compare negated by v128.not (NaN-exact).
|
|
3675
|
+
const keepMask = (keep, isLimit) => {
|
|
3676
|
+
const m = isLimit
|
|
3677
|
+
? [CMP_LANE[keep.cmp[0]], itLeft ? ['local.get', iterV] : ['local.get', maxitLane], itLeft ? ['local.get', maxitLane] : ['local.get', iterV]]
|
|
3678
|
+
: [CMP_LANE[keep.cmp[0]], lift(keep.cmp[1]), lift(keep.cmp[2])]
|
|
3679
|
+
return keep.negate ? ['v128.not', m] : m
|
|
3680
|
+
}
|
|
3681
|
+
const sbody = [
|
|
3682
|
+
...teeStmts(tees.slice(0, teeTop)),
|
|
3683
|
+
['local.set', activeV, ['v128.and', ['local.get', activeV], keepMask(keepTopC, kTop === 'limit')]],
|
|
3684
|
+
// compound `while (A && B)`: the second keep (B) also gates at the top, before the body
|
|
3685
|
+
...(compoundTop ? teeStmts(tees.slice(teeTop)).concat([['local.set', activeV, ['v128.and', ['local.get', activeV], keepMask(keepMidC, kMid === 'limit')]]]) : []),
|
|
3686
|
+
['br_if', sIn, ['i32.eqz', ['v128.any_true', ['local.get', activeV]]]],
|
|
3687
|
+
]
|
|
3688
|
+
for (let i = 0; i < ibody.length; i++) {
|
|
3689
|
+
if (!compoundTop && i === midIdx) {
|
|
3690
|
+
sbody.push(...teeStmts(tees.slice(teeTop)))
|
|
3691
|
+
sbody.push(['local.set', activeV, ['v128.and', ['local.get', activeV], keepMask(keepMidC, kMid === 'limit')]])
|
|
3692
|
+
} else {
|
|
3693
|
+
const v = ibody[i][1]
|
|
3694
|
+
if (carried.has(v)) sbody.push(['local.set', shadow.get(v), ['v128.bitselect', lift(ibody[i][2]), ['local.get', shadow.get(v)], ['local.get', activeV]]])
|
|
3695
|
+
else sbody.push(['local.set', shadow.get(v), lift(ibody[i][2])])
|
|
3696
|
+
}
|
|
3697
|
+
}
|
|
3698
|
+
sbody.push(['local.set', iterV, ['v128.bitselect', ['f64x2.add', ['local.get', iterV], ['f64x2.splat', ['f64.const', 1]]], ['local.get', iterV], ['local.get', activeV]]])
|
|
3699
|
+
simdInner = ['block', sIn, ['loop', sIl, ...sbody, ['br', sIl]]]
|
|
3700
|
+
const carriedInEpi = [...carried].filter(v => epilogue.some(s => readsVar(s, v)))
|
|
3701
|
+
epiLane = (k) => {
|
|
3702
|
+
const out = []
|
|
3703
|
+
for (const v of carriedInEpi) out.push(['local.set', v, ['f64x2.extract_lane', k, ['local.get', shadow.get(v)]]])
|
|
3704
|
+
out.push(['local.set', itVar, ['i32.trunc_f64_s', ['f64x2.extract_lane', k, ['local.get', iterV]]]])
|
|
3705
|
+
for (const s of epilogue) out.push(bump(s, k))
|
|
3706
|
+
return out
|
|
3707
|
+
}
|
|
3708
|
+
}
|
|
3709
|
+
|
|
3710
|
+
// SIMD outer loop: process a pair while x+1 is still a valid pixel, then advance every pixel IV
|
|
3711
|
+
// by 2. The scalar tail (original block) finishes any last odd pixel.
|
|
3712
|
+
const simdOuter = ['block', sOut, ['loop', sOl,
|
|
3713
|
+
['br_if', sOut, ['i32.eqz', [oExit.cmpOp, bump(['local.get', pxVar], 1), widthBound]]],
|
|
3714
|
+
...pre, simdInner, ...postLoop, ...epiLane(0), ...epiLane(1),
|
|
3715
|
+
...pixelIVs.map(p => ['local.set', p.name, [p.type + '.add', ['local.get', p.name], [p.type + '.const', 2]]]),
|
|
3716
|
+
['br', sOl]]]
|
|
3717
|
+
|
|
3718
|
+
// wrapper: hoisted preambles once (outer + inner-block invariants like BAILOUT), then the
|
|
3719
|
+
// SIMD even-pixel loop, then the original scalar block handles the odd-pixel tail.
|
|
3720
|
+
const tailBlock = ['block', oLabel, loopNode]
|
|
3721
|
+
const wrapper = ['block', nm('w'), ...preamble, ...innerPre, simdOuter, tailBlock]
|
|
3722
|
+
return { wrapper, newLocalDecls }
|
|
3723
|
+
}
|
|
3724
|
+
|
|
3725
|
+
// Math.sin/cos lower to `call $math.{sin,cos}_core` (the emit-time fast path, math.js:67); the
|
|
3726
|
+
// public `$math.{sin,cos}` wrap the same core. Their f64x2 mirrors $math.sin2/$math.cos2 (the
|
|
3727
|
+
// vectorized reduce+horner, module/math.js:543) are BIT-EXACT per lane to the scalar core — so we
|
|
3728
|
+
// can lift the call straight to the *2 helper. Phase-2 adds pow/log/atan2 here (see PPC_CALL2).
|
|
3729
|
+
// NOTE: scalar targets here must be kept out of inlineOnce's single-caller inlining (SIMD_PROTECTED
|
|
3730
|
+
// in src/wat/optimize.js) — else the call node is gone before this lift runs.
|
|
3731
|
+
const PPC_CALL2 = {
|
|
3732
|
+
'$math.sin_core': '$math.sin2', '$math.cos_core': '$math.cos2',
|
|
3733
|
+
'$math.sin': '$math.sin2', '$math.cos': '$math.cos2',
|
|
3734
|
+
'$math.pow': '$math.pow2', // 2-arg; bit-exact per-lane scalar (cancellation-sensitive — see module/math.js)
|
|
3735
|
+
'$math.atan2': '$math.atan2_2', '$math.hypot': '$math.hypot_2', // 2-arg; bit-exact extract/repack
|
|
3736
|
+
// log/exp/exp2: TRUE f64x2 polys — both lanes one evaluation (≈2×, beats V8 native log). Bit-exact
|
|
3737
|
+
// via hot-path-vectorized + scalar-edge-fallback ($math.log_v/exp_v/exp2_v, module/math.js).
|
|
3738
|
+
'$math.log': '$math.log_v', '$math.exp': '$math.exp_v', '$math.exp2': '$math.exp2_v',
|
|
3739
|
+
}
|
|
3740
|
+
|
|
3741
|
+
// Per-pixel-color vectorizer. The dual of tryDivergentEscapeVectorize for kernels with NO inner
|
|
3742
|
+
// escape loop: an outer pixel loop whose body computes an f64 value from the pixel index (via
|
|
3743
|
+
// cos/sin/sqrt/…), packs it to a u32 colour, and stores it — every pixel independent. We lift the
|
|
3744
|
+
// liftable f64 PREFIX of the body to f64x2 (two adjacent pixels per lane: the index becomes the
|
|
3745
|
+
// ramp [x, x+1]; transcendentals map to the bit-exact $math.*2 helpers; conditionals to bitselect),
|
|
3746
|
+
// then run the SCALAR pack+store once per lane (extract_lane → the original f64 local → the
|
|
3747
|
+
// untouched integer pack). The expensive transcendentals run 2-wide; the cheap pack stays scalar.
|
|
3748
|
+
// Bit-exact by construction: f64x2 arithmetic is per-lane IEEE-identical and extract_lane is exact.
|
|
3749
|
+
// A call we can't yet vectorize (pow in Phase 1) just ends the SIMD prefix — its lane local and the
|
|
3750
|
+
// rest fall to the scalar epilogue, so the kernel still partially vectorizes. The original scalar
|
|
3751
|
+
// loop, re-run as the tail, finishes the odd last pixel for free (its own `x < W` guard).
|
|
3752
|
+
function tryPerPixelColor(blockNode, fnLocals, freshIdRef, pureFuncMap) {
|
|
3753
|
+
// Outer per-pixel scaffold — shared with tryDivergentEscapeVectorize; this pass
|
|
3754
|
+
// takes the straight-line-body branch (no inner escape loop) below.
|
|
3755
|
+
const outer = matchOuterPixelLoop(blockNode)
|
|
3756
|
+
if (!outer) return null
|
|
3757
|
+
const { oLabel, loopNode, preamble, pixelIVs, pxVar, widthBound, pivType, obody, oExit } = outer
|
|
3758
|
+
|
|
3759
|
+
// ---- body: straight-line (no inner escape loop), no impure call ----
|
|
3760
|
+
for (const s of obody)
|
|
3761
|
+
if (isArr(s) && s[0] === 'block' && s.slice(1).some(c => isArr(c) && c[0] === 'loop')) return null // inner escape loop → tryDivergentEscapeVectorize's job
|
|
3762
|
+
// A non-pure call (e.g. a ray-march helper that writes a scratch global / memory) can mutate state
|
|
3763
|
+
// that a lane local — computed ONCE, before the per-lane epilogue runs the call — would then read
|
|
3764
|
+
// stale, breaking bit-exactness. $math.* helpers are pure (no global/memory writes), so allow them.
|
|
3765
|
+
// Pure user-defined functions in pureFuncMap are also safe: they have no side effects (verified
|
|
3766
|
+
// when the map is built) and liftPPC inlines them expression-level rather than emitting a call.
|
|
3767
|
+
const impureCall = (n) => isArr(n) && ((n[0] === 'call' && typeof n[1] === 'string' && !n[1].startsWith('$math.') && !(pureFuncMap && pureFuncMap.has(n[1]))) || n.some(impureCall))
|
|
3768
|
+
if (obody.some(impureCall)) return null
|
|
3769
|
+
|
|
3770
|
+
// bump: substitute every pixel IV with (IV + k) in its own type — for the lane-k epilogue.
|
|
3771
|
+
const bump = (n, k) => k === 0 ? n
|
|
3772
|
+
: (isArr(n) && n[0] === 'local.get' && pivType.has(n[1]))
|
|
3773
|
+
? [pivType.get(n[1]) + '.add', n, [pivType.get(n[1]) + '.const', k]]
|
|
3774
|
+
: (isArr(n) ? n.map(c => bump(c, k)) : n)
|
|
3775
|
+
|
|
3776
|
+
// Pixel-coordinate aliases: a local consistently CSE'd to `convert_i32_s(pixelIV)` (jz tees the f64
|
|
3777
|
+
// pixel-x once — reused for the store address AND the per-pixel math — so it lives inside the i32
|
|
3778
|
+
// offset stmt, out of reach as a lane local). Treat its reads as the ramp, recomputed per lane.
|
|
3779
|
+
const pxAlias = new Map()
|
|
3780
|
+
{
|
|
3781
|
+
const defs = new Map()
|
|
3782
|
+
const scan = (n) => { if (!isArr(n)) return; if ((n[0] === 'local.set' || n[0] === 'local.tee') && typeof n[1] === 'string') { (defs.get(n[1]) || defs.set(n[1], []).get(n[1])).push(n[2]) } for (let i = 1; i < n.length; i++) scan(n[i]) }
|
|
3783
|
+
obody.forEach(scan)
|
|
3784
|
+
for (const [name, rhss] of defs) {
|
|
3785
|
+
const j = JSON.stringify(rhss[0])
|
|
3786
|
+
if (!rhss.every(r => JSON.stringify(r) === j)) continue // multiple distinct defs → not a stable alias
|
|
3787
|
+
const r = rhss[0]
|
|
3788
|
+
if (isArr(r) && r[0] === 'f64.convert_i32_s' && isArr(r[1]) && r[1][0] === 'local.get' && pivType.get(r[1][1]) === 'i32') pxAlias.set(name, r[1][1])
|
|
3789
|
+
else if (isArr(r) && r[0] === 'local.get' && pivType.get(r[1]) === 'f64') pxAlias.set(name, r[1])
|
|
3790
|
+
}
|
|
3791
|
+
}
|
|
3792
|
+
// The two lanes of a pixel IV (or its alias): [v, v+1], in f64 (an i32 IV is converted per lane).
|
|
3793
|
+
const rampOf = (piv) => pivType.get(piv) === 'f64'
|
|
3794
|
+
? ['f64x2.replace_lane', 1, ['f64x2.splat', ['local.get', piv]], ['f64.add', ['local.get', piv], ['f64.const', 1]]]
|
|
3795
|
+
: ['f64x2.replace_lane', 1, ['f64x2.splat', ['f64.convert_i32_s', ['local.get', piv]]], ['f64.convert_i32_s', ['i32.add', ['local.get', piv], ['i32.const', 1]]]]
|
|
3796
|
+
|
|
3797
|
+
const id = freshIdRef.next++
|
|
3798
|
+
const nm = (s) => `$__ppc${id}_${s}`
|
|
3799
|
+
const laneMap = new Map() // f64 lane-local name → its v128 shadow
|
|
3800
|
+
const laneLifted = new Map() // f64 lane-local name → its lifted f64x2 expr
|
|
3801
|
+
|
|
3802
|
+
// Inline a pure user function call into a lifted f64x2 expression.
|
|
3803
|
+
// `callNode` is ['call', '$name', arg0, arg1, ...]; `outerLift` is the liftPPC fn.
|
|
3804
|
+
// Walks the callee's body, substituting params with lifted args and inlined-local
|
|
3805
|
+
// intermediates. Returns null if any step fails (bail → scalar epilogue).
|
|
3806
|
+
// SOUND: only called when callee is in pureFuncMap (no stores/global.sets/impure
|
|
3807
|
+
// calls); param names are read-only in the inlinee body (verified below).
|
|
3808
|
+
const liftPPCInline = (callNode, outerLift) => {
|
|
3809
|
+
const callee = pureFuncMap.get(callNode[1])
|
|
3810
|
+
if (!callee) return null
|
|
3811
|
+
const calleeBodyStart = findBodyStart(callee)
|
|
3812
|
+
if (calleeBodyStart < 0) return null
|
|
3813
|
+
|
|
3814
|
+
// Collect callee params in order.
|
|
3815
|
+
const calleeParams = []
|
|
3816
|
+
for (let i = 2; i < calleeBodyStart; i++) {
|
|
3817
|
+
const d = callee[i]
|
|
3818
|
+
if (isArr(d) && d[0] === 'param' && typeof d[1] === 'string') calleeParams.push(d[1])
|
|
3819
|
+
}
|
|
3820
|
+
// Args supplied by the call site (call node children after the name).
|
|
3821
|
+
const callArgs = callNode.slice(2)
|
|
3822
|
+
if (callArgs.length !== calleeParams.length) return null
|
|
3823
|
+
|
|
3824
|
+
// Lift each arg with the outer liftPPC.
|
|
3825
|
+
const substMap = new Map()
|
|
3826
|
+
for (let i = 0; i < calleeParams.length; i++) {
|
|
3827
|
+
const lifted = outerLift(callArgs[i])
|
|
3828
|
+
if (lifted === null) return null
|
|
3829
|
+
substMap.set(calleeParams[i], lifted)
|
|
3830
|
+
}
|
|
3831
|
+
|
|
3832
|
+
// Verify no local.set on a param name inside the callee body (params are read-only).
|
|
3833
|
+
for (const pname of calleeParams) {
|
|
3834
|
+
if (writesName(callee.slice(calleeBodyStart), pname)) return null
|
|
3835
|
+
}
|
|
3836
|
+
|
|
3837
|
+
// liftInline: lift a callee-body expression using substMap (params + inlined locals).
|
|
3838
|
+
// Handles f64.const, local.get from substMap, LANE_PURE.f64 ops, and PPC_CALL2 calls.
|
|
3839
|
+
// Returns null on any unsupported node.
|
|
3840
|
+
const liftInline = (n) => {
|
|
3841
|
+
if (!isArr(n)) return null
|
|
3842
|
+
const op = n[0]
|
|
3843
|
+
if (op === 'f64.const') return ['f64x2.splat', n]
|
|
3844
|
+
if (op === 'local.get' && typeof n[1] === 'string') {
|
|
3845
|
+
return substMap.has(n[1]) ? substMap.get(n[1]) : null
|
|
3846
|
+
}
|
|
3847
|
+
if (op === 'call') {
|
|
3848
|
+
const v2 = PPC_CALL2[n[1]]
|
|
3849
|
+
if (v2 && n.length === 3) { const a = liftInline(n[2]); return a && ['call', v2, a] }
|
|
3850
|
+
if (v2 && n.length === 4) { const a = liftInline(n[2]), b = liftInline(n[3]); return (a && b) ? ['call', v2, a, b] : null }
|
|
3851
|
+
return null
|
|
3852
|
+
}
|
|
3853
|
+
if (LANE_PURE.f64.has(op)) {
|
|
3854
|
+
const ks = n.slice(1).map(liftInline)
|
|
3855
|
+
return ks.some(k => k === null) ? null : [LANE_PURE.f64.get(op).simd, ...ks]
|
|
3856
|
+
}
|
|
3857
|
+
return null
|
|
3858
|
+
}
|
|
3859
|
+
|
|
3860
|
+
// Walk callee body: local.set stmts define inlined locals; return stmt gives result.
|
|
3861
|
+
for (let i = calleeBodyStart; i < callee.length; i++) {
|
|
3862
|
+
const stmt = callee[i]
|
|
3863
|
+
if (!isArr(stmt)) return null
|
|
3864
|
+
if (stmt[0] === 'local.set' && typeof stmt[1] === 'string') {
|
|
3865
|
+
const lifted = liftInline(stmt[2])
|
|
3866
|
+
if (lifted === null) return null
|
|
3867
|
+
substMap.set(stmt[1], lifted)
|
|
3868
|
+
continue
|
|
3869
|
+
}
|
|
3870
|
+
if (stmt[0] === 'return') {
|
|
3871
|
+
return liftInline(stmt[1])
|
|
3872
|
+
}
|
|
3873
|
+
// Any other statement type → bail (impure or unsupported structure).
|
|
3874
|
+
return null
|
|
3875
|
+
}
|
|
3876
|
+
return null // no return stmt found
|
|
3877
|
+
}
|
|
3878
|
+
|
|
3879
|
+
// Lift a scalar f64 expression to f64x2: pixel IV → ramp [v, v+1]; an earlier lane local → its
|
|
3880
|
+
// shadow; an invariant (local/global the loop never writes) → splat; transcendental call → the
|
|
3881
|
+
// *2 helper; conditional → bitselect; LANE_PURE.f64 op → recurse. null = not liftable (the lift
|
|
3882
|
+
// stops here and the rest becomes the scalar epilogue).
|
|
3883
|
+
const liftPPC = (n) => {
|
|
3884
|
+
if (!isArr(n)) return null
|
|
3885
|
+
const op = n[0]
|
|
3886
|
+
if (op === 'f64.const') return ['f64x2.splat', n]
|
|
3887
|
+
if (op === 'local.get') {
|
|
3888
|
+
const v = n[1]
|
|
3889
|
+
if (laneMap.has(v)) return ['local.get', laneMap.get(v)]
|
|
3890
|
+
if (pxAlias.has(v)) return rampOf(pxAlias.get(v))
|
|
3891
|
+
if (pivType.get(v) === 'f64') return rampOf(v)
|
|
3892
|
+
if (writesName(loopNode, v)) return null
|
|
3893
|
+
return ['f64x2.splat', n]
|
|
3894
|
+
}
|
|
3895
|
+
if (op === 'local.tee') { // CSE temp inside a lane expr (e.g. `dx` reused as dx*dx) → a v128 tee
|
|
3896
|
+
const lifted = liftPPC(n[2])
|
|
3897
|
+
if (lifted === null) return null
|
|
3898
|
+
const lane = laneMap.get(n[1]) || nm('t' + n[1].replace(/\W/g, ''))
|
|
3899
|
+
laneMap.set(n[1], lane) // later local.get $v in this expr resolves to the tee's lane
|
|
3900
|
+
return ['local.tee', lane, lifted]
|
|
3901
|
+
}
|
|
3902
|
+
if (op === 'global.get') return writesName(loopNode, n[1]) ? null : ['f64x2.splat', n]
|
|
3903
|
+
if (op === 'f64.convert_i32_s' && isArr(n[1]) && n[1][0] === 'local.get' && pivType.get(n[1][1]) === 'i32') return rampOf(n[1][1])
|
|
3904
|
+
if (op === 'call') {
|
|
3905
|
+
const v2 = PPC_CALL2[n[1]]
|
|
3906
|
+
if (v2 && n.length === 3) { const a = liftPPC(n[2]); return a && ['call', v2, a] }
|
|
3907
|
+
if (v2 && n.length === 4) { const a = liftPPC(n[2]), b = liftPPC(n[3]); return (a && b) ? ['call', v2, a, b] : null }
|
|
3908
|
+
// Pure user-function inline: substitute params with lifted args, walk body.
|
|
3909
|
+
if (pureFuncMap && pureFuncMap.has(n[1])) return liftPPCInline(n, liftPPC)
|
|
3910
|
+
return null
|
|
3911
|
+
}
|
|
3912
|
+
if (op === 'if') { // `cond ? X : Y` (jz lowers to (if (result f64) COND (then X)(else Y))) → bitselect
|
|
3913
|
+
if (!isArr(n[1]) || n[1][0] !== 'result' || n[1][1] !== 'f64') return null
|
|
3914
|
+
const thenN = n[3], elseN = n[4]
|
|
3915
|
+
if (!isArr(thenN) || thenN[0] !== 'then' || thenN.length !== 2) return null
|
|
3916
|
+
if (!isArr(elseN) || elseN[0] !== 'else' || elseN.length !== 2) return null
|
|
3917
|
+
let cond = n[2]
|
|
3918
|
+
if (isArr(cond) && cond[0] === 'i32.ne' && isI32Const(cond[2]) && cond[2][1] === 0) cond = cond[1]
|
|
3919
|
+
const cmp = isArr(cond) && cond.length === 3 ? CMP_LANE[cond[0]] : null
|
|
3920
|
+
if (!cmp) return null
|
|
3921
|
+
const ca = liftPPC(cond[1]), cb = liftPPC(cond[2]), x = liftPPC(thenN[1]), y = liftPPC(elseN[1])
|
|
3922
|
+
if (!ca || !cb || !x || !y) return null
|
|
3923
|
+
return ['v128.bitselect', x, y, [cmp, ca, cb]]
|
|
3924
|
+
}
|
|
3925
|
+
if (LANE_PURE.f64.has(op)) {
|
|
3926
|
+
const ks = n.slice(1).map(liftPPC)
|
|
3927
|
+
return ks.some(k => k === null) ? null : [LANE_PURE.f64.get(op).simd, ...ks]
|
|
3928
|
+
}
|
|
3929
|
+
return null
|
|
3930
|
+
}
|
|
3931
|
+
|
|
3932
|
+
// Classify each body statement: a `local.set $v EXPR` with v an f64 whose EXPR fully lifts is a
|
|
3933
|
+
// SIMD lane local (computed once per pair); everything else (the integer pack, the store, an
|
|
3934
|
+
// un-liftable call like pow in Phase 1, a recomputed i32 offset) is a scalar EPILOGUE statement,
|
|
3935
|
+
// re-run per lane. Lane locals need NOT be a contiguous prefix — `offset = w*y+x` (i32) commonly
|
|
3936
|
+
// precedes the f64 work. Processed in source order so a lane local can reference an earlier one;
|
|
3937
|
+
// liftPPC returns null on a read of any in-loop value that isn't already a lane local (incl. a
|
|
3938
|
+
// later or epilogue local), so the classification self-enforces "lane locals depend only on
|
|
3939
|
+
// IVs/invariants/earlier lane locals" — reordering all lane computes ahead of the epilogue is safe.
|
|
3940
|
+
const epilogue = []
|
|
3941
|
+
for (const s of obody) {
|
|
3942
|
+
if (isArr(s) && s[0] === 'local.set' && s.length === 3 && fnLocals.get(s[1]) === 'f64') {
|
|
3943
|
+
const before = new Set(laneMap.keys())
|
|
3944
|
+
const lifted = liftPPC(s[2])
|
|
3945
|
+
if (lifted !== null) { laneMap.set(s[1], nm('l' + s[1].replace(/\W/g, ''))); laneLifted.set(s[1], lifted); continue }
|
|
3946
|
+
for (const k of [...laneMap.keys()]) if (!before.has(k)) laneMap.delete(k) // roll back tee pollution from a failed lift
|
|
3947
|
+
}
|
|
3948
|
+
epilogue.push(s)
|
|
3949
|
+
}
|
|
3950
|
+
if (!laneMap.size) return null // nothing lifted to f64x2
|
|
3951
|
+
// HAZARD: a lane local re-written by an epilogue statement leaves its f64x2 shadow STALE.
|
|
3952
|
+
// e.g. `let fx=0; if(denom>ε){fx=…}; let mag=hypot(fx,…)` — `fx=0` lifts to a lane local
|
|
3953
|
+
// splat(0); the statement-form `if` lands in the scalar epilogue (updates only the SCALAR local),
|
|
3954
|
+
// so the lifted `hypot(fx,…)` — emitted BEFORE the epilogue — reads the stale splat(0) → all-zero.
|
|
3955
|
+
// Bail ONLY when the stale shadow actually feeds another LANE compute: a lane local whose shadow
|
|
3956
|
+
// is CONSUMED by some laneLifted expr and is ALSO reassigned in the epilogue. (A lane local merely
|
|
3957
|
+
// extracted for the scalar epilogue — e.g. `gv`, clamped by `if(gv<0)gv=0` then packed — is safe:
|
|
3958
|
+
// the clamp runs per-lane after extraction, corrupting no other lane.)
|
|
3959
|
+
{
|
|
3960
|
+
const consumedShadows = new Set()
|
|
3961
|
+
const scanShadow = (n) => { if (!isArr(n)) return; if (n[0] === 'local.get' && typeof n[1] === 'string') consumedShadows.add(n[1]); for (const c of n.slice(1)) scanShadow(c) }
|
|
3962
|
+
for (const expr of laneLifted.values()) scanShadow(expr)
|
|
3963
|
+
const hazard = (n) => isArr(n) && (((n[0] === 'local.set' || n[0] === 'local.tee') && laneMap.has(n[1]) && consumedShadows.has(laneMap.get(n[1]))) || n.slice(1).some(hazard))
|
|
3964
|
+
if (epilogue.some(hazard)) return null
|
|
3965
|
+
}
|
|
3966
|
+
// Only worth the extract overhead if a costly op (a *2 transcendental or f64x2.sqrt) got lifted.
|
|
3967
|
+
const heavy = (n) => isArr(n) && ((n[0] === 'call' && /\$math\.(sin2|cos2|pow2|log_v|atan2_2|hypot_2|exp2_2|tan2)/.test(n[1])) || n[0] === 'f64x2.sqrt' || n.some(heavy))
|
|
3968
|
+
if (![...laneLifted.values()].some(heavy)) return null // only cheap arithmetic lifted — not worth it
|
|
3969
|
+
|
|
3970
|
+
// Exactly one i32.store, found anywhere in the epilogue (jz wraps `mem[off]=…` in a `(block …)`).
|
|
3971
|
+
let storeStmt = null
|
|
3972
|
+
const findStore = (n) => { if (!isArr(n)) return; if (STORE_OPS[n[0]]) { if (storeStmt) storeStmt = false; else if (storeStmt !== false) storeStmt = n } for (const c of n) findStore(c) }
|
|
3973
|
+
epilogue.forEach(findStore)
|
|
3974
|
+
if (!storeStmt || storeStmt[0] !== 'i32.store') return null // not exactly one u32 colour store
|
|
3975
|
+
// The store cell must differ per lane, i.e. its address must depend on a pixel IV — directly
|
|
3976
|
+
// (chladni's `px[j]`) or transitively through an epilogue local (interference's `mem[offset]`,
|
|
3977
|
+
// offset=w*y+x). Follow the address's reads through epilogue local definitions to a pixel IV.
|
|
3978
|
+
// Walk every child except a set's name slot (a def nested under an `(if … then)` must still be
|
|
3979
|
+
// recorded — the n.slice(2)-everywhere form would miss it, conservatively bailing a valid kernel).
|
|
3980
|
+
const epiDef = new Map()
|
|
3981
|
+
const collect = (n) => { if (!isArr(n)) return; if (n[0] === 'local.set' && typeof n[1] === 'string' && !epiDef.has(n[1])) epiDef.set(n[1], n[2]); for (const c of (n[0] === 'local.set' ? n.slice(2) : n.slice(1))) collect(c) }
|
|
3982
|
+
epilogue.forEach(collect)
|
|
3983
|
+
const feedsIV = (n, seen = new Set()) => isArr(n) && (n[0] === 'local.get'
|
|
3984
|
+
? (pivType.has(n[1]) || (epiDef.has(n[1]) && !seen.has(n[1]) && (seen.add(n[1]), feedsIV(epiDef.get(n[1]), seen))))
|
|
3985
|
+
: n.some(c => feedsIV(c, seen)))
|
|
3986
|
+
if (!feedsIV(storeStmt[1])) return null // store cell wouldn't vary per lane → can't pair
|
|
3987
|
+
|
|
3988
|
+
// ---- epilogue safety: runs scalar per lane (each statement bumped to pixel j+k). It may read a
|
|
3989
|
+
// lane local (extracted below), an invariant/pixel-IV, or a value the epilogue itself computes —
|
|
3990
|
+
// incl. within-statement tees (e.g. the Infinity-guard temp inside an `(if … |0)` pack). Straight-
|
|
3991
|
+
// line source guarantees write-before-read, so it suffices that every read of an in-loop local is
|
|
3992
|
+
// a lane local or written somewhere in the epilogue (a lane local read is satisfied by extraction). ----
|
|
3993
|
+
{
|
|
3994
|
+
const epiWritten = new Set()
|
|
3995
|
+
const wr = (n) => { if (!isArr(n)) return; const st = (n[0] === 'local.set' || n[0] === 'local.tee') && typeof n[1] === 'string'; if (st) epiWritten.add(n[1]); for (const c of (st ? n.slice(2) : n.slice(1))) wr(c) }
|
|
3996
|
+
epilogue.forEach(wr)
|
|
3997
|
+
const reads = new Set(); const rd = (n) => { if (!isArr(n)) return; if (n[0] === 'local.get') reads.add(n[1]); else for (const c of n) rd(c) }
|
|
3998
|
+
epilogue.forEach(rd)
|
|
3999
|
+
for (const v of reads) if (writesName(loopNode, v) && !laneMap.has(v) && !epiWritten.has(v) && !pivType.has(v)) return null // reads an in-loop value with no per-lane source
|
|
4000
|
+
}
|
|
4001
|
+
const epiReads = [...laneMap.keys()].filter(v => epilogue.some(s => readsVar(s, v)))
|
|
4002
|
+
|
|
4003
|
+
// ============================ emit ============================
|
|
4004
|
+
const newLocalDecls = [...laneMap.values()].map(n => ['local', n, 'v128'])
|
|
4005
|
+
const laneCompute = [...laneLifted.keys()].map(v => ['local.set', laneMap.get(v), laneLifted.get(v)])
|
|
4006
|
+
const epiLane = (k) => [
|
|
4007
|
+
...epiReads.map(v => ['local.set', v, ['f64x2.extract_lane', k, ['local.get', laneMap.get(v)]]]),
|
|
4008
|
+
...epilogue.map(s => bump(s, k)),
|
|
4009
|
+
]
|
|
4010
|
+
const sOut = nm('ob'), sOl = nm('ol')
|
|
4011
|
+
const simdOuter = ['block', sOut, ['loop', sOl,
|
|
4012
|
+
['br_if', sOut, ['i32.eqz', [oExit.cmpOp, bump(['local.get', pxVar], 1), widthBound]]],
|
|
4013
|
+
...laneCompute, ...epiLane(0), ...epiLane(1),
|
|
4014
|
+
...pixelIVs.map(p => ['local.set', p.name, [p.type + '.add', ['local.get', p.name], [p.type + '.const', 2]]]),
|
|
4015
|
+
['br', sOl]]]
|
|
4016
|
+
const wrapper = ['block', nm('w'), ...preamble, simdOuter, ['block', oLabel, loopNode]]
|
|
4017
|
+
return { wrapper, newLocalDecls }
|
|
4018
|
+
}
|
|
4019
|
+
|
|
4020
|
+
// ---- Outer-loop strip-mine over an inner reduction (tryOuterStrip, experimental) ----
|
|
4021
|
+
//
|
|
4022
|
+
// The dual of tryPerPixelColor for pixel loops whose per-pixel value comes from an
|
|
4023
|
+
// INNER REDUCTION over invariant data — metaballs `sum += r²/((cx-bx[b])²+(cy-by[b])²+ε)`,
|
|
4024
|
+
// voronoi/lyapunov shapes. Strip-mine the OUTER pixel loop 2-wide: pixels (xi, xi+1) →
|
|
4025
|
+
// f64x2 lanes. The per-pixel coordinate (`cx = xi/W`) becomes a ramp `[cx, cx+1/W]`; the
|
|
4026
|
+
// inner loop's loads `bx[b]` are indexed by the INNER IV (same for both pixels) → splat;
|
|
4027
|
+
// the accumulator `sum` becomes an f64x2 carrying both lanes' running sums. After the inner
|
|
4028
|
+
// loop, each lane's sum is extracted and the scalar pack+store runs per lane (xi, xi+1).
|
|
4029
|
+
//
|
|
4030
|
+
// BIT-EXACT: each lane accumulates in the SAME scalar order as the original (f64x2.add is
|
|
4031
|
+
// per-lane IEEE-754-identical) — a per-lane reduction reorders nothing, unlike a horizontal
|
|
4032
|
+
// fold. The inner loop's trip count (b < count) is invariant, so its scaffold stays scalar;
|
|
4033
|
+
// only the f64 body lifts. Distinct base subtrees assumed non-aliasing (the standing model).
|
|
4034
|
+
// Gated behind cfg.experimentalOuterStrip until proven across the corpus.
|
|
4035
|
+
function tryOuterStrip(blockNode, fnLocals, freshIdRef, enabled) {
|
|
4036
|
+
if (!enabled) return null
|
|
4037
|
+
const outer = matchOuterPixelLoop(blockNode)
|
|
4038
|
+
if (!outer) return null
|
|
4039
|
+
const { oLabel, loopNode, preamble, pixelIVs, pxVar, widthBound, pivType, obody, oExit } = outer
|
|
4040
|
+
|
|
4041
|
+
// Exactly one inner loop in obody; it is the per-pixel reduction.
|
|
4042
|
+
let innerIdx = -1, innerBlock = null
|
|
4043
|
+
for (let i = 0; i < obody.length; i++) {
|
|
4044
|
+
const s = obody[i]
|
|
4045
|
+
if (isArr(s) && s[0] === 'block' && s.slice(1).some(c => isArr(c) && c[0] === 'loop')) {
|
|
4046
|
+
if (innerBlock) return null
|
|
4047
|
+
innerBlock = s; innerIdx = i
|
|
4048
|
+
}
|
|
4049
|
+
}
|
|
4050
|
+
if (!innerBlock) return null
|
|
4051
|
+
const ibl = matchBlockLoop(innerBlock, { allowPreamble: true })
|
|
4052
|
+
if (!ibl) return null
|
|
4053
|
+
if (ibl.preamble.length) return null
|
|
4054
|
+
const innerIV = ibl.incVar, ibody = ibl.body
|
|
4055
|
+
// No impure calls (a non-pure call would read stale state in the per-lane epilogue). $math.* pure.
|
|
4056
|
+
const impureCall = (n) => isArr(n) && ((n[0] === 'call' && typeof n[1] === 'string' && !n[1].startsWith('$math.')) || n.some(impureCall))
|
|
4057
|
+
if (obody.some(impureCall)) return null
|
|
4058
|
+
|
|
4059
|
+
const id = freshIdRef.next++
|
|
4060
|
+
const nm = (s) => `$__os${id}_${s}`
|
|
4061
|
+
const bump = (n, k) => k === 0 ? n
|
|
4062
|
+
: (isArr(n) && n[0] === 'local.get' && pivType.has(n[1])) ? [pivType.get(n[1]) + '.add', n, [pivType.get(n[1]) + '.const', k]]
|
|
4063
|
+
: (isArr(n) ? n.map(c => bump(c, k)) : n)
|
|
4064
|
+
const rampOf = (piv) => pivType.get(piv) === 'f64'
|
|
4065
|
+
? ['f64x2.replace_lane', 1, ['f64x2.splat', ['local.get', piv]], ['f64.add', ['local.get', piv], ['f64.const', 1]]]
|
|
4066
|
+
: ['f64x2.replace_lane', 1, ['f64x2.splat', ['f64.convert_i32_s', ['local.get', piv]]], ['f64.convert_i32_s', ['i32.add', ['local.get', piv], ['i32.const', 1]]]]
|
|
4067
|
+
const readsName = (n, name) => isArr(n) && ((n[0] === 'local.get' && n[1] === name) || n.some(c => readsName(c, name)))
|
|
4068
|
+
|
|
4069
|
+
const laneMap = new Map() // f64 lane-local (per-pixel-varying) name → its v128 shadow
|
|
4070
|
+
// Lift a scalar f64 expr to f64x2 (null = not liftable). pxVar → ramp; lane local → shadow;
|
|
4071
|
+
// pixel-invariant local/global → splat; pixel-invariant f64.load → splat(scalar load);
|
|
4072
|
+
// $math.*2 transcendental; cond → bitselect; LANE_PURE.f64 op → recurse.
|
|
4073
|
+
const liftOS = (n) => {
|
|
4074
|
+
if (!isArr(n)) return null
|
|
4075
|
+
const op = n[0]
|
|
4076
|
+
if (op === 'f64.const') return ['f64x2.splat', n]
|
|
4077
|
+
if (op === 'local.get') {
|
|
4078
|
+
const v = n[1]
|
|
4079
|
+
if (laneMap.has(v)) return ['local.get', laneMap.get(v)]
|
|
4080
|
+
if (pivType.get(v) === 'f64') return rampOf(v)
|
|
4081
|
+
if (writesName(loopNode, v)) return null
|
|
4082
|
+
return ['f64x2.splat', n]
|
|
4083
|
+
}
|
|
4084
|
+
if (op === 'f64.convert_i32_s' && isArr(n[1]) && n[1][0] === 'local.get' && pivType.get(n[1][1]) === 'i32') return rampOf(n[1][1])
|
|
4085
|
+
if (op === 'global.get') return writesName(loopNode, n[1]) ? null : ['f64x2.splat', n]
|
|
4086
|
+
if (LOAD_OPS[op] === 'f64') {
|
|
4087
|
+
// pixel-invariant load (address reads neither the pixel IV nor any per-pixel lane) is the
|
|
4088
|
+
// same value for both lanes → load once, splat. A per-pixel gather is not supported.
|
|
4089
|
+
const addr = typeof n[1] === 'string' && n[1].startsWith('offset=') ? n[2] : n[1]
|
|
4090
|
+
if (readsName(addr, pxVar) || [...laneMap.keys()].some(lv => readsName(addr, lv))) return null
|
|
4091
|
+
return ['f64x2.splat', n]
|
|
4092
|
+
}
|
|
4093
|
+
if (op === 'call') {
|
|
4094
|
+
const v2 = PPC_CALL2[n[1]]
|
|
4095
|
+
if (v2 && n.length === 3) { const a = liftOS(n[2]); return a && ['call', v2, a] }
|
|
4096
|
+
if (v2 && n.length === 4) { const a = liftOS(n[2]), b = liftOS(n[3]); return (a && b) ? ['call', v2, a, b] : null }
|
|
4097
|
+
return null
|
|
4098
|
+
}
|
|
4099
|
+
if (op === 'if') {
|
|
4100
|
+
if (!isArr(n[1]) || n[1][0] !== 'result' || n[1][1] !== 'f64') return null
|
|
4101
|
+
const thenN = n[3], elseN = n[4]
|
|
4102
|
+
if (!isArr(thenN) || thenN[0] !== 'then' || thenN.length !== 2) return null
|
|
4103
|
+
if (!isArr(elseN) || elseN[0] !== 'else' || elseN.length !== 2) return null
|
|
4104
|
+
let cond = n[2]
|
|
4105
|
+
if (isArr(cond) && cond[0] === 'i32.ne' && isI32Const(cond[2]) && cond[2][1] === 0) cond = cond[1]
|
|
4106
|
+
const cmp = isArr(cond) && cond.length === 3 ? CMP_LANE[cond[0]] : null
|
|
4107
|
+
if (!cmp) return null
|
|
4108
|
+
const ca = liftOS(cond[1]), cb = liftOS(cond[2]), x = liftOS(thenN[1]), y = liftOS(elseN[1])
|
|
4109
|
+
if (!ca || !cb || !x || !y) return null
|
|
4110
|
+
return ['v128.bitselect', x, y, [cmp, ca, cb]]
|
|
4111
|
+
}
|
|
4112
|
+
if (LANE_PURE.f64.has(op)) {
|
|
4113
|
+
const ks = n.slice(1).map(liftOS)
|
|
4114
|
+
return ks.some(k => k === null) ? null : [LANE_PURE.f64.get(op).simd, ...ks]
|
|
4115
|
+
}
|
|
4116
|
+
return null
|
|
4117
|
+
}
|
|
4118
|
+
|
|
4119
|
+
// ---- lift the inner loop body: temp f64 lane locals + f64 accumulator(s) `acc = acc + EXPR`,
|
|
4120
|
+
// the inner IV bump stays scalar. Anything else (or an unliftable expr) → bail. ----
|
|
4121
|
+
// Pre-scan: f64 accumulators `acc = acc + EXPR`. Assign their f64x2 shadows up front so laneInit
|
|
4122
|
+
// can seed them and the lift can accumulate into them (order-independent of the inner body).
|
|
4123
|
+
const accNames = new Set()
|
|
4124
|
+
for (const s of ibody) {
|
|
4125
|
+
if (!(isArr(s) && s[0] === 'local.set' && typeof s[1] === 'string' && s.length === 3)) continue
|
|
4126
|
+
const name = s[1], rhs = s[2]
|
|
4127
|
+
if (fnLocals.get(name) !== 'f64' || !isArr(rhs) || rhs[0] !== 'f64.add') continue
|
|
4128
|
+
const addend = isLocalGet(rhs[1], name) ? rhs[2] : isLocalGet(rhs[2], name) ? rhs[1] : null
|
|
4129
|
+
if (addend != null && !readsName(addend, name)) { accNames.add(name); laneMap.set(name, nm('acc' + name.replace(/\W/g, ''))) }
|
|
4130
|
+
}
|
|
4131
|
+
if (!accNames.size) return null
|
|
4132
|
+
|
|
4133
|
+
// ---- pre-inner-loop stmts (obody[<innerIdx]): per-pixel coord lanes (cx = f(xi) → ramp),
|
|
4134
|
+
// accumulator seeds (→ splat), scalar inner-IV init. MUST run before the inner-body lift so the
|
|
4135
|
+
// per-pixel coord lanes are registered when the inner loop references them. ----
|
|
4136
|
+
const laneInit = []
|
|
4137
|
+
for (let i = 0; i < innerIdx; i++) {
|
|
4138
|
+
const s = obody[i]
|
|
4139
|
+
if (!(isArr(s) && s[0] === 'local.set' && typeof s[1] === 'string' && s.length === 3)) { laneInit.push(s); continue }
|
|
4140
|
+
const name = s[1]
|
|
4141
|
+
if (accNames.has(name)) { // accumulator seed → splat
|
|
4142
|
+
const seed = liftOS(s[2])
|
|
4143
|
+
if (!seed) return null
|
|
4144
|
+
laneInit.push(['local.set', laneMap.get(name), seed]); continue
|
|
4145
|
+
}
|
|
4146
|
+
if (fnLocals.get(name) === 'f64' && readsName(s[2], pxVar)) { // per-pixel coord (cx = xi/W) → ramp lane
|
|
4147
|
+
const lane = liftOS(s[2])
|
|
4148
|
+
if (!lane) return null
|
|
4149
|
+
const sh = nm('p' + name.replace(/\W/g, ''))
|
|
4150
|
+
laneMap.set(name, sh)
|
|
4151
|
+
laneInit.push(['local.set', sh, lane]); continue
|
|
4152
|
+
}
|
|
4153
|
+
laneInit.push(s) // scalar (inner IV init `b=0`, invariant setup)
|
|
4154
|
+
}
|
|
4155
|
+
|
|
4156
|
+
// ---- lift the inner-loop body: temp f64 lane locals + accumulate into the acc shadows; the
|
|
4157
|
+
// inner IV bump stays scalar. Per-pixel coords now resolve via laneMap. ----
|
|
4158
|
+
const liftedInner = []
|
|
4159
|
+
for (const s of ibody) {
|
|
4160
|
+
if (matchInc1(s) === innerIV || matchIncN(s)?.name === innerIV) { liftedInner.push(s); continue }
|
|
4161
|
+
if (!(isArr(s) && s[0] === 'local.set' && typeof s[1] === 'string' && s.length === 3)) return null
|
|
4162
|
+
const name = s[1], rhs = s[2]
|
|
4163
|
+
if (fnLocals.get(name) !== 'f64') return null
|
|
4164
|
+
if (accNames.has(name)) {
|
|
4165
|
+
const addend = isLocalGet(rhs[1], name) ? rhs[2] : rhs[1]
|
|
4166
|
+
const lifted = liftOS(addend)
|
|
4167
|
+
if (!lifted) return null
|
|
4168
|
+
liftedInner.push(['local.set', laneMap.get(name), ['f64x2.add', ['local.get', laneMap.get(name)], lifted]]); continue
|
|
4169
|
+
}
|
|
4170
|
+
if (readsName(rhs, name)) return null // loop-carried non-accumulator → bail
|
|
4171
|
+
const lifted = liftOS(rhs)
|
|
4172
|
+
if (!lifted) return null
|
|
4173
|
+
const sh = laneMap.get(name) || nm('t' + name.replace(/\W/g, ''))
|
|
4174
|
+
laneMap.set(name, sh)
|
|
4175
|
+
liftedInner.push(['local.set', sh, lifted])
|
|
4176
|
+
}
|
|
4177
|
+
|
|
4178
|
+
// ---- epilogue (obody[>innerIdx]): the per-pixel pack+store, run scalar per lane (bumped to xi+k),
|
|
4179
|
+
// reading the extracted accumulator/lane values. Safety: every in-loop read must be a lane local,
|
|
4180
|
+
// a pixel IV, or written within the epilogue itself. ----
|
|
4181
|
+
const epilogue = obody.slice(innerIdx + 1)
|
|
4182
|
+
{
|
|
4183
|
+
const epiWritten = new Set()
|
|
4184
|
+
const wr = (n) => { if (!isArr(n)) return; const st = (n[0] === 'local.set' || n[0] === 'local.tee') && typeof n[1] === 'string'; if (st) epiWritten.add(n[1]); for (const c of (st ? n.slice(2) : n.slice(1))) wr(c) }
|
|
4185
|
+
epilogue.forEach(wr)
|
|
4186
|
+
const reads = new Set(); const rd = (n) => { if (!isArr(n)) return; if (n[0] === 'local.get') reads.add(n[1]); else for (const c of n) rd(c) }
|
|
4187
|
+
epilogue.forEach(rd)
|
|
4188
|
+
for (const v of reads) if (writesName(loopNode, v) && !laneMap.has(v) && !epiWritten.has(v) && !pivType.has(v)) return null
|
|
4189
|
+
}
|
|
4190
|
+
// store must exist + vary per lane
|
|
4191
|
+
let hasStore = false
|
|
4192
|
+
const findStore = (n) => { if (!isArr(n)) return; if (STORE_OPS[n[0]]) hasStore = true; n.forEach(findStore) }
|
|
4193
|
+
epilogue.forEach(findStore)
|
|
4194
|
+
if (!hasStore) return null
|
|
4195
|
+
|
|
4196
|
+
// ============================ emit ============================
|
|
4197
|
+
const newLocalDecls = [...new Set(laneMap.values())].map(n => ['local', n, 'v128'])
|
|
4198
|
+
const epiReads = [...laneMap.keys()].filter(v => epilogue.some(s => readsVar(s, v)))
|
|
4199
|
+
// Rebuild the inner loop with its scalar scaffold (exit + the bottom IV bump, which lives at
|
|
4200
|
+
// loopNode[incIdx] — NOT in `body`) and the lifted f64x2 body in between.
|
|
4201
|
+
const innerLoopNode = ibl.loopNode
|
|
4202
|
+
const iExit = innerLoopNode[2] // (br_if iBrk (eqz (b < count)))
|
|
4203
|
+
const iInc = innerLoopNode[ibl.incIdx] // (local.set b (i32.add b 1)) — scalar, kept
|
|
4204
|
+
const iLabelB = innerBlock[1], iLabelL = innerLoopNode[1]
|
|
4205
|
+
const innerSimd = ['block', iLabelB, ['loop', iLabelL, iExit, ...liftedInner, iInc, ['br', iLabelL]]]
|
|
4206
|
+
const laneCompute = [...laneInit, innerSimd]
|
|
4207
|
+
const epiLane = (k) => [
|
|
4208
|
+
...epiReads.map(v => ['local.set', v, ['f64x2.extract_lane', k, ['local.get', laneMap.get(v)]]]),
|
|
4209
|
+
...epilogue.map(s => bump(s, k)),
|
|
4210
|
+
]
|
|
4211
|
+
const sOut = nm('ob'), sOl = nm('ol')
|
|
4212
|
+
const simdOuter = ['block', sOut, ['loop', sOl,
|
|
4213
|
+
['br_if', sOut, ['i32.eqz', [oExit.cmpOp, bump(['local.get', pxVar], 1), widthBound]]],
|
|
4214
|
+
...laneCompute, ...epiLane(0), ...epiLane(1),
|
|
4215
|
+
...pixelIVs.map(p => ['local.set', p.name, [p.type + '.add', ['local.get', p.name], [p.type + '.const', 2]]]),
|
|
4216
|
+
['br', sOl]]]
|
|
4217
|
+
const wrapper = ['block', nm('w'), ...preamble, simdOuter, ['block', oLabel, loopNode]]
|
|
4218
|
+
return { wrapper, newLocalDecls }
|
|
4219
|
+
}
|
|
4220
|
+
|
|
4221
|
+
// ---- Mixed-lane tone-map (tryToneMap, experimental) ------------------------
|
|
4222
|
+
//
|
|
4223
|
+
// Vectorizes the log-tonemap TAIL shared by fern / bifurcation / attractors:
|
|
4224
|
+
// while (i<n){ let v=dens[i]; if(v>0){ g = trunc(min(log(v+1)*S, 255)) }
|
|
4225
|
+
// px[i] = (255<<24)|(g<<16)|(g<<8)|g }
|
|
4226
|
+
// A flat 1-D loop that loads an i32 density, lifts it to f64 for a log, truncates
|
|
4227
|
+
// back to i32, packs an ARGB word, and stores it — i32 lanes wrapping an f64 ISLAND.
|
|
4228
|
+
// The single-lane-type lift can't carry an f64 intermediate inside an i32 store
|
|
4229
|
+
// (tryVectorize bails on `f64.mul: no lane-pure SIMD mapping for i32`), so this is a
|
|
4230
|
+
// dedicated 2-wide (f64x2) hybrid: load 2 u32 (`v128.load64_zero` → i32x4 low lanes),
|
|
4231
|
+
// `f64x2.convert_low_i32x4_s` into the island, `$math.log_v` + f64x2 arith + clamp,
|
|
4232
|
+
// `i32x4.trunc_sat_f64x2_s_zero` back out, the i32 pack, then a masked
|
|
4233
|
+
// `i64.store` of `i64x2.extract_lane 0` (the low 2 lanes = 2 pixels). 2 pixels/iter.
|
|
4234
|
+
//
|
|
4235
|
+
// BIT-EXACT by construction: each lane runs the scalar op (log_v is the per-lane
|
|
4236
|
+
// extract/repack mirror; the clamp keeps L finite & in [0,255] so `trunc_sat == |0`,
|
|
4237
|
+
// the ±Inf canon is a no-op and is dropped; the pack is element-wise). The conditional
|
|
4238
|
+
// masks are emitted in the SAME lane width as the data they select (`v>0` is i32 and
|
|
4239
|
+
// gates i32 stores/values; the `L>255` clamp is f64 and gates f64) — a width mismatch
|
|
4240
|
+
// bails. No cross-lane reordering, so no ulp drift. Speculatively-evaluated arms are
|
|
4241
|
+
// trap-free (log/convert/mul/min/trunc never trap; there is no div/rem). Gated until
|
|
4242
|
+
// proven across the corpus, then promoted like the stencil/outer-strip wins.
|
|
4243
|
+
|
|
4244
|
+
const _toneStripTee = (n) => isArr(n) && n[0] === 'local.tee' && n.length === 3 ? n[2] : n
|
|
4245
|
+
|
|
4246
|
+
// `(i32.wrap_i64 (i64.trunc_sat_f64_{s,u} X))` or `(i32.trunc_sat_f64_{s,u} X)` — the
|
|
4247
|
+
// f64→i32 `|0` bridge. Returns { inner, signed } (tee on X stripped) or null.
|
|
4248
|
+
function matchTruncF64(expr) {
|
|
4249
|
+
if (!isArr(expr)) return null
|
|
4250
|
+
if (expr[0] === 'i32.wrap_i64' && isArr(expr[1])) {
|
|
4251
|
+
const t = expr[1]
|
|
4252
|
+
if (t[0] === 'i64.trunc_sat_f64_s') return { inner: _toneStripTee(t[1]), signed: true }
|
|
4253
|
+
if (t[0] === 'i64.trunc_sat_f64_u') return { inner: _toneStripTee(t[1]), signed: false }
|
|
4254
|
+
}
|
|
4255
|
+
if (expr[0] === 'i32.trunc_sat_f64_s') return { inner: _toneStripTee(expr[1]), signed: true }
|
|
4256
|
+
if (expr[0] === 'i32.trunc_sat_f64_u') return { inner: _toneStripTee(expr[1]), signed: false }
|
|
4257
|
+
return null
|
|
4258
|
+
}
|
|
4259
|
+
|
|
4260
|
+
// The `|0` of a known-finite f64: `(select (trunc X) (i32.const 0) (f64.ne X' ±Inf))`.
|
|
4261
|
+
// Since the tonemap clamps L into [0,255] before the trunc, the `≠Inf` guard is always
|
|
4262
|
+
// true, so this lowers to a plain `trunc_sat` — returns the inner f64 to truncate.
|
|
4263
|
+
function matchInfCanonTone(sel) {
|
|
4264
|
+
if (!isArr(sel) || sel[0] !== 'select' || sel.length !== 4) return null
|
|
4265
|
+
if (!(isI32Const(sel[2]) && constNum(sel[2]) === 0)) return null
|
|
4266
|
+
const c = sel[3]
|
|
4267
|
+
if (!(isArr(c) && c[0] === 'f64.ne' && isArr(c[2]) && c[2][0] === 'f64.const' && /inf/i.test(String(c[2][1])))) return null
|
|
4268
|
+
const tr = matchTruncF64(sel[1])
|
|
4269
|
+
return tr ? tr.inner : null
|
|
4270
|
+
}
|
|
4271
|
+
|
|
4272
|
+
// `base + (i<<2)` with `base` a loop-invariant array pointer (local or global). Returns
|
|
4273
|
+
// the base node (stride-4 ⇒ load64_zero/i64.store cover exactly 2 consecutive u32) or null.
|
|
4274
|
+
function matchToneAddr(addr, ind) {
|
|
4275
|
+
if (!isArr(addr) || addr[0] !== 'i32.add' || addr.length !== 3) return null
|
|
4276
|
+
const pair = (baseN, offN) => {
|
|
4277
|
+
if (!isArr(baseN) || (baseN[0] !== 'local.get' && baseN[0] !== 'global.get')) return null
|
|
4278
|
+
if (baseN[0] === 'local.get' && baseN[1] === ind) return null
|
|
4279
|
+
if (isArr(offN) && offN[0] === 'i32.shl' && offN.length === 3 && isLocalGet(offN[1], ind) && constNum(offN[2]) === 2) return baseN
|
|
4280
|
+
return null
|
|
4281
|
+
}
|
|
4282
|
+
return pair(addr[1], addr[2]) || pair(addr[2], addr[1])
|
|
4283
|
+
}
|
|
4284
|
+
|
|
4285
|
+
const _toneUnwrapArm = (arm) => { // arm = ['then'|'else', ...stmts]; unwrap a single nested block
|
|
4286
|
+
let body = arm.slice(1)
|
|
4287
|
+
if (body.length === 1 && isArr(body[0]) && body[0][0] === 'block') {
|
|
4288
|
+
const b = body[0]; let i = 1
|
|
4289
|
+
if (typeof b[i] === 'string' && b[i].startsWith('$')) i++
|
|
4290
|
+
if (isArr(b[i]) && b[i][0] === 'result') i++
|
|
4291
|
+
body = b.slice(i)
|
|
4292
|
+
}
|
|
4293
|
+
return body
|
|
4294
|
+
}
|
|
4295
|
+
|
|
4296
|
+
const _toneAppears = (n, name) => isArr(n) && (((n[0] === 'local.get' || n[0] === 'local.set' || n[0] === 'local.tee') && n[1] === name) || n.some(c => _toneAppears(c, name)))
|
|
4297
|
+
|
|
4298
|
+
// First WRITE of `name`: returns { stmtIdx, nested } (nested = inside an `if`) or null.
|
|
4299
|
+
function _toneFirstWrite(body, name) {
|
|
4300
|
+
for (let i = 0; i < body.length; i++) {
|
|
4301
|
+
let found = false, nested = false
|
|
4302
|
+
const w = (n, depth) => {
|
|
4303
|
+
if (found || !isArr(n)) return
|
|
4304
|
+
if ((n[0] === 'local.set' || n[0] === 'local.tee') && n[1] === name) { found = true; nested = depth > 0; return }
|
|
4305
|
+
const d = depth + (n[0] === 'if' ? 1 : 0)
|
|
4306
|
+
for (let j = 1; j < n.length; j++) w(n[j], d)
|
|
4307
|
+
}
|
|
4308
|
+
w(body[i], 0)
|
|
4309
|
+
if (found) return { stmtIdx: i, nested }
|
|
4310
|
+
}
|
|
4311
|
+
return null
|
|
4312
|
+
}
|
|
4313
|
+
|
|
4314
|
+
// Mixed-lane log-tonemap: i32 dens[i] → f64 log → i32 pack → px[i]. See the block comment above.
|
|
4315
|
+
function tryToneMap(bl, fnLocals, freshIdRef, enabled) {
|
|
4316
|
+
if (!enabled || !bl) return null
|
|
4317
|
+
const { incVar, bound, boundLocal, body, preamble } = bl
|
|
4318
|
+
if (!boundLocal && !isI32Const(bound)) return null // bound must be loop-invariant
|
|
4319
|
+
|
|
4320
|
+
// Shape gate: exactly one i32.store + ≥1 i32.load, all at `base+(i<<2)`, plus the f64
|
|
4321
|
+
// island signature (`f64.convert_i32_*`). Any other-width load/store declines. The
|
|
4322
|
+
// f64-convert requirement is what distinguishes this from a plain i32 map (tryVectorize,
|
|
4323
|
+
// which runs earlier and already owns those).
|
|
4324
|
+
let hasConvert = false, storeCount = 0, loadCount = 0, ok = true
|
|
4325
|
+
const scan = (n) => {
|
|
4326
|
+
if (!ok || !isArr(n)) return
|
|
4327
|
+
const o = n[0]
|
|
4328
|
+
if (o === 'f64.convert_i32_s' || o === 'f64.convert_i32_u') hasConvert = true
|
|
4329
|
+
if (o === 'i32.store') { storeCount++; if (!matchToneAddr(n[1], incVar)) ok = false; scan(n[2]); return }
|
|
4330
|
+
if (o === 'i32.load') { loadCount++; if (!matchToneAddr(n[1], incVar)) ok = false; return }
|
|
4331
|
+
if (LOAD_OPS[o] || STORE_OPS[o]) { ok = false; return } // any wider/narrower memop → not this shape
|
|
4332
|
+
for (let i = 1; i < n.length; i++) scan(n[i])
|
|
4333
|
+
}
|
|
4334
|
+
for (const s of body) scan(s)
|
|
4335
|
+
// 1 store (unconditional, attractors) or 2 (the then/else arms of a conditional store,
|
|
4336
|
+
// bifurcation/fern — both write the same pixel and collapse to one masked store).
|
|
4337
|
+
if (!ok || !hasConvert || storeCount < 1 || storeCount > 2 || loadCount < 1) return null
|
|
4338
|
+
if (body.some(hasGlobalSet)) return null
|
|
4339
|
+
|
|
4340
|
+
// Classify locals (mirrors tryVectorize): written ⇒ lane (first access must be a write,
|
|
4341
|
+
// else loop-carried), unwritten ⇒ invariant.
|
|
4342
|
+
const writes = new Set()
|
|
4343
|
+
for (const s of body) collectWrites(s, writes)
|
|
4344
|
+
if (boundLocal && writes.has(boundLocal)) return null
|
|
4345
|
+
const referenced = new Set()
|
|
4346
|
+
const collectRefs = (n) => {
|
|
4347
|
+
if (!isArr(n)) return
|
|
4348
|
+
if ((n[0] === 'local.get' || n[0] === 'local.set' || n[0] === 'local.tee') && typeof n[1] === 'string') referenced.add(n[1])
|
|
4349
|
+
for (let i = 1; i < n.length; i++) collectRefs(n[i])
|
|
4350
|
+
}
|
|
4351
|
+
for (const s of body) collectRefs(s)
|
|
4352
|
+
const localKind = new Map()
|
|
4353
|
+
for (const name of referenced) {
|
|
4354
|
+
if (name === incVar) continue
|
|
4355
|
+
if (writes.has(name)) {
|
|
4356
|
+
let firstKind = null
|
|
4357
|
+
for (const s of body) { const k = firstAccess(s, name); if (k) { firstKind = k; break } }
|
|
4358
|
+
if (firstKind === 'read') return null // loop-carried
|
|
4359
|
+
localKind.set(name, 'lane')
|
|
4360
|
+
} else localKind.set(name, 'invariant')
|
|
4361
|
+
}
|
|
4362
|
+
|
|
4363
|
+
// Liveness gate: a lane local first ASSIGNED inside an `if` is set speculatively
|
|
4364
|
+
// (unconditionally) — sound only if it never leaks past that statement (else a false
|
|
4365
|
+
// lane would read a value scalar never produced). Bail otherwise.
|
|
4366
|
+
for (const [name, kind] of localKind) {
|
|
4367
|
+
if (kind !== 'lane') continue
|
|
4368
|
+
const fw = _toneFirstWrite(body, name)
|
|
4369
|
+
if (fw && fw.nested) {
|
|
4370
|
+
for (let i = 0; i < body.length; i++) if (i !== fw.stmtIdx && _toneAppears(body[i], name)) return null
|
|
4371
|
+
}
|
|
4372
|
+
}
|
|
4373
|
+
|
|
4374
|
+
const ctx = { extraLocals: [], fail: false, failReason: null } // only liftFail (fail/failReason) + freshMask (extraLocals) read it
|
|
4375
|
+
const newLanedLocals = new Map() // origName → { laneName }
|
|
4376
|
+
const toneSetBefore = new Set() // lane locals already assigned (conditional-merge gate)
|
|
4377
|
+
const laned = (name) => { let r = newLanedLocals.get(name); if (!r) { r = { laneName: `${name}__v` }; newLanedLocals.set(name, r) } return r.laneName }
|
|
4378
|
+
const freshMask = () => { const mt = `$__mask${freshIdRef.next++}`; ctx.extraLocals.push(['local', mt, 'v128']); return mt }
|
|
4379
|
+
|
|
4380
|
+
// The ctx-using lifters are NESTED function declarations that CAPTURE the state above (like
|
|
4381
|
+
// scanForLoadsStores) — taking `ctx` as a param instead would make jz's self-host inference
|
|
4382
|
+
// mistype the recursive lifter's `ctx` (the recursive call site can't agree on i32, so it
|
|
4383
|
+
// stays boxed f64 and its callers emit a bad i64.reinterpret_f64). Capturing sidesteps that.
|
|
4384
|
+
|
|
4385
|
+
// Result lane width of a value expr ('i32' | 'f64' | 'x') — keeps a conditional's mask the
|
|
4386
|
+
// SAME width as the data it selects (a mismatch bails).
|
|
4387
|
+
function toneWidth(e) {
|
|
4388
|
+
if (!isArr(e)) return 'x'
|
|
4389
|
+
const o = e[0]
|
|
4390
|
+
if (o === 'f64.const' || o === 'f64.convert_i32_s' || o === 'f64.convert_i32_u' || o === 'call') return 'f64'
|
|
4391
|
+
if (o === 'i32.const' || o === 'i32.wrap_i64' || o === 'i32.trunc_sat_f64_s' || o === 'i32.trunc_sat_f64_u') return 'i32'
|
|
4392
|
+
if (o === 'local.get') return fnLocals.get(e[1]) === 'f64' ? 'f64' : 'i32'
|
|
4393
|
+
if (o === 'select') return toneWidth(e[1])
|
|
4394
|
+
if (o === 'if') return isArr(e[1]) && e[1][1] === 'f64' ? 'f64' : 'i32'
|
|
4395
|
+
if (o.startsWith('f64.')) return 'f64'
|
|
4396
|
+
if (o.startsWith('i32.')) return 'i32'
|
|
4397
|
+
return 'x'
|
|
4398
|
+
}
|
|
4399
|
+
|
|
4400
|
+
// Lift one value expression to v128. Result lane type comes from the op (f64.* → f64x2,
|
|
4401
|
+
// i32.* → i32x4; convert/trunc bridge between them). Bit-exact per lane.
|
|
4402
|
+
function liftV(expr) {
|
|
4403
|
+
if (!isArr(expr)) return liftFail(ctx, 'tonemap: non-expression operand')
|
|
4404
|
+
const op = expr[0]
|
|
4405
|
+
if ((op === 'f64.convert_i32_s' || op === 'f64.convert_i32_u') && expr.length === 2) {
|
|
4406
|
+
const inner = expr[1]
|
|
4407
|
+
if (isArr(inner) && (inner[0] === 'global.get' || inner[0] === 'i32.const' ||
|
|
4408
|
+
(inner[0] === 'local.get' && localKind.get(inner[1]) === 'invariant')))
|
|
4409
|
+
return ['f64x2.splat', expr] // loop-invariant convert: scalar-then-splat, bit-exact
|
|
4410
|
+
const a = liftV(inner); if (ctx.fail) return null
|
|
4411
|
+
return [op === 'f64.convert_i32_s' ? 'f64x2.convert_low_i32x4_s' : 'f64x2.convert_low_i32x4_u', a]
|
|
4412
|
+
}
|
|
4413
|
+
const tr = matchTruncF64(expr) // f64 → i32 `|0` bridge
|
|
4414
|
+
if (tr) { const a = liftV(tr.inner); if (ctx.fail) return null; return [tr.signed ? 'i32x4.trunc_sat_f64x2_s_zero' : 'i32x4.trunc_sat_f64x2_u_zero', a] }
|
|
4415
|
+
if (op === 'i32.load' && expr.length === 2) return ['v128.load64_zero', expr[1]] // address kept scalar
|
|
4416
|
+
if (op === 'i32.const') return ['i32x4.splat', expr]
|
|
4417
|
+
if (op === 'f64.const') return ['f64x2.splat', expr]
|
|
4418
|
+
if (op === 'local.get' && typeof expr[1] === 'string') {
|
|
4419
|
+
const name = expr[1], kind = localKind.get(name)
|
|
4420
|
+
if (kind === 'lane') return ['local.get', laned(name)]
|
|
4421
|
+
if (kind === 'invariant') return [fnLocals.get(name) === 'f64' ? 'f64x2.splat' : 'i32x4.splat', expr]
|
|
4422
|
+
return liftFail(ctx, `tonemap: ${name} address/induction var used as lane data`)
|
|
4423
|
+
}
|
|
4424
|
+
if (op === 'call' && PPC_CALL2[expr[1]]) { // transcendental → its 2-wide mirror
|
|
4425
|
+
const args = []
|
|
4426
|
+
for (let i = 2; i < expr.length; i++) { const a = liftV(expr[i]); if (ctx.fail) return null; args.push(a) }
|
|
4427
|
+
return ['call', PPC_CALL2[expr[1]], ...args]
|
|
4428
|
+
}
|
|
4429
|
+
if (op === 'select' && expr.length === 4) {
|
|
4430
|
+
const inf = matchInfCanonTone(expr)
|
|
4431
|
+
if (inf) { const a = liftV(inf); if (ctx.fail) return null; return ['i32x4.trunc_sat_f64x2_s_zero', a] }
|
|
4432
|
+
return liftSel(expr[1], expr[2], expr[3])
|
|
4433
|
+
}
|
|
4434
|
+
if (op === 'if' && isArr(expr[1]) && expr[1][0] === 'result' && isArr(expr[3]) && expr[3][0] === 'then' && isArr(expr[4]) && expr[4][0] === 'else')
|
|
4435
|
+
return liftSel(expr[3][1], expr[4][1], expr[2])
|
|
4436
|
+
const insl = op.startsWith('f64.') ? 'f64' : (op.startsWith('i32.') ? 'i32' : 'x')
|
|
4437
|
+
const entry = LANE_PURE[insl]?.get(op)
|
|
4438
|
+
if (entry) {
|
|
4439
|
+
const a = liftV(expr[1]); if (ctx.fail) return null
|
|
4440
|
+
if (entry.shamtScalar) {
|
|
4441
|
+
const b = expr[2]
|
|
4442
|
+
if (!isI32Const(b) && !(isArr(b) && b[0] === 'local.get' && localKind.get(b[1]) === 'invariant'))
|
|
4443
|
+
return liftFail(ctx, `tonemap: ${op}: shift amount not constant/invariant`)
|
|
4444
|
+
return [entry.simd, a, b]
|
|
4445
|
+
}
|
|
4446
|
+
if (expr.length === 2) return [entry.simd, a]
|
|
4447
|
+
const b = liftV(expr[2]); if (ctx.fail) return null
|
|
4448
|
+
return [entry.simd, a, b]
|
|
4449
|
+
}
|
|
4450
|
+
return liftFail(ctx, `tonemap: ${op}: no lane mapping`)
|
|
4451
|
+
}
|
|
4452
|
+
|
|
4453
|
+
// Lane-comparison mask, required to match the selected data width (a LOCAL `c` — never
|
|
4454
|
+
// reassign the param). Returns the mask expr or null on bail.
|
|
4455
|
+
function liftMask(cond, dataTy) {
|
|
4456
|
+
let c = cond
|
|
4457
|
+
if (isArr(c) && c[0] === 'i32.ne' && isI32Const(c[2]) && c[2][1] === 0) c = c[1]
|
|
4458
|
+
if (!isArr(c) || c.length !== 3) return liftFail(ctx, 'tonemap: condition is not a comparison')
|
|
4459
|
+
const condTy = c[0].startsWith('f64.') ? 'f64' : (c[0].startsWith('i32.') ? 'i32' : 'x')
|
|
4460
|
+
if (condTy !== dataTy) return liftFail(ctx, `tonemap: mask width ${condTy} ≠ data width ${dataTy}`)
|
|
4461
|
+
const cmp = LANE_COMPARE[condTy]?.[c[0]]
|
|
4462
|
+
if (!cmp) return liftFail(ctx, `tonemap: ${c[0]}: not a lane comparison`)
|
|
4463
|
+
const ca = liftV(c[1]); if (ctx.fail) return null
|
|
4464
|
+
const cb = liftV(c[2]); if (ctx.fail) return null
|
|
4465
|
+
return [cmp, ca, cb]
|
|
4466
|
+
}
|
|
4467
|
+
|
|
4468
|
+
// `cond ? a : b` → bitselect(a, b, mask(cond)); mask in the branch's lane width.
|
|
4469
|
+
function liftSel(a, b, cond) {
|
|
4470
|
+
const m = liftMask(cond, toneWidth(a)); if (ctx.fail) return null
|
|
4471
|
+
const av = liftV(a); if (ctx.fail) return null
|
|
4472
|
+
const bv = liftV(b); if (ctx.fail) return null
|
|
4473
|
+
const mt = freshMask()
|
|
4474
|
+
return ['block', ['result', 'v128'], ['local.set', mt, m], ['v128.bitselect', av, bv, ['local.get', mt]]]
|
|
4475
|
+
}
|
|
4476
|
+
|
|
4477
|
+
// Lift one statement, pushing v128 stmts into `out`. Sets ctx.fail on any bail.
|
|
4478
|
+
function liftS(stmt, out) {
|
|
4479
|
+
if (!isArr(stmt)) { liftFail(ctx, 'tonemap: non-array statement'); return }
|
|
4480
|
+
const op = stmt[0]
|
|
4481
|
+
if (op === 'block') {
|
|
4482
|
+
let i = 1
|
|
4483
|
+
if (typeof stmt[i] === 'string' && stmt[i].startsWith('$')) i++
|
|
4484
|
+
if (isArr(stmt[i]) && stmt[i][0] === 'result') i++
|
|
4485
|
+
for (const s of stmt.slice(i)) { liftS(s, out); if (ctx.fail) return }
|
|
4486
|
+
return
|
|
4487
|
+
}
|
|
4488
|
+
if (op === 'local.set' && typeof stmt[1] === 'string' && stmt.length === 3) {
|
|
4489
|
+
const name = stmt[1]
|
|
4490
|
+
if (localKind.get(name) !== 'lane') { liftFail(ctx, `tonemap: set of non-lane ${name}`); return }
|
|
4491
|
+
const v = liftV(stmt[2]); if (ctx.fail) return
|
|
4492
|
+
out.push(['local.set', laned(name), v]); toneSetBefore.add(name); return
|
|
4493
|
+
}
|
|
4494
|
+
if (STORE_OPS[op]) { // i32.store ADDR VAL → masked i64.store of the low 2 lanes (2 pixels)
|
|
4495
|
+
if (op !== 'i32.store' || stmt.length !== 3) { liftFail(ctx, `tonemap: unsupported store ${op}`); return }
|
|
4496
|
+
const v = liftV(stmt[2]); if (ctx.fail) return
|
|
4497
|
+
out.push(['i64.store', stmt[1], ['i64x2.extract_lane', 0, v]]); return
|
|
4498
|
+
}
|
|
4499
|
+
if (op === 'if' && isArr(stmt[2]) && stmt[2][0] === 'then') {
|
|
4500
|
+
const hasElse = isArr(stmt[3]) && stmt[3][0] === 'else'
|
|
4501
|
+
const thenStmts = _toneUnwrapArm(stmt[2])
|
|
4502
|
+
const elseStmts = hasElse ? _toneUnwrapArm(stmt[3]) : null
|
|
4503
|
+
const thenLast = thenStmts[thenStmts.length - 1]
|
|
4504
|
+
const elseLast = elseStmts && elseStmts[elseStmts.length - 1]
|
|
4505
|
+
const thenStore = isArr(thenLast) && STORE_OPS[thenLast[0]] && thenLast.length === 3
|
|
4506
|
+
const elseStore = elseLast && isArr(elseLast) && STORE_OPS[elseLast[0]] && elseLast.length === 3
|
|
4507
|
+
// (a) Conditional STORE — both arms (or then-only) end in a store to the same address.
|
|
4508
|
+
if (thenStore && (elseStore || !hasElse)) {
|
|
4509
|
+
if (elseStore && JSON.stringify(thenLast[1]) !== JSON.stringify(elseLast[1])) { liftFail(ctx, 'tonemap: arms store to different addresses'); return }
|
|
4510
|
+
for (const s of thenStmts.slice(0, -1)) { liftS(s, out); if (ctx.fail) return }
|
|
4511
|
+
if (hasElse) for (const s of elseStmts.slice(0, -1)) { liftS(s, out); if (ctx.fail) return }
|
|
4512
|
+
const thenVal = liftV(thenLast[2]); if (ctx.fail) return
|
|
4513
|
+
const elseVal = elseStore ? liftV(elseLast[2]) : ['v128.load64_zero', thenLast[1]]
|
|
4514
|
+
if (ctx.fail) return
|
|
4515
|
+
const m = liftMask(stmt[1], 'i32'); if (ctx.fail) return
|
|
4516
|
+
const mt = freshMask()
|
|
4517
|
+
out.push(['local.set', mt, m],
|
|
4518
|
+
['i64.store', thenLast[1], ['i64x2.extract_lane', 0, ['v128.bitselect', thenVal, elseVal, ['local.get', mt]]]])
|
|
4519
|
+
return
|
|
4520
|
+
}
|
|
4521
|
+
// (b) Conditional VALUE update — `if (cond) { L = …; … }` (no else) updating lane locals.
|
|
4522
|
+
if (!hasElse) {
|
|
4523
|
+
for (const s of thenStmts) {
|
|
4524
|
+
if (!(isArr(s) && s[0] === 'local.set' && typeof s[1] === 'string' && s.length === 3 && localKind.get(s[1]) === 'lane')) {
|
|
4525
|
+
liftFail(ctx, 'tonemap: no-else arm is not a lane update'); return
|
|
4526
|
+
}
|
|
4527
|
+
const name = s[1], ty = fnLocals.get(name) === 'f64' ? 'f64' : 'i32'
|
|
4528
|
+
const xv = liftV(s[2]); if (ctx.fail) return
|
|
4529
|
+
const ln = laned(name)
|
|
4530
|
+
if (toneSetBefore.has(name)) {
|
|
4531
|
+
const m = liftMask(stmt[1], ty); if (ctx.fail) return // conditional merge
|
|
4532
|
+
const mt = freshMask()
|
|
4533
|
+
out.push(['local.set', mt, m], ['local.set', ln, ['v128.bitselect', xv, ['local.get', ln], ['local.get', mt]]])
|
|
4534
|
+
} else {
|
|
4535
|
+
out.push(['local.set', ln, xv]); toneSetBefore.add(name) // first, unconditional (liveness-gated)
|
|
4536
|
+
}
|
|
4537
|
+
}
|
|
4538
|
+
return
|
|
4539
|
+
}
|
|
4540
|
+
liftFail(ctx, 'tonemap: unsupported if shape'); return
|
|
4541
|
+
}
|
|
4542
|
+
liftFail(ctx, `tonemap: unsupported statement ${op}`)
|
|
4543
|
+
}
|
|
4544
|
+
|
|
4545
|
+
const lifted = []
|
|
4546
|
+
for (const s of body) { liftS(s, lifted); if (ctx.fail) return null }
|
|
4547
|
+
if (!lifted.length) return null
|
|
4548
|
+
|
|
4549
|
+
// 2-wide SIMD wrapper (LANES=2, the f64x2 island's width). Scalar tail = original block.
|
|
4550
|
+
const LANES = 2
|
|
4551
|
+
const id = freshIdRef.next++
|
|
4552
|
+
const simdBoundName = `$__simd_bound${id}`, simdBrk = `$__simd_brk${id}`, simdLoop = `$__simd_loop${id}`
|
|
4553
|
+
const boundExpr = boundLocal ? ['local.get', boundLocal] : bound
|
|
4554
|
+
const simdBlock = ['block', simdBrk,
|
|
4555
|
+
['loop', simdLoop,
|
|
4556
|
+
['br_if', simdBrk, ['i32.eqz', ['i32.lt_s', ['local.get', incVar], ['local.get', simdBoundName]]]],
|
|
4557
|
+
...lifted,
|
|
4558
|
+
['local.set', incVar, ['i32.add', ['local.get', incVar], ['i32.const', LANES]]],
|
|
4559
|
+
['br', simdLoop]]]
|
|
4560
|
+
const boundSetup = ['local.set', simdBoundName, ['i32.and', boundExpr, ['i32.const', -LANES]]]
|
|
4561
|
+
const wrapper = ['block', ...preamble.map(cloneNode), boundSetup, simdBlock, bl.blockNode]
|
|
4562
|
+
const newLocalDecls = [
|
|
4563
|
+
['local', simdBoundName, 'i32'],
|
|
4564
|
+
...[...newLanedLocals.values()].map(({ laneName }) => ['local', laneName, 'v128']),
|
|
4565
|
+
...ctx.extraLocals,
|
|
4566
|
+
]
|
|
4567
|
+
return { wrapper, newLocalDecls }
|
|
4568
|
+
}
|
|
4569
|
+
|
|
4570
|
+
// ---- Pass entry ------------------------------------------------------------
|
|
4571
|
+
|
|
4572
|
+
/**
|
|
4573
|
+
* Walk a function looking for vectorizable (block (loop)) pairs, in-place.
|
|
4574
|
+
* Adds new locals to the function header.
|
|
4575
|
+
*/
|
|
4576
|
+
export function vectorizeLaneLocal(fn, multiAcc = false, relaxedFma = false, blurMP = true, whyNot = false, stencil = false, outerStrip = false, pureFuncMap = null, toneMap = false) {
|
|
4577
|
+
if (!isArr(fn) || fn[0] !== 'func') return
|
|
4578
|
+
const bodyStart = findBodyStart(fn)
|
|
4579
|
+
if (bodyStart < 0) return
|
|
4580
|
+
const fnName = typeof fn[1] === 'string' ? fn[1] : '(anon)'
|
|
4581
|
+
let whyNotN = 0
|
|
4582
|
+
|
|
4583
|
+
// Build local-name → wasm-type map.
|
|
4584
|
+
const fnLocals = new Map()
|
|
4585
|
+
for (let i = 2; i < bodyStart; i++) {
|
|
4586
|
+
const d = fn[i]
|
|
4587
|
+
if (isArr(d) && d[0] === 'local' && typeof d[1] === 'string' && typeof d[2] === 'string') {
|
|
4588
|
+
fnLocals.set(d[1], d[2])
|
|
4589
|
+
} else if (isArr(d) && d[0] === 'param' && typeof d[1] === 'string' && typeof d[2] === 'string') {
|
|
4590
|
+
fnLocals.set(d[1], d[2])
|
|
4591
|
+
}
|
|
4592
|
+
}
|
|
4593
|
+
|
|
4594
|
+
const freshIdRef = { next: 0 }
|
|
4595
|
+
const newLocalDeclsAll = []
|
|
4596
|
+
|
|
4597
|
+
vectorizeStraightLineF64DotPairsIn(fn, fnLocals, freshIdRef, newLocalDeclsAll, relaxedFma)
|
|
1848
4598
|
|
|
1849
4599
|
// Walk body recursively. Process inner-most matches first (post-order)
|
|
1850
4600
|
// so we don't try to vectorize an outer loop whose inner is the lane-local one.
|
|
@@ -1855,20 +4605,56 @@ export function vectorizeLaneLocal(fn) {
|
|
|
1855
4605
|
if (isArr(node[i])) walk(node, i)
|
|
1856
4606
|
}
|
|
1857
4607
|
if (node[0] === 'block') {
|
|
1858
|
-
|
|
1859
|
-
|
|
1860
|
-
|
|
1861
|
-
|
|
1862
|
-
|
|
4608
|
+
if (_whyNotActive) _whyNotReason = null
|
|
4609
|
+
// Recognition layer: match the canonical (block (loop)) scaffold ONCE; the
|
|
4610
|
+
// inner-scaffold lifters (memcpy/map/reduce/map-reduce/byte-scan) consume the
|
|
4611
|
+
// descriptor instead of each re-matching. The outer-pixel + special-shape
|
|
4612
|
+
// recognizers (divergent-escape, ramp-map, blur, channel-reduce, per-pixel)
|
|
4613
|
+
// do their own matching on the raw node. Order is preserved exactly — it is
|
|
4614
|
+
// load-bearing (first match wins).
|
|
4615
|
+
const bl = matchBlockLoop(node, { allowPreamble: true })
|
|
4616
|
+
let r = tryDivergentEscapeVectorize(node, fnLocals, freshIdRef)
|
|
4617
|
+
?? tryMemCopyFill(bl, fnLocals, freshIdRef)
|
|
4618
|
+
?? tryVectorize(bl, fnLocals, freshIdRef)
|
|
4619
|
+
?? tryReduceVectorize(bl, fnLocals, freshIdRef, multiAcc)
|
|
4620
|
+
?? tryMapReduceVectorize(bl, fnLocals, freshIdRef)
|
|
4621
|
+
?? tryStencil(node, fnLocals, freshIdRef, stencil)
|
|
4622
|
+
?? tryRampMap(node, fnLocals, freshIdRef)
|
|
4623
|
+
?? (blurMP ? tryBlurMultiPixel(node, fnLocals, freshIdRef) : null)
|
|
4624
|
+
?? tryChannelReduce(node, fnLocals, freshIdRef)
|
|
4625
|
+
?? tryByteScan(bl, fnLocals, freshIdRef)
|
|
4626
|
+
?? tryPerPixelColor(node, fnLocals, freshIdRef, pureFuncMap)
|
|
4627
|
+
?? tryOuterStrip(node, fnLocals, freshIdRef, outerStrip)
|
|
4628
|
+
?? tryToneMap(bl, fnLocals, freshIdRef, toneMap)
|
|
4629
|
+
// --why-not-simd: a canonical loop-shaped candidate that no SIMD pass took.
|
|
4630
|
+
// Reported BEFORE the scalar strength-reduce fallback (which fires on most
|
|
4631
|
+
// affine loops and would otherwise mask "didn't vectorize"). Diagnostic only.
|
|
4632
|
+
if (!r && _whyNotActive && (bl || matchOuterPixelLoop(node))) {
|
|
4633
|
+
whyNotN++
|
|
4634
|
+
warn('simd-why-not',
|
|
4635
|
+
`${fnName}: loop #${whyNotN} not vectorized — ${_whyNotReason || 'no SIMD-liftable shape (loop-carried dependency, non-affine address, or unsupported control flow)'}`,
|
|
4636
|
+
{ fn: `${fnName}#${whyNotN}` })
|
|
4637
|
+
}
|
|
4638
|
+
// Scalar IV strength-reduction is a non-SIMD fallback; it may still fire.
|
|
4639
|
+
r = r ?? tryStrengthReduceIV(bl, fnLocals, freshIdRef)
|
|
1863
4640
|
if (r) {
|
|
1864
4641
|
parent[idx] = r.wrapper
|
|
1865
4642
|
newLocalDeclsAll.push(...r.newLocalDecls)
|
|
1866
4643
|
}
|
|
1867
4644
|
}
|
|
1868
4645
|
}
|
|
4646
|
+
_whyNotActive = whyNot
|
|
1869
4647
|
for (let i = bodyStart; i < fn.length; i++) walk(fn, i)
|
|
4648
|
+
_whyNotActive = false
|
|
1870
4649
|
|
|
1871
4650
|
if (newLocalDeclsAll.length) {
|
|
1872
|
-
|
|
4651
|
+
// Sibling loops (and the straight-line dot pass) can each lift the SAME source
|
|
4652
|
+
// local to an identically-named `$name__v` v128 scratch. Post-order vectorizes
|
|
4653
|
+
// innermost-first and an outer loop bails once its inner became a wrapper block,
|
|
4654
|
+
// so no two NESTED loops ever share a lift — every collision is between
|
|
4655
|
+
// SEQUENTIAL loops, where one shared scratch is correct (each writes its lanes
|
|
4656
|
+
// before reading). Declaring a local twice is invalid wasm ("duplicate local"),
|
|
4657
|
+
// so keep one decl per name (all dups are the identical `['local', name, 'v128']`).
|
|
4658
|
+
fn.splice(bodyStart, 0, ...new Map(newLocalDeclsAll.map(d => [d[1], d])).values())
|
|
1873
4659
|
}
|
|
1874
4660
|
}
|