jz 0.5.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +315 -318
- package/bench/README.md +369 -0
- package/bench/bench.svg +102 -0
- package/cli.js +104 -30
- package/dist/interop.js +1 -0
- package/dist/jz.js +6024 -0
- package/index.d.ts +126 -0
- package/index.js +362 -84
- package/interop.js +159 -186
- package/jz.svg +5 -0
- package/jzify/arguments.js +97 -0
- package/jzify/bundler.js +382 -0
- package/jzify/classes.js +328 -0
- package/jzify/hoist-vars.js +177 -0
- package/jzify/index.js +51 -0
- package/jzify/names.js +37 -0
- package/jzify/switch.js +106 -0
- package/jzify/transform.js +349 -0
- package/layout.js +184 -0
- package/module/array.js +377 -176
- package/module/collection.js +639 -145
- package/module/console.js +55 -43
- package/module/core.js +264 -153
- package/module/date.js +15 -3
- package/module/function.js +73 -6
- package/module/index.js +2 -1
- package/module/json.js +226 -61
- package/module/math.js +551 -187
- package/module/number.js +327 -60
- package/module/object.js +474 -184
- package/module/regex.js +255 -25
- package/module/schema.js +24 -6
- package/module/simd.js +117 -0
- package/module/string.js +621 -226
- package/module/symbol.js +1 -1
- package/module/timer.js +9 -14
- package/module/typedarray.js +459 -73
- package/package.json +58 -14
- package/src/abi/index.js +39 -23
- package/src/abi/string.js +38 -41
- package/src/ast.js +460 -0
- package/src/autoload.js +29 -24
- package/src/bridge.js +111 -0
- package/src/compile/analyze-scans.js +824 -0
- package/src/compile/analyze.js +1600 -0
- package/src/compile/emit-assign.js +411 -0
- package/src/compile/emit.js +3512 -0
- package/src/compile/flow-types.js +103 -0
- package/src/{compile.js → compile/index.js} +778 -128
- package/src/{infer.js → compile/infer.js} +62 -98
- package/src/compile/loop-divmod.js +155 -0
- package/src/{narrow.js → compile/narrow.js} +409 -106
- package/src/compile/peel-stencil.js +261 -0
- package/src/compile/plan/advise.js +370 -0
- package/src/compile/plan/common.js +150 -0
- package/src/compile/plan/index.js +122 -0
- package/src/compile/plan/inline.js +682 -0
- package/src/compile/plan/literals.js +1199 -0
- package/src/compile/plan/loops.js +472 -0
- package/src/compile/plan/scope.js +649 -0
- package/src/compile/program-facts.js +423 -0
- package/src/ctx.js +220 -64
- package/src/ir.js +589 -172
- package/src/kind-traits.js +132 -0
- package/src/kind.js +524 -0
- package/src/op-policy.js +57 -0
- package/src/{optimize.js → optimize/index.js} +1432 -473
- package/src/optimize/vectorize.js +4660 -0
- package/src/param-reps.js +65 -0
- package/src/parse.js +44 -0
- package/src/{prepare.js → prepare/index.js} +649 -205
- package/src/prepare/lift-iife.js +149 -0
- package/src/reps.js +116 -0
- package/src/resolve.js +12 -3
- package/src/static.js +208 -0
- package/src/type.js +651 -0
- package/src/{assemble.js → wat/assemble.js} +275 -55
- package/src/wat/optimize.js +3938 -0
- package/transform.js +21 -0
- package/wasi.js +47 -5
- package/src/analyze.js +0 -3818
- package/src/emit.js +0 -3040
- package/src/jzify.js +0 -1580
- package/src/plan.js +0 -2132
- package/src/vectorize.js +0 -1088
- /package/src/{codegen.js → wat/codegen.js} +0 -0
|
@@ -0,0 +1,4660 @@
|
|
|
1
|
+
import { findBodyStart } from '../ir.js'
|
|
2
|
+
import { warn } from '../ctx.js'
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Lane-local SIMD-128 vectorizer.
|
|
6
|
+
*
|
|
7
|
+
* Recognizes inner loops of shape:
|
|
8
|
+
* for (let i = 0; i < N; i++) arr[i] = f(arr[i], …)
|
|
9
|
+
* where every body op is "lane-pure" — its k-th lane output depends only
|
|
10
|
+
* on k-th lane inputs. Lifts the body to SIMD-128, prefixed before the
|
|
11
|
+
* original (now tail) loop. Original loop runs the remainder.
|
|
12
|
+
*
|
|
13
|
+
* Design:
|
|
14
|
+
* • Lane-purity is a structural property, not a benchmark match. The op
|
|
15
|
+
* whitelist is the single source of truth (one entry per (lane-type, op)).
|
|
16
|
+
* • Lift is mechanical. The recognizer either matches the structure — in
|
|
17
|
+
* which case lifting is unambiguous — or skips. No bench-specific
|
|
18
|
+
* heuristics.
|
|
19
|
+
* • Tail loop is the original WAT, untouched. If anything regresses the
|
|
20
|
+
* SIMD recognizer just doesn't match, never miscompiles.
|
|
21
|
+
*
|
|
22
|
+
* Match conditions:
|
|
23
|
+
* 1. (block $brk (loop $L (br_if $brk !cond) BODY (i = i+1) (br $L)))
|
|
24
|
+
* 2. cond is `(i32.lt_s i BOUND)` or `i32.lt_u`; BOUND is loop-invariant.
|
|
25
|
+
* 3. All loads/stores in BODY use address `(add base (shl i K))` where
|
|
26
|
+
* base is loop-invariant and K matches the elem stride. Optional
|
|
27
|
+
* enclosing `local.tee` is allowed (and reused).
|
|
28
|
+
* 4. All loads share the same opcode → defines lane type.
|
|
29
|
+
* 5. All other ops in BODY are in the lane-pure whitelist for that type.
|
|
30
|
+
* 6. Each non-induction local in BODY is either purely loop-invariant
|
|
31
|
+
* (only read) or purely lane-local (first action is a write). Never
|
|
32
|
+
* both — that's a loop-carried scalar (reduction / stencil) → bail.
|
|
33
|
+
*
|
|
34
|
+
* Lift produces, before the original block:
|
|
35
|
+
* (local.set $__simd_bound{N} (i32.and BOUND (i32.const ~(LANES-1))))
|
|
36
|
+
* (block $__simd_brk{N}
|
|
37
|
+
* (loop $__simd_loop{N}
|
|
38
|
+
* (br_if $__simd_brk{N} (i32.eqz (i32.lt_s i $__simd_bound{N})))
|
|
39
|
+
* <body lifted op-by-op; lane-local locals routed to v128 shadows>
|
|
40
|
+
* (local.set $i (i32.add i (i32.const LANES)))
|
|
41
|
+
* (br $__simd_loop{N})))
|
|
42
|
+
*
|
|
43
|
+
* The original block runs immediately after with i pre-advanced; its own
|
|
44
|
+
* `i < BOUND` guard handles the tail.
|
|
45
|
+
*/
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
const isArr = n => Array.isArray(n)
|
|
50
|
+
|
|
51
|
+
const exprEq = (a, b) => JSON.stringify(a) === JSON.stringify(b)
|
|
52
|
+
const localGetName = n => isArr(n) && n[0] === 'local.get' && typeof n[1] === 'string' ? n[1] : null
|
|
53
|
+
const f64Zero = n => isArr(n) && n[0] === 'f64.const' && Number(n[1]) === 0
|
|
54
|
+
|
|
55
|
+
// jz wraps every NaN-producing float builtin (Math.sqrt/min/max/…) in a
|
|
56
|
+
// canonicalizing select so a non-canonical NaN never crosses to JS:
|
|
57
|
+
// (select C X (T.ne X X)) — "use C where X is NaN, else X".
|
|
58
|
+
// The condition `X != X` is true iff X is NaN, so this shape is unambiguously
|
|
59
|
+
// the canonicalization idiom. C is the canonical-NaN value, materialized either
|
|
60
|
+
// inline (T.const) or hoisted into a const-pool global (global.get $__fcN) when
|
|
61
|
+
// reused. We splat C verbatim — faithful regardless of what C holds — so the
|
|
62
|
+
// recognizer never needs to resolve the global's value.
|
|
63
|
+
const isSplatConst = (n, constOp) =>
|
|
64
|
+
isArr(n) && (n[0] === constOp || n[0] === 'global.get')
|
|
65
|
+
|
|
66
|
+
// Match `(select C X (T.ne X X))`. Returns { val: X, C } or null.
|
|
67
|
+
function matchCanonSelect(sel, laneType) {
|
|
68
|
+
if (!isArr(sel) || sel[0] !== 'select') return null
|
|
69
|
+
const C = sel[1], val = sel[2], cond = sel[3]
|
|
70
|
+
const neOp = laneType === 'f32' ? 'f32.ne' : 'f64.ne'
|
|
71
|
+
if (!isSplatConst(C, LANE_INFO[laneType].constOp)) return null
|
|
72
|
+
if (!(isArr(cond) && cond[0] === neOp && exprEq(cond[1], val) && exprEq(cond[2], val))) return null
|
|
73
|
+
return { val, C }
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
// Replace every `(local.tee N v)` with `(local.get N)` so a value that tee's its
|
|
77
|
+
// address in one place (the comparison) and reloads it in another (the chosen branch)
|
|
78
|
+
// compares structurally equal. Used only for matching — emission keeps the tee.
|
|
79
|
+
function normTee(n) {
|
|
80
|
+
if (!isArr(n)) return n
|
|
81
|
+
if (n[0] === 'local.tee' && n.length === 3) return ['local.get', n[1]]
|
|
82
|
+
return n.map(normTee)
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
// Recognize an integer min/max reduction body. WASM has no scalar i32.min/max, so
|
|
86
|
+
// `m = max(m, a[i])` — written `Math.max(m,a[i])|0` or `a[i]>m?a[i]:m` — lowers, after
|
|
87
|
+
// the ToInt32-through-`?:` fold, to a select-shaped body:
|
|
88
|
+
// (local.set m (if (result i32) COND (then BR_T) (else BR_E))) [or the (select …) form]
|
|
89
|
+
// where {BR_T,BR_E} = {laneLoad, m} and COND is a signed i32 comparison of the two.
|
|
90
|
+
// Returns { exprNode, isMax } — exprNode is the lane expr carrying the address tee (fed
|
|
91
|
+
// to liftExprV); null when not a clean min/max. All four comparison directions × two
|
|
92
|
+
// branch orderings collapse to `isMax` below. gt/ge (and lt/le) are equivalent for the
|
|
93
|
+
// RESULT — equal operands tie to the same value — so only the direction axis matters.
|
|
94
|
+
function matchIntMinMaxReduce(rhs, accName) {
|
|
95
|
+
if (!isArr(rhs)) return null
|
|
96
|
+
let cond, T, E
|
|
97
|
+
if (rhs[0] === 'if') {
|
|
98
|
+
let i = 1
|
|
99
|
+
if (!(isArr(rhs[i]) && rhs[i][0] === 'result' && rhs[i][1] === 'i32')) return null
|
|
100
|
+
i++
|
|
101
|
+
if (rhs.length !== i + 3) return null
|
|
102
|
+
cond = rhs[i]
|
|
103
|
+
const thenB = rhs[i + 1], elseB = rhs[i + 2]
|
|
104
|
+
if (!(isArr(thenB) && thenB[0] === 'then' && thenB.length === 2)) return null
|
|
105
|
+
if (!(isArr(elseB) && elseB[0] === 'else' && elseB.length === 2)) return null
|
|
106
|
+
T = thenB[1]; E = elseB[1]
|
|
107
|
+
} else if (rhs[0] === 'select' && rhs.length === 4) {
|
|
108
|
+
T = rhs[1]; E = rhs[2]; cond = rhs[3] // (select a b c) = a if c else b
|
|
109
|
+
} else return null
|
|
110
|
+
// Which branch is the accumulator, which is the lane EXPR.
|
|
111
|
+
let exprBr, takeExprWhenTrue
|
|
112
|
+
if (isLocalGet(E, accName) && !isLocalGet(T, accName)) { exprBr = T; takeExprWhenTrue = true }
|
|
113
|
+
else if (isLocalGet(T, accName) && !isLocalGet(E, accName)) { exprBr = E; takeExprWhenTrue = false }
|
|
114
|
+
else return null
|
|
115
|
+
// Strip a boolean-normalizing `(i32.ne X 0)` around the comparison (as liftExprV does).
|
|
116
|
+
let cmp = cond
|
|
117
|
+
if (isArr(cmp) && cmp[0] === 'i32.ne' && isI32Const(cmp[2]) && cmp[2][1] === 0) cmp = cmp[1]
|
|
118
|
+
if (!isArr(cmp) || cmp.length !== 3) return null
|
|
119
|
+
const dir = { 'i32.gt_s': 'gt', 'i32.ge_s': 'gt', 'i32.lt_s': 'lt', 'i32.le_s': 'lt' }[cmp[0]]
|
|
120
|
+
if (!dir) return null
|
|
121
|
+
// Comparison operands must be {acc, EXPR}; take the non-acc side as the canonical lane
|
|
122
|
+
// expr (it carries the address tee). exprIsLeftOfCmp records its position.
|
|
123
|
+
let condExpr, exprIsLeftOfCmp
|
|
124
|
+
if (isLocalGet(cmp[2], accName)) { condExpr = cmp[1]; exprIsLeftOfCmp = true }
|
|
125
|
+
else if (isLocalGet(cmp[1], accName)) { condExpr = cmp[2]; exprIsLeftOfCmp = false }
|
|
126
|
+
else return null
|
|
127
|
+
// The compared expr and the chosen branch must be the SAME lane (tee vs reload aside).
|
|
128
|
+
if (!exprEq(normTee(condExpr), normTee(exprBr))) return null
|
|
129
|
+
// cond true ⟺ EXPR > acc ⇒ picking EXPR-when-true is a max; picking-when-false a min.
|
|
130
|
+
const predExprGreater = dir === 'gt' ? exprIsLeftOfCmp : !exprIsLeftOfCmp
|
|
131
|
+
return { exprNode: condExpr, isMax: takeExprWhenTrue === predExprGreater }
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
// Match the un-flattened canon, emitted when a Math.* result feeds another op
|
|
135
|
+
// in expression position:
|
|
136
|
+
// (block (result T) (local.set $t CORE) (select C (local.get $t) (T.ne …)))
|
|
137
|
+
// Returns { core: CORE, C } or null.
|
|
138
|
+
function matchCanonBlock(blk, laneType) {
|
|
139
|
+
if (!isArr(blk) || blk[0] !== 'block') return null
|
|
140
|
+
let i = 1
|
|
141
|
+
if (typeof blk[i] === 'string' && blk[i].startsWith('$')) i++
|
|
142
|
+
if (!(isArr(blk[i]) && blk[i][0] === 'result')) return null
|
|
143
|
+
i++
|
|
144
|
+
if (blk.length - i !== 2) return null
|
|
145
|
+
const setStmt = blk[i]
|
|
146
|
+
if (!isArr(setStmt) || setStmt[0] !== 'local.set' || typeof setStmt[1] !== 'string') return null
|
|
147
|
+
const m = matchCanonSelect(blk[i + 1], laneType)
|
|
148
|
+
if (!m || !isLocalGet(m.val, setStmt[1])) return null
|
|
149
|
+
return { core: setStmt[2], C: m.C }
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
const matchF64MulLocals = n => {
|
|
153
|
+
if (!isArr(n) || n[0] !== 'f64.mul') return null
|
|
154
|
+
const a = localGetName(n[1])
|
|
155
|
+
const b = localGetName(n[2])
|
|
156
|
+
return a && b ? [a, b] : null
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
const matchAccumStep = (n, acc) => {
|
|
160
|
+
if (!isArr(n) || n[0] !== 'local.set' || n[1] !== acc) return null
|
|
161
|
+
const e = n[2]
|
|
162
|
+
if (!isArr(e) || e[0] !== 'f64.add') return null
|
|
163
|
+
if (localGetName(e[1]) === acc) return matchF64MulLocals(e[2])
|
|
164
|
+
if (localGetName(e[2]) === acc) return matchF64MulLocals(e[1])
|
|
165
|
+
return null
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
const matchDotStore = (n, acc) => {
|
|
169
|
+
if (!isArr(n) || n[0] !== 'local.set' || typeof n[1] !== 'string') return null
|
|
170
|
+
const e = n[2]
|
|
171
|
+
if (localGetName(e) === acc) return { out: n[1], addend: null }
|
|
172
|
+
if (!isArr(e) || e[0] !== 'f64.add') return null
|
|
173
|
+
if (localGetName(e[1]) === acc) return { out: n[1], addend: e[2] }
|
|
174
|
+
if (localGetName(e[2]) === acc) return { out: n[1], addend: e[1] }
|
|
175
|
+
return null
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
const matchF64DotSeq = (stmts, i) => {
|
|
179
|
+
const reset = stmts[i]
|
|
180
|
+
if (!isArr(reset) || reset[0] !== 'local.set' || typeof reset[1] !== 'string' || !f64Zero(reset[2])) return null
|
|
181
|
+
const acc = reset[1]
|
|
182
|
+
const left = [], right = []
|
|
183
|
+
for (let k = 0; k < 4; k++) {
|
|
184
|
+
const pair = matchAccumStep(stmts[i + 1 + k], acc)
|
|
185
|
+
if (!pair) return null
|
|
186
|
+
left.push(pair[0])
|
|
187
|
+
right.push(pair[1])
|
|
188
|
+
}
|
|
189
|
+
const store = matchDotStore(stmts[i + 5], acc)
|
|
190
|
+
return store ? { end: i + 6, acc, left, right, ...store } : null
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
const f64x2Pair = (lo, hi) => ['f64x2.replace_lane', 1, ['f64x2.splat', ['local.get', lo]], ['local.get', hi]]
|
|
194
|
+
|
|
195
|
+
// Build the 2-lane dot expression `a0*p0 + a1*p1 + a2*p2 + a3*p3`.
|
|
196
|
+
// Default: explicit mul/add pairs (one rounding per op) — bit-identical to the
|
|
197
|
+
// scalar `a*b+c` a JS engine emits. With `useRelaxedFma`, each accumulate folds
|
|
198
|
+
// to `f64x2.relaxed_madd(splat(a[i]), p[i], acc)` — one VFMADD instruction with
|
|
199
|
+
// a single rounding. Faster and more accurate, but the fused rounding diverges
|
|
200
|
+
// from the non-fused reference (the bench `fma` parity class). Opt-in only.
|
|
201
|
+
const dotPairExpr = (a, pairs, useRelaxedFma = false) => {
|
|
202
|
+
let expr = ['f64x2.mul', ['f64x2.splat', ['local.get', a[0]]], pairs[0]]
|
|
203
|
+
for (let i = 1; i < 4; i++) {
|
|
204
|
+
expr = useRelaxedFma
|
|
205
|
+
? ['f64x2.relaxed_madd', ['f64x2.splat', ['local.get', a[i]]], pairs[i], expr]
|
|
206
|
+
: ['f64x2.add', expr, ['f64x2.mul', ['f64x2.splat', ['local.get', a[i]]], pairs[i]]]
|
|
207
|
+
}
|
|
208
|
+
return expr
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
const vectorizeStraightLineF64DotPairsIn = (node, fnLocals, freshIdRef, newLocalDecls, useRelaxedFma = false) => {
|
|
212
|
+
if (!isArr(node)) return
|
|
213
|
+
for (let i = 0; i < node.length; i++) {
|
|
214
|
+
const child = node[i]
|
|
215
|
+
if (isArr(child)) vectorizeStraightLineF64DotPairsIn(child, fnLocals, freshIdRef, newLocalDecls, useRelaxedFma)
|
|
216
|
+
}
|
|
217
|
+
const addendTemps = new Map()
|
|
218
|
+
const pairTemps = new Map()
|
|
219
|
+
for (let i = 0; i < node.length;) {
|
|
220
|
+
const a = matchF64DotSeq(node, i)
|
|
221
|
+
if (!a) { i++; continue }
|
|
222
|
+
const b = matchF64DotSeq(node, a.end)
|
|
223
|
+
if (!b || a.acc !== b.acc || !exprEq(a.left, b.left) || !exprEq(a.addend, b.addend) ||
|
|
224
|
+
fnLocals.get(a.out) !== 'f64' || fnLocals.get(b.out) !== 'f64') {
|
|
225
|
+
i++
|
|
226
|
+
continue
|
|
227
|
+
}
|
|
228
|
+
const v = `$__dot2_${freshIdRef.next++}`
|
|
229
|
+
newLocalDecls.push(['local', v, 'v128'])
|
|
230
|
+
fnLocals.set(v, 'v128')
|
|
231
|
+
let prefix = []
|
|
232
|
+
let addend = a.addend
|
|
233
|
+
if (addend) {
|
|
234
|
+
const key = JSON.stringify(addend)
|
|
235
|
+
let tmp = addendTemps.get(key)
|
|
236
|
+
if (!tmp) {
|
|
237
|
+
tmp = `$__dotadd_${freshIdRef.next++}`
|
|
238
|
+
addendTemps.set(key, tmp)
|
|
239
|
+
newLocalDecls.push(['local', tmp, 'f64'])
|
|
240
|
+
fnLocals.set(tmp, 'f64')
|
|
241
|
+
prefix = [['local.set', tmp, addend]]
|
|
242
|
+
}
|
|
243
|
+
addend = ['local.get', tmp]
|
|
244
|
+
}
|
|
245
|
+
const pairs = []
|
|
246
|
+
for (let k = 0; k < 4; k++) {
|
|
247
|
+
const key = `${a.right[k]}\0${b.right[k]}`
|
|
248
|
+
let tmp = pairTemps.get(key)
|
|
249
|
+
if (!tmp) {
|
|
250
|
+
tmp = `$__dotpair_${freshIdRef.next++}`
|
|
251
|
+
pairTemps.set(key, tmp)
|
|
252
|
+
newLocalDecls.push(['local', tmp, 'v128'])
|
|
253
|
+
fnLocals.set(tmp, 'v128')
|
|
254
|
+
prefix.push(['local.set', tmp, f64x2Pair(a.right[k], b.right[k])])
|
|
255
|
+
}
|
|
256
|
+
pairs.push(['local.get', tmp])
|
|
257
|
+
}
|
|
258
|
+
const dot = dotPairExpr(a.left, pairs, useRelaxedFma)
|
|
259
|
+
const expr = addend ? ['f64x2.add', dot, ['f64x2.splat', addend]] : dot
|
|
260
|
+
node.splice(i, b.end - i,
|
|
261
|
+
...prefix,
|
|
262
|
+
['local.set', v, expr],
|
|
263
|
+
['local.set', a.out, ['f64x2.extract_lane', 0, ['local.get', v]]],
|
|
264
|
+
['local.set', b.out, ['f64x2.extract_lane', 1, ['local.get', v]]],
|
|
265
|
+
)
|
|
266
|
+
i += prefix.length + 3
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
// ---- Lane type tables ------------------------------------------------------
|
|
271
|
+
|
|
272
|
+
const LANE_INFO = {
|
|
273
|
+
i8: { lanes: 16, strideLog2: 0, stride: 1, splat: 'i8x16.splat', constOp: 'i32.const' },
|
|
274
|
+
i16: { lanes: 8, strideLog2: 1, stride: 2, splat: 'i16x8.splat', constOp: 'i32.const' },
|
|
275
|
+
i32: { lanes: 4, strideLog2: 2, stride: 4, splat: 'i32x4.splat', constOp: 'i32.const' },
|
|
276
|
+
i64: { lanes: 2, strideLog2: 3, stride: 8, splat: 'i64x2.splat', constOp: 'i64.const' },
|
|
277
|
+
f32: { lanes: 4, strideLog2: 2, stride: 4, splat: 'f32x4.splat', constOp: 'f32.const' },
|
|
278
|
+
f64: { lanes: 2, strideLog2: 3, stride: 8, splat: 'f64x2.splat', constOp: 'f64.const' },
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
// Narrow loads/stores (i32.load8_u etc.) define i8 / i16 lane types — values
|
|
282
|
+
// computed in i32 then truncated by store{8,16}, which matches i{8,16}xN wrap
|
|
283
|
+
// semantics exactly.
|
|
284
|
+
const LOAD_OPS = {
|
|
285
|
+
'i32.load8_u': 'i8', 'i32.load8_s': 'i8',
|
|
286
|
+
'i32.load16_u': 'i16','i32.load16_s': 'i16',
|
|
287
|
+
'i32.load': 'i32', 'i64.load': 'i64', 'f32.load': 'f32', 'f64.load': 'f64',
|
|
288
|
+
}
|
|
289
|
+
const STORE_OPS = {
|
|
290
|
+
'i32.store8': 'i8', 'i32.store16': 'i16',
|
|
291
|
+
'i32.store': 'i32', 'i64.store': 'i64', 'f32.store': 'f32', 'f64.store': 'f64',
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
// scalar op → SIMD op. shamtScalar:true means second operand stays scalar i32.
|
|
295
|
+
//
|
|
296
|
+
// For i8/i16 lanes the SCALAR ops are i32.* — wasm has no native i8/i16 ops,
|
|
297
|
+
// values flow as i32 and the trailing store{8,16} truncates. i{8,16}x{N}.add
|
|
298
|
+
// wraps within each lane the same way, so the observable result matches.
|
|
299
|
+
// Note: wasm SIMD has no i8x16.mul, so multiplication on byte arrays bails.
|
|
300
|
+
const LANE_PURE = {
|
|
301
|
+
// Right shifts intentionally omitted for narrow lanes: scalar emits
|
|
302
|
+
// i32.shr_{s,u} on a load8/load16 i32 (zero- or sign-extended), while
|
|
303
|
+
// i{8,16}x{N}.shr_{s,u} treats lanes as their narrow type. The two diverge
|
|
304
|
+
// when load and shift signedness mismatch (e.g. load8_u + shr_s on byte
|
|
305
|
+
// 0xFF: scalar=0x7F, SIMD=0xFF). Safe set excludes shr_*.
|
|
306
|
+
i8: new Map([
|
|
307
|
+
['i32.add', { simd: 'i8x16.add' }],
|
|
308
|
+
['i32.sub', { simd: 'i8x16.sub' }],
|
|
309
|
+
['i32.and', { simd: 'v128.and' }],
|
|
310
|
+
['i32.or', { simd: 'v128.or' }],
|
|
311
|
+
['i32.xor', { simd: 'v128.xor' }],
|
|
312
|
+
['i32.shl', { simd: 'i8x16.shl', shamtScalar: true }],
|
|
313
|
+
]),
|
|
314
|
+
i16: new Map([
|
|
315
|
+
['i32.add', { simd: 'i16x8.add' }],
|
|
316
|
+
['i32.sub', { simd: 'i16x8.sub' }],
|
|
317
|
+
['i32.mul', { simd: 'i16x8.mul' }],
|
|
318
|
+
['i32.and', { simd: 'v128.and' }],
|
|
319
|
+
['i32.or', { simd: 'v128.or' }],
|
|
320
|
+
['i32.xor', { simd: 'v128.xor' }],
|
|
321
|
+
['i32.shl', { simd: 'i16x8.shl', shamtScalar: true }],
|
|
322
|
+
]),
|
|
323
|
+
i32: new Map([
|
|
324
|
+
['i32.add', { simd: 'i32x4.add' }],
|
|
325
|
+
['i32.sub', { simd: 'i32x4.sub' }],
|
|
326
|
+
['i32.mul', { simd: 'i32x4.mul' }],
|
|
327
|
+
['i32.and', { simd: 'v128.and' }],
|
|
328
|
+
['i32.or', { simd: 'v128.or' }],
|
|
329
|
+
['i32.xor', { simd: 'v128.xor' }],
|
|
330
|
+
['i32.shl', { simd: 'i32x4.shl', shamtScalar: true }],
|
|
331
|
+
['i32.shr_s', { simd: 'i32x4.shr_s', shamtScalar: true }],
|
|
332
|
+
['i32.shr_u', { simd: 'i32x4.shr_u', shamtScalar: true }],
|
|
333
|
+
]),
|
|
334
|
+
i64: new Map([
|
|
335
|
+
['i64.add', { simd: 'i64x2.add' }],
|
|
336
|
+
['i64.sub', { simd: 'i64x2.sub' }],
|
|
337
|
+
['i64.mul', { simd: 'i64x2.mul' }],
|
|
338
|
+
['i64.and', { simd: 'v128.and' }],
|
|
339
|
+
['i64.or', { simd: 'v128.or' }],
|
|
340
|
+
['i64.xor', { simd: 'v128.xor' }],
|
|
341
|
+
['i64.shl', { simd: 'i64x2.shl', shamtScalar: true }],
|
|
342
|
+
['i64.shr_s', { simd: 'i64x2.shr_s', shamtScalar: true }],
|
|
343
|
+
['i64.shr_u', { simd: 'i64x2.shr_u', shamtScalar: true }],
|
|
344
|
+
]),
|
|
345
|
+
f32: new Map([
|
|
346
|
+
['f32.add', { simd: 'f32x4.add' }],
|
|
347
|
+
['f32.sub', { simd: 'f32x4.sub' }],
|
|
348
|
+
['f32.mul', { simd: 'f32x4.mul' }],
|
|
349
|
+
['f32.div', { simd: 'f32x4.div' }],
|
|
350
|
+
['f32.min', { simd: 'f32x4.min' }],
|
|
351
|
+
['f32.max', { simd: 'f32x4.max' }],
|
|
352
|
+
['f32.neg', { simd: 'f32x4.neg' }],
|
|
353
|
+
['f32.abs', { simd: 'f32x4.abs' }],
|
|
354
|
+
['f32.sqrt', { simd: 'f32x4.sqrt' }],
|
|
355
|
+
]),
|
|
356
|
+
f64: new Map([
|
|
357
|
+
['f64.add', { simd: 'f64x2.add' }],
|
|
358
|
+
['f64.sub', { simd: 'f64x2.sub' }],
|
|
359
|
+
['f64.mul', { simd: 'f64x2.mul' }],
|
|
360
|
+
['f64.div', { simd: 'f64x2.div' }],
|
|
361
|
+
['f64.min', { simd: 'f64x2.min' }],
|
|
362
|
+
['f64.max', { simd: 'f64x2.max' }],
|
|
363
|
+
['f64.neg', { simd: 'f64x2.neg' }],
|
|
364
|
+
['f64.abs', { simd: 'f64x2.abs' }],
|
|
365
|
+
['f64.sqrt', { simd: 'f64x2.sqrt' }],
|
|
366
|
+
]),
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
// Horizontal reductions: associative+commutative ops applied to one
|
|
370
|
+
// loop-carried accumulator. Each entry maps the SCALAR op (which is also
|
|
371
|
+
// the op used to combine the SIMD result back into the accumulator at the
|
|
372
|
+
// end) to its SIMD lane op, lane extractor, and identity element.
|
|
373
|
+
//
|
|
374
|
+
// Floats (add, mul) are not strictly associative — vectorized order produces
|
|
375
|
+
// ulp-level differences from scalar order. Acceptable for typical use
|
|
376
|
+
// (reductions over typed arrays of well-conditioned data); strict-equal
|
|
377
|
+
// callers must keep the pass off.
|
|
378
|
+
//
|
|
379
|
+
// Integer mul (`p *= a[i]`) IS associative+commutative mod 2³² / 2⁶⁴, so its
|
|
380
|
+
// vectorization is value-exact. Identity is 1 (the multiplicative neutral).
|
|
381
|
+
//
|
|
382
|
+
// Narrow lanes (i8/i16) intentionally absent: `s += a[i]` with a u8/u16
|
|
383
|
+
// load expands the value to i32 before the add, so the accumulator's lane
|
|
384
|
+
// type is always wider than the load's element type. That widening would
|
|
385
|
+
// require pairwise/extending-add ops (i16x8.extadd_pairwise_*) — separate
|
|
386
|
+
// recognizer. Integer min/max likewise: WASM has no scalar i32.min, so they
|
|
387
|
+
// arrive as a `select`, not a binary op — a separate recognizer branch.
|
|
388
|
+
const REDUCE_OPS = {
|
|
389
|
+
i32: {
|
|
390
|
+
'i32.add': { simd: 'i32x4.add', extract: 'i32x4.extract_lane', laneType: 'i32', constNode: ['i32.const', 0] },
|
|
391
|
+
'i32.mul': { simd: 'i32x4.mul', extract: 'i32x4.extract_lane', laneType: 'i32', constNode: ['i32.const', 1] },
|
|
392
|
+
'i32.xor': { simd: 'v128.xor', extract: 'i32x4.extract_lane', laneType: 'i32', constNode: ['i32.const', 0] },
|
|
393
|
+
'i32.and': { simd: 'v128.and', extract: 'i32x4.extract_lane', laneType: 'i32', constNode: ['i32.const', -1] },
|
|
394
|
+
'i32.or': { simd: 'v128.or', extract: 'i32x4.extract_lane', laneType: 'i32', constNode: ['i32.const', 0] },
|
|
395
|
+
},
|
|
396
|
+
i64: {
|
|
397
|
+
'i64.add': { simd: 'i64x2.add', extract: 'i64x2.extract_lane', laneType: 'i64', constNode: ['i64.const', 0] },
|
|
398
|
+
'i64.mul': { simd: 'i64x2.mul', extract: 'i64x2.extract_lane', laneType: 'i64', constNode: ['i64.const', 1] },
|
|
399
|
+
'i64.xor': { simd: 'v128.xor', extract: 'i64x2.extract_lane', laneType: 'i64', constNode: ['i64.const', 0] },
|
|
400
|
+
'i64.and': { simd: 'v128.and', extract: 'i64x2.extract_lane', laneType: 'i64', constNode: ['i64.const', -1] },
|
|
401
|
+
'i64.or': { simd: 'v128.or', extract: 'i64x2.extract_lane', laneType: 'i64', constNode: ['i64.const', 0] },
|
|
402
|
+
},
|
|
403
|
+
f32: {
|
|
404
|
+
'f32.add': { simd: 'f32x4.add', extract: 'f32x4.extract_lane', laneType: 'f32', constNode: ['f32.const', 0] },
|
|
405
|
+
'f32.mul': { simd: 'f32x4.mul', extract: 'f32x4.extract_lane', laneType: 'f32', constNode: ['f32.const', 1] },
|
|
406
|
+
},
|
|
407
|
+
f64: {
|
|
408
|
+
'f64.add': { simd: 'f64x2.add', extract: 'f64x2.extract_lane', laneType: 'f64', constNode: ['f64.const', 0] },
|
|
409
|
+
'f64.mul': { simd: 'f64x2.mul', extract: 'f64x2.extract_lane', laneType: 'f64', constNode: ['f64.const', 1] },
|
|
410
|
+
},
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
// Widening byte/short sums: an i32 accumulator fed by ONE bare narrow load
|
|
414
|
+
// (`s += u8[i]`). The lane data is i8/i16 but the accumulator is i32, so the
|
|
415
|
+
// plain lane-add path can't apply — instead each 16-byte vector collapses via
|
|
416
|
+
// extadd_pairwise into i32x4 partial sums. VALUE-EXACT mod 2³² (unlike float
|
|
417
|
+
// reductions): pairwise intermediates can't overflow (2×255 < 2¹⁶, 2×(−128)
|
|
418
|
+
// fits i16; the i16→i32 step extends before adding), and wrap-add is
|
|
419
|
+
// associative+commutative. Restricted to a BARE load: arithmetic on the
|
|
420
|
+
// narrow lanes before widening would wrap at lane width where the scalar
|
|
421
|
+
// code widens first.
|
|
422
|
+
const WIDEN_LOADS = {
|
|
423
|
+
'i32.load8_u': { laneType: 'i8', steps: ['i16x8.extadd_pairwise_i8x16_u', 'i32x4.extadd_pairwise_i16x8_u'] },
|
|
424
|
+
'i32.load8_s': { laneType: 'i8', steps: ['i16x8.extadd_pairwise_i8x16_s', 'i32x4.extadd_pairwise_i16x8_s'] },
|
|
425
|
+
'i32.load16_u': { laneType: 'i16', steps: ['i32x4.extadd_pairwise_i16x8_u'] },
|
|
426
|
+
'i32.load16_s': { laneType: 'i16', steps: ['i32x4.extadd_pairwise_i16x8_s'] },
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
// Widening min/max over a BARE narrow load (`m = Math.max(m, u8[i])` with an
|
|
430
|
+
// i32 accumulator). Unlike the widening SUM there is no overflow concern:
|
|
431
|
+
// min/max at the load's own lane width over its own sign is value-exact, so
|
|
432
|
+
// the fold stays at lane width (16/8 lanes per vector) and only the final
|
|
433
|
+
// horizontal merge widens, via the sign-matched extract. Identity seeds the
|
|
434
|
+
// vector accumulator with the op's neutral: type-min for max, type-max for min.
|
|
435
|
+
const MINMAX_WIDEN = {
|
|
436
|
+
'i32.load8_u': { pre: 'i8x16', sign: 'u', laneType: 'i8', lo: 0, hi: 255 },
|
|
437
|
+
'i32.load8_s': { pre: 'i8x16', sign: 's', laneType: 'i8', lo: -128, hi: 127 },
|
|
438
|
+
'i32.load16_u': { pre: 'i16x8', sign: 'u', laneType: 'i16', lo: 0, hi: 65535 },
|
|
439
|
+
'i32.load16_s': { pre: 'i16x8', sign: 's', laneType: 'i16', lo: -32768, hi: 32767 },
|
|
440
|
+
}
|
|
441
|
+
// jz's number model converts narrow loads to f64 before Math.min/max, so the
|
|
442
|
+
// canon reduce arrives as (f64.max acc (f64.convert_i32_x LOAD)). The convert
|
|
443
|
+
// sign must match the load sign for the lane fold to be value-exact.
|
|
444
|
+
const MINMAX_CVT = { 'f64.convert_i32_u': 'u', 'f64.convert_i32_s': 's' }
|
|
445
|
+
|
|
446
|
+
// op-name → REDUCE entry across all lane types (the op-name itself encodes
|
|
447
|
+
// the lane type prefix, e.g. `i32.add` ⇒ i32 lanes).
|
|
448
|
+
const REDUCE_OP_LOOKUP = (() => {
|
|
449
|
+
const m = new Map()
|
|
450
|
+
for (const lt of Object.keys(REDUCE_OPS))
|
|
451
|
+
for (const op of Object.keys(REDUCE_OPS[lt]))
|
|
452
|
+
m.set(op, REDUCE_OPS[lt][op])
|
|
453
|
+
return m
|
|
454
|
+
})()
|
|
455
|
+
|
|
456
|
+
// Min/max reductions (`m = Math.max(m, a[i])`). jz wraps every Math.min/max in
|
|
457
|
+
// a NaN-canonicalizing select, so these arrive as a TWO-statement body —
|
|
458
|
+
// (local.set $cn (OP (local.get $acc) EXPR))
|
|
459
|
+
// (local.set $acc (select C (local.get $cn) (OP-type.ne $cn $cn)))
|
|
460
|
+
// — handled separately from the bare single-statement reductions above.
|
|
461
|
+
//
|
|
462
|
+
// max/min ARE associative and commutative (exact reassociation, unlike add),
|
|
463
|
+
// so vectorization is value-exact, INCLUDING NaN: f64x2.max/min propagate a
|
|
464
|
+
// NaN lane just as scalar does, and we re-apply the canon to the merged result
|
|
465
|
+
// so the final NaN bit pattern is canonical even when N is a multiple of LANES
|
|
466
|
+
// (zero tail iterations). Identity is the op's annihilator-free neutral:
|
|
467
|
+
// -inf for max, +inf for min.
|
|
468
|
+
const REDUCE_CANON = {
|
|
469
|
+
'f64.max': { simd: 'f64x2.max', extract: 'f64x2.extract_lane', laneType: 'f64', identity: ['f64.const', '-inf'] },
|
|
470
|
+
'f64.min': { simd: 'f64x2.min', extract: 'f64x2.extract_lane', laneType: 'f64', identity: ['f64.const', 'inf'] },
|
|
471
|
+
'f32.max': { simd: 'f32x4.max', extract: 'f32x4.extract_lane', laneType: 'f32', identity: ['f32.const', '-inf'] },
|
|
472
|
+
'f32.min': { simd: 'f32x4.min', extract: 'f32x4.extract_lane', laneType: 'f32', identity: ['f32.const', 'inf'] },
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
// Scalar comparison op → SIMD lane comparison, per lane type. Used to vectorize a
|
|
476
|
+
// conditional map `buf[i] = cond ? X : Y`, which jz lowers to `(if (result T) COND
|
|
477
|
+
// (then X)(else Y))`: COND becomes an all-ones/all-zeros lane mask fed to
|
|
478
|
+
// `v128.bitselect`. NaN behaves identically lane-wise — every ordered compare is
|
|
479
|
+
// false on a NaN operand in both scalar and SIMD, and `ne` is true — so no
|
|
480
|
+
// canonicalization is needed. i64x2 has no unsigned compares in baseline SIMD, so
|
|
481
|
+
// those simply aren't listed (the loop stays scalar).
|
|
482
|
+
const LANE_COMPARE = {
|
|
483
|
+
f64: { 'f64.eq': 'f64x2.eq', 'f64.ne': 'f64x2.ne', 'f64.lt': 'f64x2.lt', 'f64.gt': 'f64x2.gt', 'f64.le': 'f64x2.le', 'f64.ge': 'f64x2.ge' },
|
|
484
|
+
f32: { 'f32.eq': 'f32x4.eq', 'f32.ne': 'f32x4.ne', 'f32.lt': 'f32x4.lt', 'f32.gt': 'f32x4.gt', 'f32.le': 'f32x4.le', 'f32.ge': 'f32x4.ge' },
|
|
485
|
+
i32: { 'i32.eq': 'i32x4.eq', 'i32.ne': 'i32x4.ne', 'i32.lt_s': 'i32x4.lt_s', 'i32.lt_u': 'i32x4.lt_u', 'i32.gt_s': 'i32x4.gt_s', 'i32.gt_u': 'i32x4.gt_u', 'i32.le_s': 'i32x4.le_s', 'i32.le_u': 'i32x4.le_u', 'i32.ge_s': 'i32x4.ge_s', 'i32.ge_u': 'i32x4.ge_u' },
|
|
486
|
+
i64: { 'i64.eq': 'i64x2.eq', 'i64.ne': 'i64x2.ne', 'i64.lt_s': 'i64x2.lt_s', 'i64.gt_s': 'i64x2.gt_s', 'i64.le_s': 'i64x2.le_s', 'i64.ge_s': 'i64x2.ge_s' },
|
|
487
|
+
}
|
|
488
|
+
|
|
489
|
+
// ---- Recognizer ------------------------------------------------------------
|
|
490
|
+
|
|
491
|
+
function isLocalGet(node, name) {
|
|
492
|
+
return isArr(node) && node[0] === 'local.get' && (name == null || node[1] === name)
|
|
493
|
+
}
|
|
494
|
+
function isI32Const(node) {
|
|
495
|
+
return isArr(node) && node[0] === 'i32.const'
|
|
496
|
+
}
|
|
497
|
+
function constNum(node) {
|
|
498
|
+
if (!isI32Const(node)) return null
|
|
499
|
+
const v = node[1]
|
|
500
|
+
return typeof v === 'number' ? v : (typeof v === 'string' ? parseInt(v, 10) : null)
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
/**
|
|
504
|
+
* Match increment shape `(local.set $X (i32.add (local.get $X) (i32.const 1)))`.
|
|
505
|
+
* Returns $X or null.
|
|
506
|
+
*/
|
|
507
|
+
function matchInc1(stmt) {
|
|
508
|
+
if (!isArr(stmt) || stmt[0] !== 'local.set' || stmt.length !== 3) return null
|
|
509
|
+
const x = stmt[1]
|
|
510
|
+
const v = stmt[2]
|
|
511
|
+
if (!isArr(v) || v[0] !== 'i32.add' || v.length !== 3) return null
|
|
512
|
+
if (!isLocalGet(v[1], x)) return null
|
|
513
|
+
if (constNum(v[2]) !== 1) return null
|
|
514
|
+
return x
|
|
515
|
+
}
|
|
516
|
+
|
|
517
|
+
/**
|
|
518
|
+
* Match increment shape `(local.set $X (i32.add (local.get $X) (i32.const C)))`
|
|
519
|
+
* for any constant C. Returns { name, c } or null. Generalizes matchInc1 to the
|
|
520
|
+
* strided-pointer bumps (`p += stride`) the ramp-map recognizer must scale.
|
|
521
|
+
*/
|
|
522
|
+
function matchIncN(stmt) {
|
|
523
|
+
if (!isArr(stmt) || stmt[0] !== 'local.set' || stmt.length !== 3) return null
|
|
524
|
+
const x = stmt[1], v = stmt[2]
|
|
525
|
+
if (!isArr(v) || v[0] !== 'i32.add' || v.length !== 3) return null
|
|
526
|
+
if (!isLocalGet(v[1], x)) return null
|
|
527
|
+
const c = constNum(v[2])
|
|
528
|
+
return c == null ? null : { name: x, c }
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
/**
|
|
532
|
+
* Match `(br_if $LABEL (i32.eqz (i32.lt_{s,u} (local.get $I) BOUND)))`.
|
|
533
|
+
* Returns { ind, bound } or null.
|
|
534
|
+
*/
|
|
535
|
+
function matchExitBrIf(stmt, label) {
|
|
536
|
+
if (!isArr(stmt) || stmt[0] !== 'br_if' || stmt[1] !== label) return null
|
|
537
|
+
const cond = stmt[2]
|
|
538
|
+
if (!isArr(cond) || cond[0] !== 'i32.eqz') return null
|
|
539
|
+
const cmp = cond[1]
|
|
540
|
+
if (!isArr(cmp) || (cmp[0] !== 'i32.lt_s' && cmp[0] !== 'i32.lt_u')) return null
|
|
541
|
+
if (!isLocalGet(cmp[1])) return null
|
|
542
|
+
return { ind: cmp[1][1], bound: cmp[2] }
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
/**
|
|
546
|
+
* Walk node, collect set of local names that are written via local.set/local.tee
|
|
547
|
+
* anywhere within. Used to detect loop-invariant locals.
|
|
548
|
+
*/
|
|
549
|
+
function collectWrites(node, out) {
|
|
550
|
+
if (!isArr(node)) return
|
|
551
|
+
const op = node[0]
|
|
552
|
+
if ((op === 'local.set' || op === 'local.tee') && typeof node[1] === 'string') {
|
|
553
|
+
out.add(node[1])
|
|
554
|
+
}
|
|
555
|
+
for (let i = 0; i < node.length; i++) collectWrites(node[i], out)
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
/**
|
|
559
|
+
* Return the FIRST kind of access for `name` in straight-line walk order.
|
|
560
|
+
* 'write' — local.set/local.tee seen first
|
|
561
|
+
* 'read' — local.get seen first
|
|
562
|
+
* null — not referenced
|
|
563
|
+
*/
|
|
564
|
+
function firstAccess(node, name) {
|
|
565
|
+
if (!isArr(node)) return null
|
|
566
|
+
const op = node[0]
|
|
567
|
+
// Walk children first — operands evaluate before the op. For local.set/tee
|
|
568
|
+
// the VALUE child (idx 2) runs before the write, so a `local.get name` in
|
|
569
|
+
// the value of `local.set name` is a read-before-write.
|
|
570
|
+
if ((op === 'local.set' || op === 'local.tee') && node[1] === name) {
|
|
571
|
+
if (node.length >= 3) {
|
|
572
|
+
const r = firstAccess(node[2], name)
|
|
573
|
+
if (r) return r
|
|
574
|
+
}
|
|
575
|
+
return 'write'
|
|
576
|
+
}
|
|
577
|
+
if (op === 'local.get' && node[1] === name) return 'read'
|
|
578
|
+
for (let i = 1; i < node.length; i++) {
|
|
579
|
+
const r = firstAccess(node[i], name)
|
|
580
|
+
if (r) return r
|
|
581
|
+
}
|
|
582
|
+
return null
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
/**
|
|
586
|
+
* Match the index-offset operand of a lane address — the part that scales the
|
|
587
|
+
* induction variable inside `(i32.add base OFFSET)`. OFFSET is one of:
|
|
588
|
+
* (i32.shl (local.get IND) (i32.const K)) → strideLog2 K
|
|
589
|
+
* (local.get IND) → strideLog2 0
|
|
590
|
+
* (local.tee $T <either of the above>) → records $T
|
|
591
|
+
* (local.get $T) where $T is a recorded offset-tee → that tee's strideLog2
|
|
592
|
+
*
|
|
593
|
+
* The tee'd form arises from CSE: a map loop `b[i] = f(a[i])` over two distinct
|
|
594
|
+
* base pointers shares one `i << K` offset (`(local.tee $T (i32.shl i K))` in
|
|
595
|
+
* the first address, `(local.get $T)` in the second). `offsetTees` (Map
|
|
596
|
+
* name→strideLog2) carries that across calls.
|
|
597
|
+
*
|
|
598
|
+
* Returns { strideLog2, teeName?: string } or null.
|
|
599
|
+
*/
|
|
600
|
+
function matchLaneOffset(off, ind, offsetTees) {
|
|
601
|
+
if (isArr(off) && off[0] === 'local.get' && typeof off[1] === 'string' &&
|
|
602
|
+
offsetTees && offsetTees.has(off[1])) {
|
|
603
|
+
return { strideLog2: offsetTees.get(off[1]), teeName: null }
|
|
604
|
+
}
|
|
605
|
+
let teeName = null
|
|
606
|
+
let n = off
|
|
607
|
+
if (isArr(n) && n[0] === 'local.tee' && n.length === 3) { teeName = n[1]; n = n[2] }
|
|
608
|
+
// (i32.shl (local.get ind) (i32.const K))
|
|
609
|
+
if (isArr(n) && n[0] === 'i32.shl' && n.length === 3 && isLocalGet(n[1], ind)) {
|
|
610
|
+
const k = constNum(n[2])
|
|
611
|
+
if (k != null && k >= 0 && k <= 3) return { strideLog2: k, teeName }
|
|
612
|
+
}
|
|
613
|
+
// (local.get ind) — stride 1
|
|
614
|
+
if (isLocalGet(n, ind)) return { strideLog2: 0, teeName }
|
|
615
|
+
return null
|
|
616
|
+
}
|
|
617
|
+
|
|
618
|
+
/**
|
|
619
|
+
* Match an address expression `(i32.add base OFFSET)`, with optional outer
|
|
620
|
+
* `(local.tee $A ...)`. OFFSET is matched by matchLaneOffset (which also
|
|
621
|
+
* accepts a CSE'd `(local.tee $T (i32.shl ind K))` / `(local.get $T)` pair).
|
|
622
|
+
* Also accepts `(local.get $A)` when $A is a previously-recorded address tee.
|
|
623
|
+
*
|
|
624
|
+
* Returns { strideLog2, base, teeName?, offsetTeeName?, viaLocal? } or null.
|
|
625
|
+
* `strideLog2` = K for i32.shl form, 0 for plain add form.
|
|
626
|
+
* `base` is the loop-invariant base subtree.
|
|
627
|
+
*/
|
|
628
|
+
function matchLaneAddr(addr, ind, addrLocals, offsetTees) {
|
|
629
|
+
let teeName = null
|
|
630
|
+
let n = addr
|
|
631
|
+
// (local.get $A) where $A holds a previously-tee'd FULL lane-address.
|
|
632
|
+
if (isArr(n) && n[0] === 'local.get' && typeof n[1] === 'string' && addrLocals && addrLocals.has(n[1])) {
|
|
633
|
+
const e = addrLocals.get(n[1])
|
|
634
|
+
return { strideLog2: e.strideLog2, base: e.base, teeName: null, viaLocal: n[1] }
|
|
635
|
+
}
|
|
636
|
+
if (isArr(n) && n[0] === 'local.tee' && n.length === 3) {
|
|
637
|
+
teeName = n[1]
|
|
638
|
+
n = n[2]
|
|
639
|
+
}
|
|
640
|
+
if (!isArr(n) || n[0] !== 'i32.add' || n.length !== 3) return null
|
|
641
|
+
const a = n[1], b = n[2]
|
|
642
|
+
const off = matchLaneOffset(b, ind, offsetTees)
|
|
643
|
+
if (!off) return null
|
|
644
|
+
return { strideLog2: off.strideLog2, base: a, teeName, offsetTeeName: off.teeName }
|
|
645
|
+
}
|
|
646
|
+
|
|
647
|
+
/**
|
|
648
|
+
* A scalar i32 local that is ONLY ever assigned a lane offset — `(i32.shl ind K)`
|
|
649
|
+
* (or bare `ind` for stride 0) — is a CSE'd offset shared across base pointers.
|
|
650
|
+
* Returns the consistent strideLog2, or null if any write to it diverges.
|
|
651
|
+
* This soundness check backs every `(local.get $T)` resolved via `offsetTees`.
|
|
652
|
+
*/
|
|
653
|
+
function _offsetLocalStride(body, name, ind) {
|
|
654
|
+
let stride = null, found = false, ok = true
|
|
655
|
+
function walk(n) {
|
|
656
|
+
if (!isArr(n)) return
|
|
657
|
+
if ((n[0] === 'local.tee' || n[0] === 'local.set') && n[1] === name && n.length === 3) {
|
|
658
|
+
found = true
|
|
659
|
+
const v = n[2]
|
|
660
|
+
let k = null
|
|
661
|
+
if (isArr(v) && v[0] === 'i32.shl' && v.length === 3 && isLocalGet(v[1], ind)) {
|
|
662
|
+
k = constNum(v[2])
|
|
663
|
+
if (k == null || k < 0 || k > 3) ok = false
|
|
664
|
+
} else if (isLocalGet(v, ind)) {
|
|
665
|
+
k = 0
|
|
666
|
+
} else ok = false
|
|
667
|
+
if (k != null) {
|
|
668
|
+
if (stride == null) stride = k
|
|
669
|
+
else if (stride !== k) ok = false
|
|
670
|
+
}
|
|
671
|
+
}
|
|
672
|
+
for (let i = 1; i < n.length; i++) walk(n[i])
|
|
673
|
+
}
|
|
674
|
+
for (const s of body) walk(s)
|
|
675
|
+
return found && ok ? stride : null
|
|
676
|
+
}
|
|
677
|
+
|
|
678
|
+
// True if any node in the tree writes a global. When false for a loop body,
|
|
679
|
+
// every `global.get` inside it is loop-invariant — safe to splat for SIMD.
|
|
680
|
+
const hasGlobalSet = (node) => {
|
|
681
|
+
if (!isArr(node)) return false
|
|
682
|
+
if (node[0] === 'global.set') return true
|
|
683
|
+
for (let i = 1; i < node.length; i++) if (hasGlobalSet(node[i])) return true
|
|
684
|
+
return false
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
// True if EXPR carries any side effect — a call, a memory store/op, or a global
|
|
688
|
+
// write. Used to reject an impure preamble we would otherwise clone (run twice).
|
|
689
|
+
const hasSideEffect = (node) => {
|
|
690
|
+
if (!isArr(node)) return false
|
|
691
|
+
const op = node[0]
|
|
692
|
+
if (op === 'call' || op === 'call_indirect' || op === 'global.set'
|
|
693
|
+
|| (typeof op === 'string' && (op.includes('.store') || op.startsWith('memory.')))) return true
|
|
694
|
+
for (let i = 1; i < node.length; i++) if (hasSideEffect(node[i])) return true
|
|
695
|
+
return false
|
|
696
|
+
}
|
|
697
|
+
|
|
698
|
+
// ---- Recognize a (block (loop)) pair --------------------------------------
|
|
699
|
+
|
|
700
|
+
/**
|
|
701
|
+
* Match the canonical vectorizable loop SCAFFOLD shared by every inner-loop
|
|
702
|
+
* recognizer:
|
|
703
|
+
* (block $blk [preamble…]
|
|
704
|
+
* (loop $loop
|
|
705
|
+
* (br_if $blk (i32.eqz (i32.lt_{s,u} $i BOUND))) ; exit guard
|
|
706
|
+
* BODY…
|
|
707
|
+
* (local.set $i (i32.add $i 1)) ; bottom increment
|
|
708
|
+
* (br $loop)))
|
|
709
|
+
*
|
|
710
|
+
* Returns the structural FACTS only — no policy — or null when the shape
|
|
711
|
+
* doesn't match:
|
|
712
|
+
* { blockNode, blockLabel, loopNode, loopLabel, endIdx, incIdx, incVar,
|
|
713
|
+
* exitInfo, bound, boundLocal, body, preamble }
|
|
714
|
+
* - `blockNode` is the original block, embedded verbatim as the scalar tail by
|
|
715
|
+
* each lifter's wrapper (the never-miscompile remainder loop).
|
|
716
|
+
* - `body` = loopNode.slice(3, incIdx) (between exit guard and increment)
|
|
717
|
+
* - `bound` = the raw BOUND expr; `boundLocal` = its local name when it is
|
|
718
|
+
* `(local.get $L)`, else null. Bound shape is NOT rejected here — callers that
|
|
719
|
+
* require a local-or-const bound check it themselves (tryStrengthReduceIV is
|
|
720
|
+
* bound-shape-agnostic, so baking a rejection in would change its behavior).
|
|
721
|
+
*
|
|
722
|
+
* `opts.allowPreamble` (default false): when true, LICM-hoisted invariant
|
|
723
|
+
* `(local.set $__li* EXPR)` statements BEFORE the loop are collected into
|
|
724
|
+
* `preamble` (pure & loop-invariant by construction — safe to clone/re-run);
|
|
725
|
+
* a non-`$__li` preamble, an impure value, or any array content AFTER the loop
|
|
726
|
+
* bails. When false, ANY non-loop array content in the block bails.
|
|
727
|
+
*/
|
|
728
|
+
function matchBlockLoop(blockNode, opts = {}) {
|
|
729
|
+
if (!isArr(blockNode) || blockNode[0] !== 'block') return null
|
|
730
|
+
const allowPreamble = !!opts.allowPreamble
|
|
731
|
+
let blockLabel = null, loopNode = null
|
|
732
|
+
const preamble = []
|
|
733
|
+
for (let i = 1; i < blockNode.length; i++) {
|
|
734
|
+
const c = blockNode[i]
|
|
735
|
+
if (typeof c === 'string' && c.startsWith('$') && blockLabel == null && i === 1) { blockLabel = c; continue }
|
|
736
|
+
if (isArr(c) && c[0] === 'loop') {
|
|
737
|
+
if (loopNode) return null // multiple loops
|
|
738
|
+
loopNode = c
|
|
739
|
+
} else if (isArr(c)) {
|
|
740
|
+
// `loopNode` truthy ⇒ this content is AFTER the loop ⇒ bail (even for a $__li set).
|
|
741
|
+
// A LICM-hoisted invariant is `$__liN`; INLINING renames it (e.g. `$__inl7___li0`). Default
|
|
742
|
+
// accepts only un-inlined `$__li*` (keeps the existing recognizers byte-identical);
|
|
743
|
+
// `allowInlinedLi` (gated callers only) also accepts the `__liN` marker anywhere — both are
|
|
744
|
+
// pure & loop-invariant by construction (belt-and-suspenders: hasSideEffect guard).
|
|
745
|
+
// Under allowInlinedLi a block preamble is loop-invariant by construction (jz hoists only
|
|
746
|
+
// invariants before the loop; IV-dependent work lives in the body), so any PURE local.set is
|
|
747
|
+
// safe to clone ahead of the SIMD — covers $__inl*__li* (schrodinger) AND $_pg0-style
|
|
748
|
+
// peephole-hoisted bounds (slime). The hasSideEffect guard rejects impure setups.
|
|
749
|
+
const liOk = typeof c[1] === 'string' && (opts.allowInlinedLi ? true : c[1].startsWith('$__li'))
|
|
750
|
+
if (!allowPreamble || loopNode || c[0] !== 'local.set' || !liOk || hasSideEffect(c[2])) return null
|
|
751
|
+
preamble.push(c)
|
|
752
|
+
}
|
|
753
|
+
}
|
|
754
|
+
if (!loopNode || !blockLabel) return null
|
|
755
|
+
|
|
756
|
+
const loopLabel = typeof loopNode[1] === 'string' && loopNode[1].startsWith('$') ? loopNode[1] : null
|
|
757
|
+
if (!loopLabel) return null
|
|
758
|
+
|
|
759
|
+
const endIdx = loopNode.length - 1
|
|
760
|
+
if (!(isArr(loopNode[endIdx]) && loopNode[endIdx][0] === 'br' && loopNode[endIdx][1] === loopLabel)) return null
|
|
761
|
+
const incIdx = endIdx - 1
|
|
762
|
+
let incVar = matchInc1(loopNode[incIdx])
|
|
763
|
+
// CSE'd increment (gated): O3 may fold `x+1` into a body tee (the `xe = x+1` wrap) and write the
|
|
764
|
+
// increment as `x = $t` reusing it. Recover the IV when `$t` is `(tee/set $t (i32.add x 1))` in body.
|
|
765
|
+
if (!incVar && opts.allowInlinedLi) {
|
|
766
|
+
const inc = loopNode[incIdx]
|
|
767
|
+
if (isArr(inc) && inc[0] === 'local.set' && inc.length === 3 && isLocalGet(inc[2])) {
|
|
768
|
+
const copyOf = inc[2][1], iv = inc[1]
|
|
769
|
+
const findInc1 = (m) => isArr(m) && (((m[0] === 'local.set' || m[0] === 'local.tee') && m[1] === copyOf && isArr(m[2]) && m[2][0] === 'i32.add' && isLocalGet(m[2][1], iv) && constNum(m[2][2]) === 1) || m.some(findInc1))
|
|
770
|
+
if (loopNode.slice(3, incIdx).some(findInc1)) incVar = iv
|
|
771
|
+
}
|
|
772
|
+
}
|
|
773
|
+
if (!incVar) return null
|
|
774
|
+
|
|
775
|
+
const exitInfo = matchExitBrIf(loopNode[2], blockLabel)
|
|
776
|
+
if (!exitInfo || exitInfo.ind !== incVar) return null
|
|
777
|
+
|
|
778
|
+
const bound = exitInfo.bound
|
|
779
|
+
const boundLocal = isArr(bound) && bound[0] === 'local.get' && typeof bound[1] === 'string' ? bound[1] : null
|
|
780
|
+
const body = loopNode.slice(3, incIdx)
|
|
781
|
+
return { blockNode, blockLabel, loopNode, loopLabel, endIdx, incIdx, incVar, exitInfo, bound, boundLocal, body, preamble }
|
|
782
|
+
}
|
|
783
|
+
|
|
784
|
+
/**
|
|
785
|
+
* Try to vectorize the inner loop. Returns the replacement node array
|
|
786
|
+
* (synthetic outer block) or null on no match.
|
|
787
|
+
*/
|
|
788
|
+
function tryVectorize(bl, fnLocals, freshIdRef) {
|
|
789
|
+
// Consumes the shared scaffold descriptor (matchBlockLoop, computed once by the
|
|
790
|
+
// dispatch). The LICM `$__li` preamble is cloned ahead of the SIMD block; each
|
|
791
|
+
// set is pure & loop-invariant, so the kept scalar tail harmlessly re-runs it.
|
|
792
|
+
if (!bl) return null
|
|
793
|
+
const { incVar, bound, boundLocal, body, preamble } = bl
|
|
794
|
+
|
|
795
|
+
// Bound must be loop-invariant: (local.get $L) or (i32.const N).
|
|
796
|
+
if (!boundLocal && !isI32Const(bound)) return null
|
|
797
|
+
|
|
798
|
+
// Detect lane type from the FIRST load in body.
|
|
799
|
+
let laneType = null
|
|
800
|
+
let stride = -1
|
|
801
|
+
const loadStoreSites = [] // {parent, idx, kind:'load'|'store'}
|
|
802
|
+
// Address tees: name → {strideLog2, base}. A `(local.tee NAME (lane-addr))`
|
|
803
|
+
// both validates the load's address AND records NAME so the matching store's
|
|
804
|
+
// `(local.get NAME)` is accepted as the same lane address.
|
|
805
|
+
const addrLocals = new Map()
|
|
806
|
+
// Offset tees: name → strideLog2. A CSE'd `i << K` shared across base
|
|
807
|
+
// pointers (map loops over distinct arrays). Soundness re-checked post-scan.
|
|
808
|
+
const offsetTees = new Map()
|
|
809
|
+
|
|
810
|
+
function scanForLoadsStores(node, parent, pi) {
|
|
811
|
+
if (!isArr(node)) return true
|
|
812
|
+
const op = node[0]
|
|
813
|
+
if (LOAD_OPS[op]) {
|
|
814
|
+
if (laneType == null) {
|
|
815
|
+
laneType = LOAD_OPS[op]
|
|
816
|
+
stride = LANE_INFO[laneType].stride
|
|
817
|
+
} else if (LOAD_OPS[op] !== laneType) {
|
|
818
|
+
return false
|
|
819
|
+
}
|
|
820
|
+
const m = matchLaneAddr(node[1], incVar, addrLocals, offsetTees)
|
|
821
|
+
if (!m) return false
|
|
822
|
+
if ((1 << m.strideLog2) !== stride) return false
|
|
823
|
+
if (m.teeName) addrLocals.set(m.teeName, { strideLog2: m.strideLog2, base: m.base })
|
|
824
|
+
if (m.offsetTeeName) offsetTees.set(m.offsetTeeName, m.strideLog2)
|
|
825
|
+
loadStoreSites.push({ parent, idx: pi, kind: 'load' })
|
|
826
|
+
return true
|
|
827
|
+
}
|
|
828
|
+
if (STORE_OPS[op]) {
|
|
829
|
+
const sty = STORE_OPS[op]
|
|
830
|
+
if (laneType != null && sty !== laneType) return false
|
|
831
|
+
if (laneType == null) { laneType = sty; stride = LANE_INFO[laneType].stride }
|
|
832
|
+
const m = matchLaneAddr(node[1], incVar, addrLocals, offsetTees)
|
|
833
|
+
if (!m) return false
|
|
834
|
+
if ((1 << m.strideLog2) !== stride) return false
|
|
835
|
+
if (m.teeName) addrLocals.set(m.teeName, { strideLog2: m.strideLog2, base: m.base })
|
|
836
|
+
if (m.offsetTeeName) offsetTees.set(m.offsetTeeName, m.strideLog2)
|
|
837
|
+
loadStoreSites.push({ parent, idx: pi, kind: 'store' })
|
|
838
|
+
// Recurse into VALUE child (idx 2) — it's data, not address.
|
|
839
|
+
if (!scanForLoadsStores(node[2], node, 2)) return false
|
|
840
|
+
return true
|
|
841
|
+
}
|
|
842
|
+
// local.set/tee of an address local outside a load/store context (e.g.
|
|
843
|
+
// `(local.set $a (i32.add base (i32.shl i 2)))` as a standalone stmt) —
|
|
844
|
+
// record so a later `(local.get $a)` resolves.
|
|
845
|
+
if ((op === 'local.set' || op === 'local.tee') && typeof node[1] === 'string' && node.length === 3) {
|
|
846
|
+
const valM = matchLaneAddr(['local.tee', node[1], node[2]], incVar, addrLocals, offsetTees)
|
|
847
|
+
if (valM && valM.teeName) {
|
|
848
|
+
addrLocals.set(valM.teeName, { strideLog2: valM.strideLog2, base: valM.base })
|
|
849
|
+
}
|
|
850
|
+
// Standalone offset compute: `(local.set $t (i32.shl i K))`.
|
|
851
|
+
const offM = matchLaneOffset(node[2], incVar, offsetTees)
|
|
852
|
+
if (offM) offsetTees.set(node[1], offM.strideLog2)
|
|
853
|
+
}
|
|
854
|
+
// Recurse into all children
|
|
855
|
+
for (let i = 1; i < node.length; i++) {
|
|
856
|
+
if (!scanForLoadsStores(node[i], node, i)) return false
|
|
857
|
+
}
|
|
858
|
+
return true
|
|
859
|
+
}
|
|
860
|
+
for (const stmt of body) {
|
|
861
|
+
if (!scanForLoadsStores(stmt, null, -1)) return null
|
|
862
|
+
}
|
|
863
|
+
if (body.some(hasGlobalSet)) return null // a global write breaks the "global.get is invariant" splat
|
|
864
|
+
if (!laneType) return null // no memory ops — vectorizing buys nothing
|
|
865
|
+
if (loadStoreSites.length === 0) return null
|
|
866
|
+
|
|
867
|
+
// Soundness gate for offset-tee resolution: every `(local.get $T)` we
|
|
868
|
+
// accepted as `i << K` is only valid if EVERY write of $T is that offset.
|
|
869
|
+
for (const [name, k] of offsetTees) {
|
|
870
|
+
if (_offsetLocalStride(body, name, incVar) !== k) return null
|
|
871
|
+
}
|
|
872
|
+
|
|
873
|
+
// Classify all locals referenced in body.
|
|
874
|
+
// - induction var (incVar): exempt
|
|
875
|
+
// - bound local (if any): must be invariant
|
|
876
|
+
// - each other local: first access must not be a read-then-written pattern
|
|
877
|
+
const writes = new Set()
|
|
878
|
+
for (const s of body) collectWrites(s, writes)
|
|
879
|
+
if (boundLocal && writes.has(boundLocal)) return null // bound varies in body → bail
|
|
880
|
+
|
|
881
|
+
const localKind = new Map() // name → 'lane' | 'invariant' | 'addr'
|
|
882
|
+
// Walk to collect ALL referenced names
|
|
883
|
+
const referenced = new Set()
|
|
884
|
+
const collectRefs = (n) => {
|
|
885
|
+
if (!isArr(n)) return
|
|
886
|
+
const op = n[0]
|
|
887
|
+
if ((op === 'local.get' || op === 'local.set' || op === 'local.tee') && typeof n[1] === 'string')
|
|
888
|
+
referenced.add(n[1])
|
|
889
|
+
for (let i = 1; i < n.length; i++) collectRefs(n[i])
|
|
890
|
+
}
|
|
891
|
+
for (const s of body) collectRefs(s)
|
|
892
|
+
|
|
893
|
+
for (const name of referenced) {
|
|
894
|
+
if (name === incVar) continue
|
|
895
|
+
if (writes.has(name)) {
|
|
896
|
+
// Must be lane-local: first access is a write.
|
|
897
|
+
let firstKind = null
|
|
898
|
+
for (const s of body) {
|
|
899
|
+
const k = firstAccess(s, name)
|
|
900
|
+
if (k) { firstKind = k; break }
|
|
901
|
+
}
|
|
902
|
+
if (firstKind === 'read') return null // loop-carried (reduction or stencil)
|
|
903
|
+
// Discriminate lane-data vs address-tee. Address tees hold i32 addresses,
|
|
904
|
+
// not vector data. We classify by checking the local's declared type.
|
|
905
|
+
const decl = fnLocals.get(name)
|
|
906
|
+
if (decl === 'i32' && (offsetTees.has(name) || _isAddressLocal(body, name, incVar))) {
|
|
907
|
+
localKind.set(name, 'addr')
|
|
908
|
+
} else {
|
|
909
|
+
localKind.set(name, 'lane')
|
|
910
|
+
}
|
|
911
|
+
} else {
|
|
912
|
+
localKind.set(name, 'invariant')
|
|
913
|
+
}
|
|
914
|
+
}
|
|
915
|
+
|
|
916
|
+
// Build lifted body. If anything fails to lift, bail.
|
|
917
|
+
const newLanedLocals = new Map() // origName → { laneName, simdType }
|
|
918
|
+
const extraLocals = [] // canon temps allocated during lift
|
|
919
|
+
const ctx = { laneType, incVar, rampVar: null, rampTemp: null, widenLoads: false, localKind, newLanedLocals, extraLocals, freshIdRef, fail: false, failReason: null }
|
|
920
|
+
const lifted = []
|
|
921
|
+
for (const s of body) {
|
|
922
|
+
const r = liftStmt(s, ctx)
|
|
923
|
+
if (ctx.fail) return null
|
|
924
|
+
if (r != null) {
|
|
925
|
+
if (Array.isArray(r) && r[0] === '__seq__') lifted.push(...r.slice(1))
|
|
926
|
+
else lifted.push(r)
|
|
927
|
+
}
|
|
928
|
+
}
|
|
929
|
+
if (lifted.length === 0) return null
|
|
930
|
+
|
|
931
|
+
// Generate fresh names
|
|
932
|
+
const id = freshIdRef.next++
|
|
933
|
+
const simdBoundName = `$__simd_bound${id}`
|
|
934
|
+
const simdBrkLabel = `$__simd_brk${id}`
|
|
935
|
+
const simdLoopLabel = `$__simd_loop${id}`
|
|
936
|
+
|
|
937
|
+
const info = LANE_INFO[laneType]
|
|
938
|
+
const lanes = info.lanes
|
|
939
|
+
const mask = -lanes // bit pattern ~(lanes-1) in i32 two's complement
|
|
940
|
+
|
|
941
|
+
// Build SIMD prefix block.
|
|
942
|
+
const boundExpr = boundLocal
|
|
943
|
+
? ['local.get', boundLocal]
|
|
944
|
+
: bound // i32.const N
|
|
945
|
+
const simdBlock = ['block', simdBrkLabel,
|
|
946
|
+
['loop', simdLoopLabel,
|
|
947
|
+
['br_if', simdBrkLabel,
|
|
948
|
+
['i32.eqz', ['i32.lt_s', ['local.get', incVar], ['local.get', simdBoundName]]]],
|
|
949
|
+
...lifted,
|
|
950
|
+
['local.set', incVar,
|
|
951
|
+
['i32.add', ['local.get', incVar], ['i32.const', lanes]]],
|
|
952
|
+
['br', simdLoopLabel]
|
|
953
|
+
]
|
|
954
|
+
]
|
|
955
|
+
|
|
956
|
+
// Bound setup: simdBoundName = bound & ~(lanes-1)
|
|
957
|
+
const boundSetup = ['local.set', simdBoundName,
|
|
958
|
+
['i32.and', boundExpr, ['i32.const', mask]]]
|
|
959
|
+
|
|
960
|
+
// Synthetic outer wrapper — has no result, no label, just sequences.
|
|
961
|
+
// A clone of any LICM-hoisted preamble runs first (so the SIMD block sees the
|
|
962
|
+
// invariant); the original block is preserved unchanged as the scalar tail (which
|
|
963
|
+
// re-runs the preamble harmlessly — it is loop-invariant).
|
|
964
|
+
const wrapper = ['block', ...preamble.map(cloneNode), boundSetup, simdBlock, bl.blockNode]
|
|
965
|
+
|
|
966
|
+
// Locals to add to function header.
|
|
967
|
+
const newLocalDecls = [
|
|
968
|
+
['local', simdBoundName, 'i32'],
|
|
969
|
+
...[...newLanedLocals.values()].map(({ laneName }) => ['local', laneName, 'v128']),
|
|
970
|
+
...extraLocals,
|
|
971
|
+
]
|
|
972
|
+
|
|
973
|
+
return { wrapper, newLocalDecls }
|
|
974
|
+
}
|
|
975
|
+
|
|
976
|
+
// ---- Stencil recognizer (neighbour loads: a[i±δ], a[c±δ], a[rn+x]) --------
|
|
977
|
+
//
|
|
978
|
+
// Vectorizes a map whose loads read NEIGHBOURING elements — `b[i] = f(a[i-1],
|
|
979
|
+
// a[i], a[i+1])` and the 2-D form `b[c] = f(a[c-1], a[c+1], a[rn+x], …)` where
|
|
980
|
+
// `c = rc + x` is a derived induction var (rc loop-invariant, x the IV).
|
|
981
|
+
//
|
|
982
|
+
// The lift is bit-exact BY CONSTRUCTION: a scalar `f64.load` at `base+(idx<<K)`
|
|
983
|
+
// becomes `v128.load` at the SAME address; for f64x2 lanes (x, x+1) that covers
|
|
984
|
+
// `(elem[idx], elem[idx+1])` — exactly the bytes the two scalar iterations read.
|
|
985
|
+
// No new memory is touched (scalar tail handles the remainder) ⇒ no boundary
|
|
986
|
+
// special-casing, no new OOB. The neighbour `a[i+1]` arrives as
|
|
987
|
+
// `(f64.load offset=8 …)` → `v128.load offset=8` (the +1-shifted pair). Stride-1
|
|
988
|
+
// in the IV is required (consecutive lanes ⇒ consecutive elements): every index
|
|
989
|
+
// must be affine in the IV with coefficient exactly 1 (`ivCoeff`).
|
|
990
|
+
//
|
|
991
|
+
// Correctness gates:
|
|
992
|
+
// • f64/f32 lanes only — float data locals vs i32 index/address locals are
|
|
993
|
+
// type-distinct, so localKind is by type. An i32-used-as-data case bails in
|
|
994
|
+
// the lifter (never miscompiles). Integer-lane stencils (types collide) decline.
|
|
995
|
+
// • In-place bail: if the WRITTEN base is also accessed at a DIFFERENT element,
|
|
996
|
+
// SIMD reads the old value where scalar reads the just-written one (loop-
|
|
997
|
+
// carried) ⇒ null. Offset-0 read of the written array is safe.
|
|
998
|
+
// • Distinct base subtrees ⇒ assumed non-aliasing — the SAME assumption the
|
|
999
|
+
// plain map path already relies on. A ping-pong buffer swap (waves) is OUTSIDE
|
|
1000
|
+
// the loop, so in-loop bases stay distinct globals — safe without a runtime guard.
|
|
1001
|
+
// • Reassociation: summing neighbours reorders f64 adds across lanes (ulp, like
|
|
1002
|
+
// float reductions) — gated behind cfg.experimentalStencil until proven.
|
|
1003
|
+
function tryStencil(node, fnLocals, freshIdRef, enabled) {
|
|
1004
|
+
if (!enabled) return null
|
|
1005
|
+
// Gated, so match here with allowInlinedLi (accepts inlined LICM preambles `$__inl7___li*` —
|
|
1006
|
+
// grid loops like schrodinger's stepR carry the row-base `y*w` as such a hoisted invariant).
|
|
1007
|
+
const bl = matchBlockLoop(node, { allowPreamble: true, allowInlinedLi: true })
|
|
1008
|
+
if (!bl) return null
|
|
1009
|
+
const { incVar, bound, body, preamble } = bl // preamble: LICM-hoisted $__li invariants
|
|
1010
|
+
if (body.some(hasGlobalSet)) return null
|
|
1011
|
+
|
|
1012
|
+
// Leaf-stencil guard: a stencil body is pure array arithmetic. A NESTED LOOP (the outer loop of a
|
|
1013
|
+
// 2-D sweep, whose body contains the inner loop) or a non-$math call must NOT be lifted as a
|
|
1014
|
+
// stencil — its "neighbour reads" would be the nested loop's loads, misaligned. waves/schrodinger/
|
|
1015
|
+
// metaballs bodies are pure arithmetic (math calls allowed), so they pass.
|
|
1016
|
+
const hasNestedLoopOrCall = (n) => isArr(n) && (n[0] === 'loop'
|
|
1017
|
+
|| (n[0] === 'call' && (typeof n[1] !== 'string' || !n[1].startsWith('$math.'))) || n[0] === 'call_indirect'
|
|
1018
|
+
|| n.some(hasNestedLoopOrCall))
|
|
1019
|
+
if (body.some(hasNestedLoopOrCall)) return null
|
|
1020
|
+
|
|
1021
|
+
const writes = new Set()
|
|
1022
|
+
for (const s of body) collectWrites(s, writes)
|
|
1023
|
+
|
|
1024
|
+
// Bound is re-evaluated for the SIMD guard, so it must be a PURE loop-invariant
|
|
1025
|
+
// i32 expression (const / unwritten local / global / +,-,* thereof). Unlike the
|
|
1026
|
+
// plain-map path (bare local-or-const only), stencils commonly bound by `w-1`.
|
|
1027
|
+
const boundPureInv = (n) =>
|
|
1028
|
+
isI32Const(n) ? true
|
|
1029
|
+
: isLocalGet(n) ? !writes.has(n[1])
|
|
1030
|
+
: (isArr(n) && n[0] === 'global.get') ? true
|
|
1031
|
+
: (isArr(n) && (n[0] === 'i32.add' || n[0] === 'i32.sub' || n[0] === 'i32.mul') && n.length === 3)
|
|
1032
|
+
? boundPureInv(n[1]) && boundPureInv(n[2])
|
|
1033
|
+
: false
|
|
1034
|
+
if (!boundPureInv(bound)) return null
|
|
1035
|
+
|
|
1036
|
+
// Element-index coefficient in the IV: 0 (loop-invariant), 1 (stride-1 affine —
|
|
1037
|
+
// IV, a derived IV, or either ± invariant), or null (anything else).
|
|
1038
|
+
const derived = new Set()
|
|
1039
|
+
let needsPeel = false
|
|
1040
|
+
const rightBs = []
|
|
1041
|
+
const unTee = (b) => (isArr(b) && b[0] === 'local.tee' && b.length === 3) ? b[2] : b // CSE folds x±1 into a tee
|
|
1042
|
+
const isStep = (b, op) => { b = unTee(b); return isArr(b) && b[0] === op && b.length === 3 && isLocalGet(b[1], incVar) && isI32Const(b[2]) && constNum(b[2]) === 1 }
|
|
1043
|
+
const isZeroGuard = (g) => isArr(g) && ((g[0] === 'i32.eqz' && isLocalGet(g[1], incVar)) || (g[0] === 'i32.eq' && isLocalGet(g[1], incVar) && isI32Const(g[2]) && constNum(g[2]) === 0))
|
|
1044
|
+
// Toroidal wrap-select: `xw = x>0?x-1:w-1` / `xe = x<w-1?x+1:0`. Fires its wrap value only at a
|
|
1045
|
+
// boundary column the peel covers — LEFT (interior x-1) at x=0, RIGHT (interior x+1) at x=B.
|
|
1046
|
+
// Returns null | {dir:'L'} | {dir:'R',B}. Sound for ANY B: simdBound caps at min(bound,…B)-(lanes-1)
|
|
1047
|
+
// so no chunk reaches x=B (no need to prove B==bound-1, which may be hoisted out of reach).
|
|
1048
|
+
const isWrapSelect = (e) => {
|
|
1049
|
+
if (!isArr(e) || e[0] !== 'select' || e.length !== 4) return null
|
|
1050
|
+
const g = e[3]
|
|
1051
|
+
if (isStep(e[1], 'i32.sub') && ivCoeff(e[2]) === 0 && isArr(g) && g[0] === 'i32.gt_s' && isLocalGet(g[1], incVar) && isI32Const(g[2]) && constNum(g[2]) === 0) return { dir: 'L' }
|
|
1052
|
+
if (isStep(e[2], 'i32.sub') && ivCoeff(e[1]) === 0 && isZeroGuard(g)) return { dir: 'L' }
|
|
1053
|
+
if (isStep(e[1], 'i32.add') && ivCoeff(e[2]) === 0 && isArr(g) && g[0] === 'i32.lt_s' && isLocalGet(g[1], incVar)) return { dir: 'R', B: g[2] }
|
|
1054
|
+
if (isStep(e[2], 'i32.add') && ivCoeff(e[1]) === 0 && isArr(g) && g[0] === 'i32.eq' && isLocalGet(g[1], incVar)) return { dir: 'R', B: g[2] }
|
|
1055
|
+
return null
|
|
1056
|
+
}
|
|
1057
|
+
const ivCoeff = (n) => {
|
|
1058
|
+
if (isLocalGet(n)) {
|
|
1059
|
+
const nm = n[1]
|
|
1060
|
+
if (nm === incVar || derived.has(nm)) return 1
|
|
1061
|
+
return writes.has(nm) ? null : 0 // unwritten ⇒ loop-invariant
|
|
1062
|
+
}
|
|
1063
|
+
if (isI32Const(n)) return 0
|
|
1064
|
+
if (isArr(n) && n[0] === 'global.get') return 0
|
|
1065
|
+
if (isArr(n) && (n[0] === 'i32.add' || n[0] === 'i32.sub') && n.length === 3) {
|
|
1066
|
+
const a = ivCoeff(n[1]), b = ivCoeff(n[2])
|
|
1067
|
+
if (a == null || b == null) return null
|
|
1068
|
+
const c = n[0] === 'i32.add' ? a + b : a - b
|
|
1069
|
+
return c === 0 || c === 1 ? c : null
|
|
1070
|
+
}
|
|
1071
|
+
// `y*w` (inline row base, e.g. idx = y*w + x): invariant×invariant ⇒ coeff 0.
|
|
1072
|
+
// Any IV-dependent factor would be non-unit-stride (stride-w) ⇒ reject.
|
|
1073
|
+
if (isArr(n) && (n[0] === 'i32.mul' || n[0] === 'f64.mul') && n.length === 3)
|
|
1074
|
+
return ivCoeff(n[1]) === 0 && ivCoeff(n[2]) === 0 ? 0 : null
|
|
1075
|
+
// Float-derived index (grid loops compute the row base `y*w` in f64): the index arrives as
|
|
1076
|
+
// `idx = select(wrap(trunc_sat(INV + convert(x))), 0, ≠Inf)`. For an integer counter x,
|
|
1077
|
+
// trunc(C + x) = trunc(C) + x ⇒ stride-1 (the i32 lane offset is added before the trunc); the
|
|
1078
|
+
// Infinity-canon select takes the trunc branch for finite coords (grid indices are finite).
|
|
1079
|
+
// f64.add/sub mirror i32.add/sub; convert/wrap/trunc_sat/tee are coeff-transparent.
|
|
1080
|
+
if (isArr(n) && (n[0] === 'f64.add' || n[0] === 'f64.sub') && n.length === 3) {
|
|
1081
|
+
const a = ivCoeff(n[1]), b = ivCoeff(n[2])
|
|
1082
|
+
if (a == null || b == null) return null
|
|
1083
|
+
const c = n[0] === 'f64.add' ? a + b : a - b
|
|
1084
|
+
return c === 0 || c === 1 ? c : null
|
|
1085
|
+
}
|
|
1086
|
+
if (isArr(n) && (n[0] === 'f64.convert_i32_s' || n[0] === 'i32.wrap_i64' || n[0] === 'i64.trunc_sat_f64_s') && n.length === 2)
|
|
1087
|
+
return ivCoeff(n[1])
|
|
1088
|
+
if (isArr(n) && n[0] === 'local.tee' && n.length === 3) return ivCoeff(n[2])
|
|
1089
|
+
if (isArr(n) && n[0] === 'select') {
|
|
1090
|
+
// Toroidal wrap-select (inline in an address or named): stride-1 interior; flag the peel.
|
|
1091
|
+
const w = isWrapSelect(n)
|
|
1092
|
+
if (w) { needsPeel = true; if (w.dir === 'R' && !rightBs.some(b => exprEq(b, w.B))) rightBs.push(w.B); return 1 }
|
|
1093
|
+
// jz overflow-canon `select(wrap(trunc_sat(…)), 0, ≠Inf)`: finite (grids) ⇒ the trunc branch.
|
|
1094
|
+
if (n.length === 4 && isI32Const(n[2]) && isArr(n[3]) && n[3][0] === 'f64.ne' && isArr(n[3][2]) && n[3][2][0] === 'f64.const' && /inf/i.test(String(n[3][2][1])))
|
|
1095
|
+
return ivCoeff(n[1])
|
|
1096
|
+
}
|
|
1097
|
+
return null
|
|
1098
|
+
}
|
|
1099
|
+
const countSets = (name) => {
|
|
1100
|
+
let k = 0
|
|
1101
|
+
const w = (x) => { if (!isArr(x)) return; if ((x[0] === 'local.set' || x[0] === 'local.tee') && x[1] === name) k++; for (let i = 1; i < x.length; i++) w(x[i]) }
|
|
1102
|
+
for (const s of body) w(s)
|
|
1103
|
+
return k
|
|
1104
|
+
}
|
|
1105
|
+
// Derived IVs: `c = INV + x` (coeff 1) or a toroidal wrap-select; set exactly once, first access a
|
|
1106
|
+
// write. RECURSES into nested tees — O3 CSEs `rc+x` into `(local.tee $pe (i32.add rc x))` inside a
|
|
1107
|
+
// load address, reused by the store. (ivCoeff returns 1 for a wrap-select and flags needsPeel.)
|
|
1108
|
+
for (let pass = 0; pass < 4; pass++) {
|
|
1109
|
+
let added = false
|
|
1110
|
+
const consider = (name, def) => {
|
|
1111
|
+
if (derived.has(name) || fnLocals.get(name) !== 'i32' || countSets(name) !== 1 || ivCoeff(def) !== 1) return
|
|
1112
|
+
let fk = null; for (const t of body) { const k = firstAccess(t, name); if (k) { fk = k; break } }
|
|
1113
|
+
if (fk === 'write') { derived.add(name); added = true }
|
|
1114
|
+
}
|
|
1115
|
+
const walk = (x) => { if (!isArr(x)) return; if ((x[0] === 'local.set' || x[0] === 'local.tee') && typeof x[1] === 'string' && x.length === 3) consider(x[1], x[2]); for (let i = 1; i < x.length; i++) walk(x[i]) }
|
|
1116
|
+
for (const s of body) walk(s)
|
|
1117
|
+
if (!added) break
|
|
1118
|
+
}
|
|
1119
|
+
|
|
1120
|
+
// Scan loads/stores: address `base + (IDX<<K)`, ivCoeff(IDX)=1, base invariant.
|
|
1121
|
+
let laneType = null, stride = -1
|
|
1122
|
+
const offTees = new Map() // $pe → IDX expr (from $pe = IDX<<K)
|
|
1123
|
+
const addrTees = new Map() // $ab → { base, idx }
|
|
1124
|
+
const sites = [] // { kind, base, idx, memBytes }
|
|
1125
|
+
const isInvBase = (b) => (isArr(b) && b[0] === 'global.get') || (isLocalGet(b) && !writes.has(b[1]))
|
|
1126
|
+
const matchAddr = (addr, expectStride = stride) => {
|
|
1127
|
+
let teeName = null, n = addr
|
|
1128
|
+
if (isArr(n) && n[0] === 'local.tee' && n.length === 3) { teeName = n[1]; n = n[2] }
|
|
1129
|
+
if (isLocalGet(n) && addrTees.has(n[1])) { const e = addrTees.get(n[1]); if (teeName) addrTees.set(teeName, e); return e }
|
|
1130
|
+
if (!isArr(n) || n[0] !== 'i32.add' || n.length !== 3) return null
|
|
1131
|
+
const tryOff = (off) => {
|
|
1132
|
+
let ot = null, o = off
|
|
1133
|
+
if (isArr(o) && o[0] === 'local.tee' && o.length === 3) { ot = o[1]; o = o[2] }
|
|
1134
|
+
if (isLocalGet(o) && offTees.has(o[1])) return { idx: offTees.get(o[1]) }
|
|
1135
|
+
if (isArr(o) && o[0] === 'i32.shl' && o.length === 3 && isI32Const(o[2]) && (1 << o[2][1]) === expectStride && ivCoeff(o[1]) === 1) {
|
|
1136
|
+
if (ot) offTees.set(ot, o[1])
|
|
1137
|
+
return { idx: o[1] }
|
|
1138
|
+
}
|
|
1139
|
+
return null
|
|
1140
|
+
}
|
|
1141
|
+
for (const [bi, oi] of [[1, 2], [2, 1]]) {
|
|
1142
|
+
if (!isInvBase(n[bi])) continue
|
|
1143
|
+
const om = tryOff(n[oi])
|
|
1144
|
+
if (om) { const e = { base: n[bi], idx: om.idx }; if (teeName) addrTees.set(teeName, e); return e }
|
|
1145
|
+
}
|
|
1146
|
+
return null
|
|
1147
|
+
}
|
|
1148
|
+
const scan = (node, parent, pi) => {
|
|
1149
|
+
if (!isArr(node)) return true
|
|
1150
|
+
const op = node[0]
|
|
1151
|
+
if (LOAD_OPS[op]) {
|
|
1152
|
+
let addr = node[1], memBytes = 0
|
|
1153
|
+
if (typeof addr === 'string' && addr.startsWith('offset=')) { memBytes = +addr.slice(7); addr = node[2] }
|
|
1154
|
+
const lt = LOAD_OPS[op]
|
|
1155
|
+
if (laneType == null) { if (lt !== 'f64' && lt !== 'f32') return false; laneType = lt; stride = LANE_INFO[lt].stride }
|
|
1156
|
+
else if (lt !== laneType && !(lt === 'f32' && laneType === 'f64')) return false // f32→f64 widening OK
|
|
1157
|
+
// Validate the address at the LOAD's own element stride (f64=8, widening f32=4); the index
|
|
1158
|
+
// must still be stride-1 in elements (ivCoeff===1). The f32 load is promoted in liftExprV.
|
|
1159
|
+
const m = matchAddr(addr, LANE_INFO[lt].stride)
|
|
1160
|
+
if (!m) return false
|
|
1161
|
+
sites.push({ kind: 'load', base: m.base, idx: m.idx, memBytes })
|
|
1162
|
+
return true
|
|
1163
|
+
}
|
|
1164
|
+
if (STORE_OPS[op]) {
|
|
1165
|
+
if (node.length !== 3) return false
|
|
1166
|
+
const st = STORE_OPS[op]
|
|
1167
|
+
if (laneType == null) { if (st !== 'f64' && st !== 'f32') return false; laneType = st; stride = LANE_INFO[st].stride }
|
|
1168
|
+
else if (st !== laneType) return false
|
|
1169
|
+
const m = matchAddr(node[1])
|
|
1170
|
+
if (!m) return false
|
|
1171
|
+
sites.push({ kind: 'store', base: m.base, idx: m.idx, memBytes: 0 })
|
|
1172
|
+
return scan(node[2], node, 2) // value child only
|
|
1173
|
+
}
|
|
1174
|
+
if ((op === 'local.set' || op === 'local.tee') && typeof node[1] === 'string' && node.length === 3) {
|
|
1175
|
+
const v = node[2]
|
|
1176
|
+
if (isArr(v) && v[0] === 'i32.shl' && v.length === 3 && isI32Const(v[2]) && stride > 0 && (1 << v[2][1]) === stride && ivCoeff(v[1]) === 1) offTees.set(node[1], v[1])
|
|
1177
|
+
else matchAddr(['local.tee', node[1], v])
|
|
1178
|
+
}
|
|
1179
|
+
for (let i = 1; i < node.length; i++) if (!scan(node[i], node, i)) return false
|
|
1180
|
+
return true
|
|
1181
|
+
}
|
|
1182
|
+
for (const s of body) if (!scan(s, null, -1)) return null
|
|
1183
|
+
if (!laneType || !sites.some(s => s.kind === "store") || !sites.some(s => s.kind === "load")) return null
|
|
1184
|
+
|
|
1185
|
+
// In-place / loop-carried gate: every access to a WRITTEN base must touch the
|
|
1186
|
+
// SAME element (idx + memarg). Else SIMD reads stale data vs scalar.
|
|
1187
|
+
const elemKey = (s) => `${JSON.stringify(normTee(s.idx))}@${s.memBytes / stride}`
|
|
1188
|
+
for (const st of sites) {
|
|
1189
|
+
if (st.kind !== 'store') continue
|
|
1190
|
+
for (const s of sites) if (exprEq(normTee(s.base), normTee(st.base)) && elemKey(s) !== elemKey(st)) return null
|
|
1191
|
+
}
|
|
1192
|
+
// A pure offset-0 map (every access the same element, no memarg) is tryVectorize's
|
|
1193
|
+
// job — it ran first. Nothing stencil-specific here. (Defensive; ?? order ensures it.)
|
|
1194
|
+
const k0 = elemKey(sites[0])
|
|
1195
|
+
if (sites.every(s => elemKey(s) === k0)) return null
|
|
1196
|
+
|
|
1197
|
+
// Classify locals by TYPE: i32 → addr (index/address, kept scalar), laneType
|
|
1198
|
+
// written → lane (first access must be a write), laneType unwritten → invariant.
|
|
1199
|
+
const referenced = new Set()
|
|
1200
|
+
const collectRefs = (n) => { if (!isArr(n)) return; if ((n[0] === 'local.get' || n[0] === 'local.set' || n[0] === 'local.tee') && typeof n[1] === 'string') referenced.add(n[1]); for (let i = 1; i < n.length; i++) collectRefs(n[i]) }
|
|
1201
|
+
for (const s of body) collectRefs(s)
|
|
1202
|
+
const localKind = new Map()
|
|
1203
|
+
for (const name of referenced) {
|
|
1204
|
+
if (name === incVar) continue
|
|
1205
|
+
const ty = fnLocals.get(name)
|
|
1206
|
+
if (ty === 'i32') { localKind.set(name, 'addr'); continue }
|
|
1207
|
+
if (ty === laneType) {
|
|
1208
|
+
if (writes.has(name)) {
|
|
1209
|
+
let fk = null; for (const s of body) { const k = firstAccess(s, name); if (k) { fk = k; break } }
|
|
1210
|
+
if (fk === 'read') return null // loop-carried float local
|
|
1211
|
+
localKind.set(name, 'lane')
|
|
1212
|
+
} else localKind.set(name, 'invariant')
|
|
1213
|
+
continue
|
|
1214
|
+
}
|
|
1215
|
+
if (!writes.has(name)) { localKind.set(name, 'invariant'); continue }
|
|
1216
|
+
return null // written non-i32 non-lane local
|
|
1217
|
+
}
|
|
1218
|
+
|
|
1219
|
+
// Lift through the shared lifter (addresses kept verbatim; loads → v128.load).
|
|
1220
|
+
const newLanedLocals = new Map(), extraLocals = []
|
|
1221
|
+
const ctx = { laneType, incVar, rampVar: null, rampTemp: null, widenLoads: false, localKind, newLanedLocals, extraLocals, freshIdRef, fail: false, failReason: null }
|
|
1222
|
+
const lifted = []
|
|
1223
|
+
for (const s of body) {
|
|
1224
|
+
const r = liftStmt(s, ctx)
|
|
1225
|
+
if (ctx.fail) return null
|
|
1226
|
+
if (r != null) { if (Array.isArray(r) && r[0] === '__seq__') lifted.push(...r.slice(1)); else lifted.push(r) }
|
|
1227
|
+
}
|
|
1228
|
+
if (!lifted.length) return null
|
|
1229
|
+
|
|
1230
|
+
const id = freshIdRef.next++
|
|
1231
|
+
const simdBoundName = `$__simd_bound${id}`, simdBrkLabel = `$__simd_brk${id}`, simdLoopLabel = `$__simd_loop${id}`
|
|
1232
|
+
const info = LANE_INFO[laneType], lanes = info.lanes
|
|
1233
|
+
const boundExpr = cloneNode(bound) // cloned: also lives in the scalar-tail exit guard
|
|
1234
|
+
// Overshoot-safe bound: a full lanes-wide chunk [x,x+lanes) must stay < bound for
|
|
1235
|
+
// ANY start x (stencils start at 1). `bound-(lanes-1)` — NOT `& ~(lanes-1)`, which
|
|
1236
|
+
// overshoots for a non-multiple start. SIMD reads ⊆ scalar reads ⇒ no new OOB.
|
|
1237
|
+
// A toroidal-wrap stencil additionally PEELS both boundary columns scalar: cap the SIMD at
|
|
1238
|
+
// `min(bound, …rightWrapBoundaries) - (lanes-1)` so no chunk reaches a right-wrap column x=B,
|
|
1239
|
+
// and run x=0 scalar below (where the left wrap fires) so the SIMD starts in the wrap-free interior.
|
|
1240
|
+
const simdCap = rightBs.reduce((acc, b) => ['select', cloneNode(b), acc, ['i32.lt_s', cloneNode(b), acc]], boundExpr)
|
|
1241
|
+
const boundSetup = ['local.set', simdBoundName, ['i32.sub', simdCap, ['i32.const', lanes - 1]]]
|
|
1242
|
+
const simdBlock = ['block', simdBrkLabel,
|
|
1243
|
+
['loop', simdLoopLabel,
|
|
1244
|
+
['br_if', simdBrkLabel, ['i32.eqz', ['i32.lt_s', ['local.get', incVar], ['local.get', simdBoundName]]]],
|
|
1245
|
+
...lifted,
|
|
1246
|
+
['local.set', incVar, ['i32.add', ['local.get', incVar], ['i32.const', lanes]]],
|
|
1247
|
+
['br', simdLoopLabel]]]
|
|
1248
|
+
// Left-boundary peel for a wrap stencil: run the original scalar body once for x=0 (where the wrap
|
|
1249
|
+
// takes its WRAP branch), advancing x to 1 so the SIMD starts in the wrap-free interior. Guarded so
|
|
1250
|
+
// an empty loop (x ≥ bound) is untouched. Right boundary + odd tail: the kept scalar tail (blockNode).
|
|
1251
|
+
const peelStmts = needsPeel
|
|
1252
|
+
? [['if', ['i32.lt_s', ['local.get', incVar], cloneNode(bound)],
|
|
1253
|
+
['then', ...body.map(cloneNode), cloneNode(bl.loopNode[bl.incIdx])]]]
|
|
1254
|
+
: []
|
|
1255
|
+
// LICM-hoisted $__li invariants run ahead of the SIMD block (the scalar tail's
|
|
1256
|
+
// copy inside bl.blockNode re-runs them harmlessly — pure & loop-invariant).
|
|
1257
|
+
const wrapper = ['block', ...preamble.map(cloneNode), ...peelStmts, boundSetup, simdBlock, bl.blockNode]
|
|
1258
|
+
const newLocalDecls = [['local', simdBoundName, 'i32'], ...[...newLanedLocals.values()].map(({ laneName }) => ['local', laneName, 'v128']), ...extraLocals]
|
|
1259
|
+
return { wrapper, newLocalDecls }
|
|
1260
|
+
}
|
|
1261
|
+
|
|
1262
|
+
// ---- Reduction recognizer -------------------------------------------------
|
|
1263
|
+
//
|
|
1264
|
+
// Matches inner loops of shape:
|
|
1265
|
+
// for (let i = 0; i < N; i++) S = OP(S, EXPR(arr[i], ...))
|
|
1266
|
+
// where OP is associative+commutative (REDUCE_OPS table) and EXPR is lane-
|
|
1267
|
+
// pure (operates on the loaded element with at most loop-invariant data).
|
|
1268
|
+
// S is a SCALAR loop-carried accumulator — exempt from the lane-local
|
|
1269
|
+
// "first access must be a write" check.
|
|
1270
|
+
//
|
|
1271
|
+
// Lift:
|
|
1272
|
+
// acc = splat(IDENTITY)
|
|
1273
|
+
// for (i = 0; i < bound & ~(L-1); i += L) acc = OP_v(acc, lifted EXPR)
|
|
1274
|
+
// S = OP(S, horizontal_reduce(acc))
|
|
1275
|
+
// <original scalar tail handles the remainder>
|
|
1276
|
+
//
|
|
1277
|
+
// Float adds are not strictly associative — vectorized reduction differs
|
|
1278
|
+
// from scalar reduction by ulps. Acceptable when bit-exact equality is not
|
|
1279
|
+
// required (which it isn't, by spec, in JS engines either).
|
|
1280
|
+
function tryReduceVectorize(bl, fnLocals, freshIdRef, multiAcc = false) {
|
|
1281
|
+
// Same scaffold as tryVectorize, but no preamble: a reduction block is just the loop.
|
|
1282
|
+
if (!bl || bl.preamble.length) return null
|
|
1283
|
+
const { loopNode, incIdx, incVar } = bl
|
|
1284
|
+
|
|
1285
|
+
// Body is either a bare single-statement reduction —
|
|
1286
|
+
// (local.set $acc (OP (local.get $acc) EXPR)) add/xor/and/or
|
|
1287
|
+
// — or a NaN-canonicalized two-statement min/max reduction —
|
|
1288
|
+
// (local.set $cn (OP (local.get $acc) EXPR))
|
|
1289
|
+
// (local.set $acc (select C (local.get $cn) (T.ne $cn $cn)))
|
|
1290
|
+
const bodyLen = incIdx - 3
|
|
1291
|
+
let accName, opName, reduceEntry, exprNode, canonC = null
|
|
1292
|
+
if (bodyLen === 1) {
|
|
1293
|
+
const stmt = loopNode[3]
|
|
1294
|
+
if (!isArr(stmt) || stmt[0] !== 'local.set' || stmt.length !== 3) return null
|
|
1295
|
+
accName = stmt[1]
|
|
1296
|
+
if (typeof accName !== 'string') return null
|
|
1297
|
+
const rhs = stmt[2]
|
|
1298
|
+
if (!isArr(rhs)) return null
|
|
1299
|
+
const minmax = matchIntMinMaxReduce(rhs, accName)
|
|
1300
|
+
if (minmax) {
|
|
1301
|
+
// Synthetic entry: WASM has the SIMD i32x4.max_s/min_s but no scalar i32.max, so the
|
|
1302
|
+
// horizontal fold + merge below use select (flagged by minmaxSelect). Identity is the
|
|
1303
|
+
// op's neutral — INT_MIN for max, INT_MAX for min. A bare narrow load instead folds
|
|
1304
|
+
// at its own lane width/sign (MINMAX_WIDEN), 16 or 8 lanes per vector.
|
|
1305
|
+
const w = isArr(minmax.exprNode) && minmax.exprNode.length === 2
|
|
1306
|
+
? MINMAX_WIDEN[minmax.exprNode[0]] : null
|
|
1307
|
+
reduceEntry = w ? {
|
|
1308
|
+
simd: `${w.pre}.${minmax.isMax ? 'max' : 'min'}_${w.sign}`,
|
|
1309
|
+
extract: `${w.pre}.extract_lane_${w.sign}`, laneType: w.laneType,
|
|
1310
|
+
identity: ['i32.const', minmax.isMax ? w.lo : w.hi],
|
|
1311
|
+
minmaxSelect: true, isMax: minmax.isMax, accI32: true,
|
|
1312
|
+
} : {
|
|
1313
|
+
simd: minmax.isMax ? 'i32x4.max_s' : 'i32x4.min_s',
|
|
1314
|
+
extract: 'i32x4.extract_lane', laneType: 'i32',
|
|
1315
|
+
identity: ['i32.const', minmax.isMax ? -2147483648 : 2147483647],
|
|
1316
|
+
minmaxSelect: true, isMax: minmax.isMax,
|
|
1317
|
+
}
|
|
1318
|
+
exprNode = minmax.exprNode
|
|
1319
|
+
} else if (rhs[0] === 'block') {
|
|
1320
|
+
// Un-flattened NaN-canon float min/max — the same reduction as the two-statement
|
|
1321
|
+
// canon (bodyLen===2 below), but with the cn-temp set + select still wrapped in a
|
|
1322
|
+
// value-block: (local.set acc (block (result T) (local.set cn (OP acc expr))
|
|
1323
|
+
// (select C (local.get cn) (T.ne cn cn)))). mergeBlocks normally hoists this to the
|
|
1324
|
+
// flat form; recognize the block form directly so vectorization doesn't hinge on
|
|
1325
|
+
// that hoist having run.
|
|
1326
|
+
let bi = 1
|
|
1327
|
+
if (typeof rhs[bi] === 'string' && rhs[bi].startsWith('$')) bi++
|
|
1328
|
+
if (isArr(rhs[bi]) && rhs[bi][0] === 'result') bi++
|
|
1329
|
+
const inner = rhs[bi]
|
|
1330
|
+
const op = isArr(inner) && inner[0] === 'local.set' && isArr(inner[2]) ? inner[2][0] : null
|
|
1331
|
+
reduceEntry = op ? REDUCE_CANON[op] : null
|
|
1332
|
+
if (!reduceEntry) return null
|
|
1333
|
+
const cb = matchCanonBlock(rhs, reduceEntry.laneType)
|
|
1334
|
+
if (!cb || !isArr(cb.core) || cb.core.length !== 3 || !isLocalGet(cb.core[1], accName)) return null
|
|
1335
|
+
opName = op
|
|
1336
|
+
exprNode = cb.core[2]
|
|
1337
|
+
canonC = cb.C
|
|
1338
|
+
} else {
|
|
1339
|
+
if (rhs.length !== 3) return null
|
|
1340
|
+
opName = rhs[0]
|
|
1341
|
+
reduceEntry = REDUCE_OP_LOOKUP.get(opName)
|
|
1342
|
+
if (!reduceEntry || !isLocalGet(rhs[1], accName)) return null
|
|
1343
|
+
exprNode = rhs[2]
|
|
1344
|
+
}
|
|
1345
|
+
} else if (bodyLen === 2) {
|
|
1346
|
+
const s1 = loopNode[3], s2 = loopNode[4]
|
|
1347
|
+
if (!isArr(s1) || s1[0] !== 'local.set' || s1.length !== 3) return null
|
|
1348
|
+
if (!isArr(s2) || s2[0] !== 'local.set' || s2.length !== 3) return null
|
|
1349
|
+
const cnName = s1[1], rhs = s1[2]
|
|
1350
|
+
if (typeof cnName !== 'string' || !isArr(rhs) || rhs.length !== 3) return null
|
|
1351
|
+
opName = rhs[0]
|
|
1352
|
+
reduceEntry = REDUCE_CANON[opName]
|
|
1353
|
+
if (!reduceEntry) return null
|
|
1354
|
+
accName = s2[1]
|
|
1355
|
+
if (typeof accName !== 'string' || accName === cnName) return null
|
|
1356
|
+
const canon = matchCanonSelect(s2[2], reduceEntry.laneType)
|
|
1357
|
+
if (!canon || !isLocalGet(canon.val, cnName)) return null
|
|
1358
|
+
if (!isLocalGet(rhs[1], accName)) return null
|
|
1359
|
+
canonC = canon.C
|
|
1360
|
+
exprNode = rhs[2]
|
|
1361
|
+
} else return null
|
|
1362
|
+
|
|
1363
|
+
// Offset-indexed reductions (matmul `s += A[ai+k]*Bt[bj+k]`): the index `ai+k`
|
|
1364
|
+
// lowers to `(i32.shl (i32.add ai i) K)`, which matchLaneAddr rejects (the IV is
|
|
1365
|
+
// not the bare shift operand). Fold the loop-invariant part into the base —
|
|
1366
|
+
// (base + (INV+i)<<K) → ((base + INV<<K) + i<<K)
|
|
1367
|
+
// so the offset is the bare IV the matcher/lifter already accept. The byte address
|
|
1368
|
+
// is unchanged, so the v128.load reads the same consecutive pair → bit-exact. INV
|
|
1369
|
+
// must be loop-invariant (not written in the loop) and IV-free (coefficient 1).
|
|
1370
|
+
{
|
|
1371
|
+
const writtenInLoop = new Set()
|
|
1372
|
+
;(function wr(n) { if (!isArr(n)) return; if ((n[0] === 'local.set' || n[0] === 'local.tee') && typeof n[1] === 'string') writtenInLoop.add(n[1]); for (let i = 1; i < n.length; i++) wr(n[i]) })(loopNode)
|
|
1373
|
+
const invFree = (n) => !isArr(n) || (!(n[0] === 'local.get' && (n[1] === incVar || writtenInLoop.has(n[1]))) && n.every((c, i) => i === 0 || invFree(c)))
|
|
1374
|
+
let folded = false
|
|
1375
|
+
const foldAddr = (n) => {
|
|
1376
|
+
if (!isArr(n)) return n
|
|
1377
|
+
if (n[0] === 'i32.add' && n.length === 3) {
|
|
1378
|
+
for (const [base, off] of [[n[1], n[2]], [n[2], n[1]]]) {
|
|
1379
|
+
if (isArr(off) && off[0] === 'i32.shl' && off.length === 3 && isArr(off[1]) && off[1][0] === 'i32.add' && off[1].length === 3) {
|
|
1380
|
+
const k = constNum(off[2]), x = off[1][1], y = off[1][2]
|
|
1381
|
+
const xIV = isLocalGet(x, incVar), yIV = isLocalGet(y, incVar)
|
|
1382
|
+
if (k != null && k >= 0 && k <= 3 && xIV !== yIV) {
|
|
1383
|
+
const inv = xIV ? y : x
|
|
1384
|
+
if (invFree(inv)) {
|
|
1385
|
+
folded = true
|
|
1386
|
+
return ['i32.add', ['i32.add', foldAddr(base), ['i32.shl', inv, ['i32.const', k]]], ['i32.shl', ['local.get', incVar], ['i32.const', k]]]
|
|
1387
|
+
}
|
|
1388
|
+
}
|
|
1389
|
+
}
|
|
1390
|
+
}
|
|
1391
|
+
}
|
|
1392
|
+
return n.map(foldAddr)
|
|
1393
|
+
}
|
|
1394
|
+
const fe = foldAddr(exprNode)
|
|
1395
|
+
if (folded) exprNode = fe
|
|
1396
|
+
}
|
|
1397
|
+
|
|
1398
|
+
// Accumulator's declared local type must match the lane element type.
|
|
1399
|
+
// Exception: the widening byte/short sum — i32 accumulator fed by ONE bare
|
|
1400
|
+
// narrow load (`s += u8[i]`), whose LANE type is i8/i16 but reduces into i32.
|
|
1401
|
+
const accType = fnLocals.get(accName)
|
|
1402
|
+
const widen = (opName === 'i32.add' && accType === 'i32' && canonC == null
|
|
1403
|
+
&& isArr(exprNode) && exprNode.length === 2 && WIDEN_LOADS[exprNode[0]]) || null
|
|
1404
|
+
// Widening float min/max: the canon over a sign-matched converted narrow load
|
|
1405
|
+
// (`m = Math.max(m, u8[i])`, acc f64) folds at the load's own width — exact,
|
|
1406
|
+
// since min/max never rounds and u8…i16 values are exact in f64. Only the one
|
|
1407
|
+
// horizontal result converts to f64 for the merge (+ re-canon for a NaN acc).
|
|
1408
|
+
if (canonC != null && (opName === 'f64.max' || opName === 'f64.min') && accType === 'f64'
|
|
1409
|
+
&& isArr(exprNode) && exprNode.length === 2 && MINMAX_CVT[exprNode[0]] && isArr(exprNode[1])) {
|
|
1410
|
+
const w = MINMAX_WIDEN[exprNode[1][0]]
|
|
1411
|
+
if (w && w.sign === MINMAX_CVT[exprNode[0]]) {
|
|
1412
|
+
const isMax = opName === 'f64.max'
|
|
1413
|
+
reduceEntry = {
|
|
1414
|
+
simd: `${w.pre}.${isMax ? 'max' : 'min'}_${w.sign}`,
|
|
1415
|
+
extract: `${w.pre}.extract_lane_${w.sign}`, laneType: w.laneType,
|
|
1416
|
+
identity: ['i32.const', isMax ? w.lo : w.hi],
|
|
1417
|
+
minmaxSelect: true, isMax, accF64: exprNode[0], canonC,
|
|
1418
|
+
}
|
|
1419
|
+
exprNode = exprNode[1]
|
|
1420
|
+
canonC = null
|
|
1421
|
+
}
|
|
1422
|
+
}
|
|
1423
|
+
if (!widen && accType !== (reduceEntry.accI32 ? 'i32' : reduceEntry.accF64 ? 'f64' : reduceEntry.laneType)) return null
|
|
1424
|
+
|
|
1425
|
+
// Bound must be loop-invariant: (local.get $L) or (i32.const N).
|
|
1426
|
+
const { bound, boundLocal } = bl
|
|
1427
|
+
if (!boundLocal && !isI32Const(bound)) return null
|
|
1428
|
+
|
|
1429
|
+
// Scan EXPR for lane-aligned loads. Stores forbidden. Re-references of
|
|
1430
|
+
// accName forbidden (the accumulator only appears in the outer wrapper).
|
|
1431
|
+
const laneType = widen ? widen.laneType : reduceEntry.laneType
|
|
1432
|
+
const stride = LANE_INFO[laneType].stride
|
|
1433
|
+
const addrLocals = new Map()
|
|
1434
|
+
const offsetTees = new Map()
|
|
1435
|
+
let loadCount = 0
|
|
1436
|
+
function scanExpr(node) {
|
|
1437
|
+
if (!isArr(node)) return true
|
|
1438
|
+
const op = node[0]
|
|
1439
|
+
if (LOAD_OPS[op]) {
|
|
1440
|
+
if (LOAD_OPS[op] !== laneType) return false
|
|
1441
|
+
const m = matchLaneAddr(node[1], incVar, addrLocals, offsetTees)
|
|
1442
|
+
if (!m) return false
|
|
1443
|
+
if ((1 << m.strideLog2) !== stride) return false
|
|
1444
|
+
if (m.teeName) addrLocals.set(m.teeName, { strideLog2: m.strideLog2, base: m.base })
|
|
1445
|
+
if (m.offsetTeeName) offsetTees.set(m.offsetTeeName, m.strideLog2)
|
|
1446
|
+
loadCount++
|
|
1447
|
+
return true
|
|
1448
|
+
}
|
|
1449
|
+
if (STORE_OPS[op]) return false
|
|
1450
|
+
if (op === 'local.set' || op === 'local.tee') return false // no intermediates
|
|
1451
|
+
if (op === 'local.get' && node[1] === accName) return false
|
|
1452
|
+
for (let i = 1; i < node.length; i++) if (!scanExpr(node[i])) return false
|
|
1453
|
+
return true
|
|
1454
|
+
}
|
|
1455
|
+
if (!scanExpr(exprNode)) return null
|
|
1456
|
+
if (loadCount === 0) return null
|
|
1457
|
+
// Soundness gate for offset-tee resolution (see tryVectorize).
|
|
1458
|
+
for (const [name, k] of offsetTees) {
|
|
1459
|
+
if (_offsetLocalStride([exprNode], name, incVar) !== k) return null
|
|
1460
|
+
}
|
|
1461
|
+
|
|
1462
|
+
// Classify locals referenced in EXPR. Anything not the induction var or an
|
|
1463
|
+
// address-tee is invariant (we forbade local.set/tee in scanExpr).
|
|
1464
|
+
const referenced = new Set()
|
|
1465
|
+
const collectRefs = (n) => {
|
|
1466
|
+
if (!isArr(n)) return
|
|
1467
|
+
if (n[0] === 'local.get' && typeof n[1] === 'string') referenced.add(n[1])
|
|
1468
|
+
for (let i = 1; i < n.length; i++) collectRefs(n[i])
|
|
1469
|
+
}
|
|
1470
|
+
collectRefs(exprNode)
|
|
1471
|
+
const localKind = new Map()
|
|
1472
|
+
for (const name of referenced) {
|
|
1473
|
+
if (name === incVar) continue
|
|
1474
|
+
if (addrLocals.has(name) || offsetTees.has(name)) { localKind.set(name, 'addr'); continue }
|
|
1475
|
+
localKind.set(name, 'invariant')
|
|
1476
|
+
}
|
|
1477
|
+
for (const name of addrLocals.keys()) localKind.set(name, 'addr')
|
|
1478
|
+
for (const name of offsetTees.keys()) localKind.set(name, 'addr')
|
|
1479
|
+
|
|
1480
|
+
const ctx = { laneType, incVar, rampVar: null, rampTemp: null, widenLoads: false, localKind, newLanedLocals: new Map(), extraLocals: [], freshIdRef, fail: false, failReason: null }
|
|
1481
|
+
const liftedExpr = liftExprV(exprNode, ctx)
|
|
1482
|
+
if (ctx.fail) return null
|
|
1483
|
+
if (ctx.newLanedLocals.size > 0 || ctx.extraLocals.length > 0) return null
|
|
1484
|
+
|
|
1485
|
+
// Synthesize SIMD prefix block + horizontal reduce + (preserved scalar tail).
|
|
1486
|
+
const id = freshIdRef.next++
|
|
1487
|
+
const simdBoundName = `$__simd_bound${id}`
|
|
1488
|
+
const simdAccName = `$__simd_acc${id}` // accumulator 0 — the one the merge folds
|
|
1489
|
+
const simdBrkLabel = `$__simd_brk${id}`
|
|
1490
|
+
const simdLoopLabel = `$__simd_loop${id}`
|
|
1491
|
+
const info = LANE_INFO[laneType]
|
|
1492
|
+
const lanes = info.lanes
|
|
1493
|
+
const boundExpr = boundLocal ? ['local.get', boundLocal] : bound
|
|
1494
|
+
|
|
1495
|
+
// Multi-accumulator unroll. A reduction's loop-carried accumulator is a latency
|
|
1496
|
+
// chain — each iteration's op waits on the previous result, so a single vector
|
|
1497
|
+
// accumulator runs at FP-op latency, not throughput. N INDEPENDENT accumulators
|
|
1498
|
+
// (each summing every Nth lane-chunk, combined at the end) expose instruction-
|
|
1499
|
+
// level parallelism and hide the latency — ~2x on a dot/FIR reduction. It is
|
|
1500
|
+
// DETERMINISTIC: only the reduction's reassociation widens (8 partial sums vs 2),
|
|
1501
|
+
// the same kind the existing 2-lane fold already does, identical on every engine.
|
|
1502
|
+
// Restricted to the plain horizontal-fold FP path (not min/max-select, the
|
|
1503
|
+
// narrow-widening sums, or NaN-canon — those have their own fold shapes).
|
|
1504
|
+
const plainReduce = !reduceEntry.minmaxSelect && !widen && canonC == null
|
|
1505
|
+
const NACC = (multiAcc && plainReduce && (laneType === 'f64' || laneType === 'f32')) ? 4 : 1
|
|
1506
|
+
const accK = (k) => k === 0 ? simdAccName : `$__simd_acc${id}_${k}`
|
|
1507
|
+
const laneBytes = lanes * stride
|
|
1508
|
+
|
|
1509
|
+
// Widening sum: the ACCUMULATOR vector is i32x4 regardless of the (narrow)
|
|
1510
|
+
// lane type; each iteration's 16-byte load collapses via extadd_pairwise.
|
|
1511
|
+
const accSplat = widen ? 'i32x4.splat' : info.splat
|
|
1512
|
+
const accumOperand = widen ? widen.steps.reduce((e, s) => [s, e], liftedExpr) : liftedExpr
|
|
1513
|
+
// Accumulator k reads the same lane-aligned data as acc 0, shifted by k chunks
|
|
1514
|
+
// (k·laneBytes). Acc 0 keeps the address tees (it sets them); acc k>0 reads the
|
|
1515
|
+
// tee'd address (normTee → local.get) and adds the byte offset to each load.
|
|
1516
|
+
const offsetLoads = (node, off) => !isArr(node) ? node
|
|
1517
|
+
: node[0] === 'v128.load' ? ['v128.load', ['i32.add', node[1], ['i32.const', off]]]
|
|
1518
|
+
: node.map(c => offsetLoads(c, off))
|
|
1519
|
+
const accOperandFor = (k) => k === 0 ? accumOperand : offsetLoads(normTee(accumOperand), k * laneBytes)
|
|
1520
|
+
|
|
1521
|
+
const initAcc = []
|
|
1522
|
+
for (let k = 0; k < NACC; k++) initAcc.push(['local.set', accK(k), [accSplat, reduceEntry.constNode ?? reduceEntry.identity]])
|
|
1523
|
+
const loopBody = []
|
|
1524
|
+
for (let k = 0; k < NACC; k++) loopBody.push(['local.set', accK(k), [reduceEntry.simd, ['local.get', accK(k)], accOperandFor(k)]])
|
|
1525
|
+
loopBody.push(['local.set', incVar, ['i32.add', ['local.get', incVar], ['i32.const', lanes * NACC]]])
|
|
1526
|
+
const simdBlock = ['block', simdBrkLabel,
|
|
1527
|
+
['loop', simdLoopLabel,
|
|
1528
|
+
['br_if', simdBrkLabel,
|
|
1529
|
+
['i32.eqz', ['i32.lt_s', ['local.get', incVar], ['local.get', simdBoundName]]]],
|
|
1530
|
+
...loopBody,
|
|
1531
|
+
['br', simdLoopLabel]
|
|
1532
|
+
]
|
|
1533
|
+
]
|
|
1534
|
+
// Combine the N accumulators into acc 0 (lane-wise) before the horizontal fold.
|
|
1535
|
+
const combineAccs = []
|
|
1536
|
+
for (let k = 1; k < NACC; k++) combineAccs.push(['local.set', simdAccName, [reduceEntry.simd, ['local.get', simdAccName], ['local.get', accK(k)]]])
|
|
1537
|
+
|
|
1538
|
+
// Horizontal fold + merge into the live accumulator.
|
|
1539
|
+
const extraDecls = []
|
|
1540
|
+
let mergeStmts
|
|
1541
|
+
if (reduceEntry.minmaxSelect) {
|
|
1542
|
+
// No scalar i32.max/min — fold via select through an i32 temp (no exponential
|
|
1543
|
+
// operand duplication): ht = lane0; ht = minmax(ht, lane_k); acc = minmax(acc, ht).
|
|
1544
|
+
// `select(a,b,(gt|lt)_s a b)` = a when it's the larger/smaller, i.e. minmax(a,b).
|
|
1545
|
+
const cmpOp = reduceEntry.isMax ? 'i32.gt_s' : 'i32.lt_s'
|
|
1546
|
+
const ht = `$__simd_h${id}`
|
|
1547
|
+
extraDecls.push(['local', ht, 'i32'])
|
|
1548
|
+
const lane = (k) => [reduceEntry.extract, k, ['local.get', simdAccName]]
|
|
1549
|
+
const minmaxSel = (a, b) => ['select', a, b, [cmpOp, a, b]]
|
|
1550
|
+
mergeStmts = [['local.set', ht, lane(0)]]
|
|
1551
|
+
for (let k = 1; k < lanes; k++) mergeStmts.push(['local.set', ht, minmaxSel(lane(k), ['local.get', ht])])
|
|
1552
|
+
if (reduceEntry.accF64) {
|
|
1553
|
+
// Widening canon merge: one convert of the horizontal result, the scalar
|
|
1554
|
+
// f64 op against the live acc, then re-canon (a NaN-seeded acc must still
|
|
1555
|
+
// cross as the canonical NaN when the scalar tail is empty).
|
|
1556
|
+
mergeStmts.push(['local.set', accName,
|
|
1557
|
+
[opName, ['local.get', accName], [reduceEntry.accF64, ['local.get', ht]]]])
|
|
1558
|
+
mergeStmts.push(['local.set', accName,
|
|
1559
|
+
['select', reduceEntry.canonC, ['local.get', accName],
|
|
1560
|
+
['f64.ne', ['local.get', accName], ['local.get', accName]]]])
|
|
1561
|
+
} else {
|
|
1562
|
+
mergeStmts.push(['local.set', accName, minmaxSel(['local.get', accName], ['local.get', ht])])
|
|
1563
|
+
}
|
|
1564
|
+
} else {
|
|
1565
|
+
// Horizontal fold: scalar.op(extract 0, extract 1, …, extract L-1).
|
|
1566
|
+
// Widening sum folds the 4 i32x4 PARTIALS, not the (narrow) data lanes.
|
|
1567
|
+
const foldLanes = widen ? 4 : lanes
|
|
1568
|
+
let horiz = [reduceEntry.extract, 0, ['local.get', simdAccName]]
|
|
1569
|
+
for (let k = 1; k < foldLanes; k++) {
|
|
1570
|
+
horiz = [opName, horiz, [reduceEntry.extract, k, ['local.get', simdAccName]]]
|
|
1571
|
+
}
|
|
1572
|
+
// Merge the SIMD result into the live accumulator. For canon (min/max) the
|
|
1573
|
+
// merged value is re-canonicalized so a NaN that surfaced only in the SIMD
|
|
1574
|
+
// range still crosses as the canonical NaN when the scalar tail is empty.
|
|
1575
|
+
const merged = [opName, ['local.get', accName], horiz]
|
|
1576
|
+
mergeStmts = canonC == null
|
|
1577
|
+
? [['local.set', accName, merged]]
|
|
1578
|
+
: [['local.set', accName, merged],
|
|
1579
|
+
['local.set', accName,
|
|
1580
|
+
['select', canonC, ['local.get', accName],
|
|
1581
|
+
[`${laneType}.ne`, ['local.get', accName], ['local.get', accName]]]]]
|
|
1582
|
+
}
|
|
1583
|
+
// Overshoot-safe SIMD bound: stop while a full `lanes`-wide load stays in
|
|
1584
|
+
// range, for ANY induction start (the min/max idiom seeds m=a[0] and starts
|
|
1585
|
+
// at i=1, which `& ~(lanes-1)` masking would run one lane past the end). For
|
|
1586
|
+
// a lane-aligned start this yields the same iteration set as masking; the
|
|
1587
|
+
// scalar tail (original `i<bound` guard) cleans up regardless.
|
|
1588
|
+
// A full N·lanes-wide step (all N accumulators) must stay in range.
|
|
1589
|
+
const boundSetup = ['local.set', simdBoundName, ['i32.sub', boundExpr, ['i32.const', lanes * NACC - 1]]]
|
|
1590
|
+
|
|
1591
|
+
// Narrow-widened entries seed the vector acc with a LANE-domain neutral (e.g.
|
|
1592
|
+
// 0 for u8-max) — only neutral once real lanes fold in. Guard the whole SIMD
|
|
1593
|
+
// prefix incl. the merge so a zero-iteration range can't clamp the live acc
|
|
1594
|
+
// toward the identity. Full-width entries use absolute neutrals; unguarded.
|
|
1595
|
+
// (The guarded path is always NACC=1 — accI32/accF64 are non-plain reductions.)
|
|
1596
|
+
const core = reduceEntry.accI32 || reduceEntry.accF64
|
|
1597
|
+
? [['if', ['i32.lt_s', ['local.get', incVar], ['local.get', simdBoundName]],
|
|
1598
|
+
['then', ...initAcc, simdBlock, ...combineAccs, ...mergeStmts]]]
|
|
1599
|
+
: [...initAcc, simdBlock, ...combineAccs, ...mergeStmts]
|
|
1600
|
+
const wrapper = ['block', boundSetup, ...core, bl.blockNode]
|
|
1601
|
+
const newLocalDecls = [
|
|
1602
|
+
['local', simdBoundName, 'i32'],
|
|
1603
|
+
['local', simdAccName, 'v128'],
|
|
1604
|
+
...Array.from({ length: NACC - 1 }, (_, k) => ['local', accK(k + 1), 'v128']),
|
|
1605
|
+
...extraDecls,
|
|
1606
|
+
]
|
|
1607
|
+
return { wrapper, newLocalDecls }
|
|
1608
|
+
}
|
|
1609
|
+
|
|
1610
|
+
// Bit-exact f64 map-reduce (the direct-summation n-body force loop). A loop that
|
|
1611
|
+
// accumulates one or more f64 reductions whose per-iteration contribution is computed
|
|
1612
|
+
// INDEPENDENTLY of the accumulators. Process 2 iterations per step in f64x2 — every lane
|
|
1613
|
+
// op (sub/mul/add/div/sqrt) is IEEE-754-identical to scalar f64 (no FMA in non-relaxed
|
|
1614
|
+
// SIMD) — then accumulate each accumulator's two lane contributions IN SCALAR ORDER, so
|
|
1615
|
+
// the reduction is BIT-EXACT (unlike the reassociating tryReduceVectorize). Wins when the
|
|
1616
|
+
// per-element compute is expensive (a sqrt + reciprocal) so the 2-wide arithmetic
|
|
1617
|
+
// outweighs the serial lane-accumulation. The original block is preserved as the ≤1
|
|
1618
|
+
// scalar remainder, continuing the accumulators. Returns {wrapper, newLocalDecls} or null.
|
|
1619
|
+
function tryMapReduceVectorize(bl, fnLocals, freshIdRef) {
|
|
1620
|
+
if (!bl || bl.preamble.length) return null
|
|
1621
|
+
const { incVar, bound, boundLocal, body } = bl
|
|
1622
|
+
if (!boundLocal && !isI32Const(bound)) return null
|
|
1623
|
+
if (body.length < 2) return null
|
|
1624
|
+
|
|
1625
|
+
// Every body stmt must be `(local.set $x EXPR)`. An accumulator reads its own target
|
|
1626
|
+
// through `f64.add` (`acc = acc + EXPR`); the rest are per-iteration lane locals. (One
|
|
1627
|
+
// write per acc — duplicate writes would break the per-acc ordering.)
|
|
1628
|
+
for (const s of body) if (!(isArr(s) && s[0] === 'local.set' && typeof s[1] === 'string' && s.length === 3)) return null
|
|
1629
|
+
const accSet = new Set()
|
|
1630
|
+
for (const s of body) if (isArr(s[2]) && s[2][0] === 'f64.add' && isLocalGet(s[2][1], s[1])) accSet.add(s[1])
|
|
1631
|
+
if (!accSet.size) return null
|
|
1632
|
+
const writeCount = new Map()
|
|
1633
|
+
for (const s of body) writeCount.set(s[1], (writeCount.get(s[1]) || 0) + 1)
|
|
1634
|
+
for (const a of accSet) { if (writeCount.get(a) !== 1 || fnLocals.get(a) !== 'f64') return null }
|
|
1635
|
+
|
|
1636
|
+
// Address tees: locals that equal `ind << K`. f64 loads must be stride-8 (K=3) so one
|
|
1637
|
+
// f64x2.load (16 bytes) covers iterations j and j+1 — consecutive elements.
|
|
1638
|
+
const offsetTees = new Map()
|
|
1639
|
+
const allNames = new Set()
|
|
1640
|
+
const gather = (n) => { if (!isArr(n)) return; if ((n[0] === 'local.set' || n[0] === 'local.tee') && typeof n[1] === 'string') allNames.add(n[1]); for (let i = 1; i < n.length; i++) gather(n[i]) }
|
|
1641
|
+
for (const s of body) gather(s)
|
|
1642
|
+
for (const name of allNames) { const k = _offsetLocalStride(body, name, incVar); if (k != null) offsetTees.set(name, k) }
|
|
1643
|
+
|
|
1644
|
+
// f64x2 lift: load → f64x2.load (2 consecutive), const/invariant → splat, a lane local
|
|
1645
|
+
// → its f64x2 temp, sub/mul/add/div → f64x2.OP, sqrt → f64x2.sqrt. Anything else bails.
|
|
1646
|
+
const laneV = new Map()
|
|
1647
|
+
const newLocalDecls = []
|
|
1648
|
+
const fresh = () => { const n = `$__mr${freshIdRef.next++}`; newLocalDecls.push(['local', n, 'v128']); return n }
|
|
1649
|
+
let bad = false
|
|
1650
|
+
const lift = (e) => {
|
|
1651
|
+
if (bad || !isArr(e)) { bad = true; return null }
|
|
1652
|
+
const op = e[0]
|
|
1653
|
+
if (op === 'f64.const') return ['f64x2.splat', e]
|
|
1654
|
+
if (op === 'f64.load') {
|
|
1655
|
+
const m = matchLaneAddr(e[1], incVar, new Map(), offsetTees)
|
|
1656
|
+
if (!m || m.strideLog2 !== 3) { bad = true; return null }
|
|
1657
|
+
return ['v128.load', e[1]] // 16 bytes = 2 consecutive f64s; the f64x2 op reads them
|
|
1658
|
+
}
|
|
1659
|
+
if (op === 'local.get' && typeof e[1] === 'string') {
|
|
1660
|
+
if (e[1] === incVar || accSet.has(e[1])) { bad = true; return null } // IV-as-data / acc-dependent contribution
|
|
1661
|
+
if (laneV.has(e[1])) return ['local.get', laneV.get(e[1])]
|
|
1662
|
+
if (writeCount.has(e[1])) { bad = true; return null } // a body local used BEFORE its set this iteration → loop-carried, not a fresh lane
|
|
1663
|
+
return ['f64x2.splat', e] // genuine loop-invariant scalar (xi, …)
|
|
1664
|
+
}
|
|
1665
|
+
if ((op === 'f64.add' || op === 'f64.sub' || op === 'f64.mul' || op === 'f64.div') && e.length === 3)
|
|
1666
|
+
return [op.replace('f64.', 'f64x2.'), lift(e[1]), lift(e[2])]
|
|
1667
|
+
if (op === 'f64.sqrt' && e.length === 2) return ['f64x2.sqrt', lift(e[1])]
|
|
1668
|
+
bad = true; return null
|
|
1669
|
+
}
|
|
1670
|
+
|
|
1671
|
+
// Lifted body: setup lanes → f64x2 temps (in order, so the offset tee in the first load
|
|
1672
|
+
// is set before later loads read it); each accumulator → a temp + two in-order adds.
|
|
1673
|
+
const lifted = []
|
|
1674
|
+
for (const s of body) {
|
|
1675
|
+
if (accSet.has(s[1])) {
|
|
1676
|
+
const cV = fresh()
|
|
1677
|
+
const v = lift(s[2][2])
|
|
1678
|
+
if (bad) return null
|
|
1679
|
+
lifted.push(['local.set', cV, v],
|
|
1680
|
+
['local.set', s[1], ['f64.add', ['local.get', s[1]], ['f64x2.extract_lane', 0, ['local.get', cV]]]],
|
|
1681
|
+
['local.set', s[1], ['f64.add', ['local.get', s[1]], ['f64x2.extract_lane', 1, ['local.get', cV]]]])
|
|
1682
|
+
} else {
|
|
1683
|
+
const tv = fresh()
|
|
1684
|
+
laneV.set(s[1], tv)
|
|
1685
|
+
const v = lift(s[2])
|
|
1686
|
+
if (bad) return null
|
|
1687
|
+
lifted.push(['local.set', tv, v])
|
|
1688
|
+
}
|
|
1689
|
+
}
|
|
1690
|
+
if (bad || !lifted.length) return null
|
|
1691
|
+
|
|
1692
|
+
// SIMD prefix over the even prefix [0, bound & ~1); the original block is the ≤1 scalar
|
|
1693
|
+
// remainder (continues j and the accumulators). IV advances by 2.
|
|
1694
|
+
const id = freshIdRef.next++
|
|
1695
|
+
const simdBoundName = `$__mrb${id}`, simdBrk = `$__mrbrk${id}`, simdLoop = `$__mrl${id}`
|
|
1696
|
+
const boundExpr = boundLocal ? ['local.get', boundLocal] : ['i32.const', constNum(bound)]
|
|
1697
|
+
const simdBlock = ['block', simdBrk,
|
|
1698
|
+
['loop', simdLoop,
|
|
1699
|
+
['br_if', simdBrk, ['i32.eqz', ['i32.lt_s', ['local.get', incVar], ['local.get', simdBoundName]]]],
|
|
1700
|
+
...lifted,
|
|
1701
|
+
['local.set', incVar, ['i32.add', ['local.get', incVar], ['i32.const', 2]]],
|
|
1702
|
+
['br', simdLoop]]]
|
|
1703
|
+
const boundSetup = ['local.set', simdBoundName, ['i32.and', boundExpr, ['i32.const', -2]]]
|
|
1704
|
+
const wrapper = ['block', boundSetup, simdBlock, bl.blockNode]
|
|
1705
|
+
return { wrapper, newLocalDecls: [['local', simdBoundName, 'i32'], ...newLocalDecls] }
|
|
1706
|
+
}
|
|
1707
|
+
|
|
1708
|
+
// Scalar locals that are ALWAYS computed as `(i32.add base (i32.shl ind K))`
|
|
1709
|
+
// or aliased to such an address are "address tees", not lane data. They stay
|
|
1710
|
+
// scalar i32 in the lifted body.
|
|
1711
|
+
function _isAddressLocal(body, name, ind) {
|
|
1712
|
+
let onlyAsAddrTee = true
|
|
1713
|
+
let foundTee = false
|
|
1714
|
+
function walk(n) {
|
|
1715
|
+
if (!isArr(n)) return
|
|
1716
|
+
if (n[0] === 'local.tee' && n[1] === name) {
|
|
1717
|
+
foundTee = true
|
|
1718
|
+
// Check the value is a lane-address shape
|
|
1719
|
+
const m = matchLaneAddr(['local.tee', name, n[2]], ind)
|
|
1720
|
+
if (!m) onlyAsAddrTee = false
|
|
1721
|
+
return
|
|
1722
|
+
}
|
|
1723
|
+
if (n[0] === 'local.set' && n[1] === name) {
|
|
1724
|
+
// A set-not-tee: check value shape
|
|
1725
|
+
const m = matchLaneAddr(['local.tee', name, n[2]], ind)
|
|
1726
|
+
if (!m) onlyAsAddrTee = false
|
|
1727
|
+
foundTee = true
|
|
1728
|
+
return
|
|
1729
|
+
}
|
|
1730
|
+
for (let i = 1; i < n.length; i++) walk(n[i])
|
|
1731
|
+
}
|
|
1732
|
+
for (const s of body) walk(s)
|
|
1733
|
+
return foundTee && onlyAsAddrTee
|
|
1734
|
+
}
|
|
1735
|
+
|
|
1736
|
+
// ---- Lifter ----------------------------------------------------------------
|
|
1737
|
+
|
|
1738
|
+
function getOrAllocLanedLocal(name, ctx) {
|
|
1739
|
+
let r = ctx.newLanedLocals.get(name)
|
|
1740
|
+
if (!r) {
|
|
1741
|
+
r = { laneName: `${name}__v`, origName: name }
|
|
1742
|
+
ctx.newLanedLocals.set(name, r)
|
|
1743
|
+
}
|
|
1744
|
+
return r
|
|
1745
|
+
}
|
|
1746
|
+
|
|
1747
|
+
// Wrap an already-lifted v128 value `coreV` in per-lane NaN canonicalization:
|
|
1748
|
+
// v128.bitselect(splat(C), coreV, laneNe(coreV, coreV))
|
|
1749
|
+
// coreV is referenced three times. When it's a bare local.get (the common
|
|
1750
|
+
// flattened form, where the core was already hoisted to a temp) we share it
|
|
1751
|
+
// directly — matching the scalar select, which likewise reads the temp thrice.
|
|
1752
|
+
// Otherwise we materialize a fresh v128 temp so the core evaluates once.
|
|
1753
|
+
function liftCanon(coreV, C, ctx, info) {
|
|
1754
|
+
const laneNe = ctx.laneType === 'f32' ? 'f32x4.ne' : 'f64x2.ne'
|
|
1755
|
+
const splatC = [info.splat, C]
|
|
1756
|
+
if (isArr(coreV) && coreV[0] === 'local.get') {
|
|
1757
|
+
return ['v128.bitselect', splatC, coreV, [laneNe, coreV, coreV]]
|
|
1758
|
+
}
|
|
1759
|
+
const tmp = `$__canon${ctx.freshIdRef.next++}`
|
|
1760
|
+
ctx.extraLocals.push(['local', tmp, 'v128'])
|
|
1761
|
+
const g = ['local.get', tmp]
|
|
1762
|
+
return ['block', ['result', 'v128'],
|
|
1763
|
+
['local.set', tmp, coreV],
|
|
1764
|
+
['v128.bitselect', splatC, g, [laneNe, g, g]]]
|
|
1765
|
+
}
|
|
1766
|
+
|
|
1767
|
+
// --why-not-simd diagnostics. `_whyNotActive` is armed only for the duration of a
|
|
1768
|
+
// vectorizeLaneLocal call made with the flag on (cleared on exit — never leaks into
|
|
1769
|
+
// codegen, which never reads it). `_whyNotReason` captures the FIRST (deepest) lift
|
|
1770
|
+
// bail for the block currently under the recognizer chain; the walk reads it after.
|
|
1771
|
+
let _whyNotActive = false
|
|
1772
|
+
let _whyNotReason = null
|
|
1773
|
+
|
|
1774
|
+
// Mark a lift bail and record its reason. First-write-wins: the innermost failing op
|
|
1775
|
+
// sets ctx.failReason; outer frames see ctx.fail already set and return without
|
|
1776
|
+
// overwriting, so the reason names the actual blocking op, not a wrapper.
|
|
1777
|
+
const liftFail = (ctx, reason) => {
|
|
1778
|
+
ctx.fail = true
|
|
1779
|
+
if (ctx.failReason == null) ctx.failReason = reason
|
|
1780
|
+
if (_whyNotActive && _whyNotReason == null) _whyNotReason = reason
|
|
1781
|
+
return null
|
|
1782
|
+
}
|
|
1783
|
+
|
|
1784
|
+
/** Lift a statement. Returns lifted stmt, or null to skip, or ['__seq__', ...] for multiple. */
|
|
1785
|
+
function liftStmt(stmt, ctx) {
|
|
1786
|
+
if (!isArr(stmt)) {
|
|
1787
|
+
// Bare strings like "drop" — produced by stack-form WAT. We unwrap value-blocks
|
|
1788
|
+
// separately so an isolated "drop" should not appear here, but tolerate it.
|
|
1789
|
+
if (stmt === 'drop') return null
|
|
1790
|
+
return liftFail(ctx, 'non-array statement')
|
|
1791
|
+
}
|
|
1792
|
+
const op = stmt[0]
|
|
1793
|
+
|
|
1794
|
+
if (op === 'local.set' && typeof stmt[1] === 'string' && stmt.length === 3) {
|
|
1795
|
+
const name = stmt[1]
|
|
1796
|
+
const kind = ctx.localKind.get(name)
|
|
1797
|
+
if (kind === 'addr') {
|
|
1798
|
+
// Address-only local: lift the value as-is (it's i32 arithmetic on ind).
|
|
1799
|
+
return ['local.set', name, stmt[2]]
|
|
1800
|
+
}
|
|
1801
|
+
if (kind === 'lane') {
|
|
1802
|
+
const { laneName } = getOrAllocLanedLocal(name, ctx)
|
|
1803
|
+
const v = liftExprV(stmt[2], ctx)
|
|
1804
|
+
if (ctx.fail) return null
|
|
1805
|
+
return ['local.set', laneName, v]
|
|
1806
|
+
}
|
|
1807
|
+
return liftFail(ctx, `local.set ${name}: loop-carried or unclassified local`)
|
|
1808
|
+
}
|
|
1809
|
+
|
|
1810
|
+
if (STORE_OPS[op]) {
|
|
1811
|
+
const simdStore = 'v128.store'
|
|
1812
|
+
const addr = stmt[1] // we leave addresses as-is (scalar i32 expressions)
|
|
1813
|
+
const val = liftExprV(stmt[2], ctx)
|
|
1814
|
+
if (ctx.fail) return null
|
|
1815
|
+
// Handle memarg if present (last positional after addr/val): unlikely in
|
|
1816
|
+
// pre-watr IR for this shape; bail if more than 3 children.
|
|
1817
|
+
if (stmt.length !== 3) return liftFail(ctx, `${op} with memarg`)
|
|
1818
|
+
return [simdStore, addr, val]
|
|
1819
|
+
}
|
|
1820
|
+
|
|
1821
|
+
// (block (result T) STMTS... TAIL_EXPR) followed by sibling "drop" — we get
|
|
1822
|
+
// the block alone here; the "drop" is a separate sibling and is returned as
|
|
1823
|
+
// null by the next call. Strip the wrapper, lift the inner stmts; the
|
|
1824
|
+
// dropped-tail expr is discarded.
|
|
1825
|
+
if (op === 'block') {
|
|
1826
|
+
// Block may be: ['block', LABEL?, RESULT?, ...stmts]
|
|
1827
|
+
let i = 1
|
|
1828
|
+
if (typeof stmt[i] === 'string' && stmt[i].startsWith('$')) i++
|
|
1829
|
+
const hasResult = isArr(stmt[i]) && stmt[i][0] === 'result'
|
|
1830
|
+
if (hasResult) i++
|
|
1831
|
+
const inner = stmt.slice(i)
|
|
1832
|
+
const stmts = hasResult ? inner.slice(0, inner.length - 1) : inner
|
|
1833
|
+
const out = ['__seq__']
|
|
1834
|
+
for (const s of stmts) {
|
|
1835
|
+
const lifted = liftStmt(s, ctx)
|
|
1836
|
+
if (ctx.fail) return null
|
|
1837
|
+
if (lifted == null) continue
|
|
1838
|
+
if (Array.isArray(lifted) && lifted[0] === '__seq__') out.push(...lifted.slice(1))
|
|
1839
|
+
else out.push(lifted)
|
|
1840
|
+
}
|
|
1841
|
+
return out
|
|
1842
|
+
}
|
|
1843
|
+
|
|
1844
|
+
// Standalone conditional store: `if (COND) { …inter; store(addr,A) } [else { …inter; store(addr,B) }]`.
|
|
1845
|
+
// Both arms end in a store to the SAME address; a missing else keeps the current value. Speculatively
|
|
1846
|
+
// lift both arms (intermediate sets become lane locals; masked lanes are discarded — lane-pure ops
|
|
1847
|
+
// are trap-free) and emit ONE store of `bitselect(A, B, mask(COND))`. Unlocks per-pixel conditional
|
|
1848
|
+
// maps like lorenz's i32x4 trail fade (`if (p & 0xffffff) px[i] = fade(p)`).
|
|
1849
|
+
if (op === 'if' && isArr(stmt[2]) && stmt[2][0] === 'then') {
|
|
1850
|
+
const armOf = (arm) => {
|
|
1851
|
+
let body = arm.slice(1)
|
|
1852
|
+
if (body.length === 1 && isArr(body[0]) && body[0][0] === 'block') { // unwrap a single block arm
|
|
1853
|
+
const b = body[0]; let i = 1
|
|
1854
|
+
if (typeof b[i] === 'string' && b[i].startsWith('$')) i++
|
|
1855
|
+
if (isArr(b[i]) && b[i][0] === 'result') i++
|
|
1856
|
+
body = b.slice(i)
|
|
1857
|
+
}
|
|
1858
|
+
const last = body[body.length - 1]
|
|
1859
|
+
if (!isArr(last) || !STORE_OPS[last[0]] || last.length !== 3) return null
|
|
1860
|
+
return { inter: body.slice(0, -1), addr: last[1], val: last[2], store: last[0] }
|
|
1861
|
+
}
|
|
1862
|
+
const thenA = armOf(stmt[2]), elseA = (isArr(stmt[3]) && stmt[3][0] === 'else') ? armOf(stmt[3]) : null
|
|
1863
|
+
if (!thenA || (isArr(stmt[3]) && !elseA)) return liftFail(ctx, 'if-store: arm is not a conditional store')
|
|
1864
|
+
if (elseA && (JSON.stringify(thenA.addr) !== JSON.stringify(elseA.addr) || thenA.store !== elseA.store)) return liftFail(ctx, 'if-store: arms store differently')
|
|
1865
|
+
// mask from COND: a lane comparison, or (i32) a truthy test `lift(cond) != 0`.
|
|
1866
|
+
let cond = stmt[1]
|
|
1867
|
+
if (isArr(cond) && cond[0] === 'i32.ne' && isI32Const(cond[2]) && cond[2][1] === 0) cond = cond[1]
|
|
1868
|
+
const cmp = isArr(cond) && cond.length === 3 ? LANE_COMPARE[ctx.laneType]?.[cond[0]] : null
|
|
1869
|
+
let mask
|
|
1870
|
+
if (cmp) { const ca = liftExprV(cond[1], ctx); if (ctx.fail) return null; const cb = liftExprV(cond[2], ctx); if (ctx.fail) return null; mask = [cmp, ca, cb] }
|
|
1871
|
+
else if (ctx.laneType === 'i32') { const lc = liftExprV(cond, ctx); if (ctx.fail) return null; mask = ['i32x4.ne', lc, ['i32x4.splat', ['i32.const', 0]]] }
|
|
1872
|
+
else return liftFail(ctx, 'if-store: non-comparison condition')
|
|
1873
|
+
const out = ['__seq__']
|
|
1874
|
+
const liftInter = (arm) => { for (const s of arm.inter) { const l = liftStmt(s, ctx); if (ctx.fail) return false; if (l != null) { if (Array.isArray(l) && l[0] === '__seq__') out.push(...l.slice(1)); else out.push(l) } } return true }
|
|
1875
|
+
if (!liftInter(thenA)) return null
|
|
1876
|
+
if (elseA && !liftInter(elseA)) return null
|
|
1877
|
+
const thenVal = liftExprV(thenA.val, ctx); if (ctx.fail) return null
|
|
1878
|
+
const elseVal = elseA ? liftExprV(elseA.val, ctx) : ['v128.load', thenA.addr] // no else ⇒ keep current value
|
|
1879
|
+
if (ctx.fail) return null
|
|
1880
|
+
const mtmp = `$__mask${ctx.freshIdRef.next++}`
|
|
1881
|
+
ctx.extraLocals.push(['local', mtmp, 'v128'])
|
|
1882
|
+
out.push(['local.set', mtmp, mask], ['v128.store', thenA.addr, ['v128.bitselect', thenVal, elseVal, ['local.get', mtmp]]])
|
|
1883
|
+
return out
|
|
1884
|
+
}
|
|
1885
|
+
|
|
1886
|
+
// Standalone expression-as-statement (e.g. a load that gets dropped) — bail.
|
|
1887
|
+
return liftFail(ctx, `standalone ${op} statement`)
|
|
1888
|
+
}
|
|
1889
|
+
|
|
1890
|
+
/** Lift a value expression into v128 context. */
|
|
1891
|
+
function liftExprV(expr, ctx) {
|
|
1892
|
+
if (!isArr(expr)) return liftFail(ctx, 'non-expression operand')
|
|
1893
|
+
const op = expr[0]
|
|
1894
|
+
const info = LANE_INFO[ctx.laneType]
|
|
1895
|
+
|
|
1896
|
+
// Widening byte-map: a narrow UNSIGNED load feeding i32-lane arithmetic. Load
|
|
1897
|
+
// the 4 elements as a partial vector and zero-extend to i32x4. Only the
|
|
1898
|
+
// widening recognizer sets ctx.widenLoads; tryVectorize ties the lane type to
|
|
1899
|
+
// the load width and never reaches here with a narrow load under i32 lanes.
|
|
1900
|
+
if (ctx.widenLoads && ctx.laneType === 'i32') {
|
|
1901
|
+
if (op === 'i32.load8_u')
|
|
1902
|
+
return ['i32x4.extend_low_i16x8_u', ['i16x8.extend_low_i8x16_u', ['v128.load32_zero', expr[1]]]]
|
|
1903
|
+
if (op === 'i32.load16_u')
|
|
1904
|
+
return ['i32x4.extend_low_i16x8_u', ['v128.load64_zero', expr[1]]]
|
|
1905
|
+
}
|
|
1906
|
+
|
|
1907
|
+
// Widening f32→f64 load: a Float32Array read promoted to f64 (`f64.promote_f32(f32.load …)`,
|
|
1908
|
+
// e.g. schrodinger's f32 potential `V[idx]` inside an f64 stencil). Load the 2 f32 lanes
|
|
1909
|
+
// (load64_zero = 8 bytes = V[idx],V[idx+1]) and promote to f64x2 — the consecutive pair the
|
|
1910
|
+
// two f64 lanes need. Only in f64-lane context; bit-exact (same promote the scalar does).
|
|
1911
|
+
if (op === 'f64.promote_f32' && ctx.laneType === 'f64' && isArr(expr[1]) && expr[1][0] === 'f32.load') {
|
|
1912
|
+
const ld = expr[1]
|
|
1913
|
+
const addr = typeof ld[1] === 'string' && ld[1].startsWith('offset=') ? ['v128.load64_zero', ld[1], ld[2]] : ['v128.load64_zero', ld[1]]
|
|
1914
|
+
return ['f64x2.promote_low_f32x4', addr]
|
|
1915
|
+
}
|
|
1916
|
+
|
|
1917
|
+
// Loads → v128.load (preserving address, including any local.tee).
|
|
1918
|
+
if (LOAD_OPS[op]) {
|
|
1919
|
+
if (LOAD_OPS[op] !== ctx.laneType) return liftFail(ctx, `${op}: load type ≠ lane type ${ctx.laneType}`)
|
|
1920
|
+
// memarg form `(T.load offset=N addr)` — the stencil neighbour `a[i+1]` jz folds
|
|
1921
|
+
// onto `a[i]`'s address tee. `v128.load offset=N` reads the N-byte-shifted vector,
|
|
1922
|
+
// i.e. the (a[i+1], a[i+2]) pair — exactly the δ-shifted lane data. Preserve it.
|
|
1923
|
+
if (typeof expr[1] === 'string' && expr[1].startsWith('offset=')) return ['v128.load', expr[1], expr[2]]
|
|
1924
|
+
return ['v128.load', expr[1]]
|
|
1925
|
+
}
|
|
1926
|
+
|
|
1927
|
+
// Constants → splat.
|
|
1928
|
+
if (op === info.constOp) {
|
|
1929
|
+
return [info.splat, expr]
|
|
1930
|
+
}
|
|
1931
|
+
|
|
1932
|
+
// local.get
|
|
1933
|
+
if (op === 'local.get' && typeof expr[1] === 'string') {
|
|
1934
|
+
const name = expr[1]
|
|
1935
|
+
// Induction variable used AS DATA (ramp-map) → splat to a ramp vector
|
|
1936
|
+
// [i, i+1, … i+LANES-1]. Only set by tryRampMap (i32 lanes); other
|
|
1937
|
+
// recognizers leave ctx.rampVar undefined, so the IV stays address-only.
|
|
1938
|
+
if (name === ctx.rampVar) {
|
|
1939
|
+
// The ramp [i, i+1, i+2, i+3] is materialized once per iteration into
|
|
1940
|
+
// ctx.rampTemp (set at the top of the lifted body); every use reads it.
|
|
1941
|
+
return ['local.get', ctx.rampTemp]
|
|
1942
|
+
}
|
|
1943
|
+
const kind = ctx.localKind.get(name)
|
|
1944
|
+
if (kind === 'lane') {
|
|
1945
|
+
const { laneName } = getOrAllocLanedLocal(name, ctx)
|
|
1946
|
+
return ['local.get', laneName]
|
|
1947
|
+
}
|
|
1948
|
+
if (kind === 'invariant') {
|
|
1949
|
+
return [info.splat, ['local.get', name]]
|
|
1950
|
+
}
|
|
1951
|
+
if (kind === 'addr' || name === ctx.incVar) {
|
|
1952
|
+
return liftFail(ctx, `${name}: address/induction var used as lane data`)
|
|
1953
|
+
}
|
|
1954
|
+
return liftFail(ctx, `${name}: unclassified local in value position`)
|
|
1955
|
+
}
|
|
1956
|
+
|
|
1957
|
+
// Loop-invariant global (e.g. a hoistConstantPool'd const, or any global the
|
|
1958
|
+
// loop never writes) → splat. The recognizer bails when the body contains a
|
|
1959
|
+
// global.set, so every global.get reaching here is invariant across lanes.
|
|
1960
|
+
if (op === 'global.get' && typeof expr[1] === 'string') {
|
|
1961
|
+
return [info.splat, expr]
|
|
1962
|
+
}
|
|
1963
|
+
|
|
1964
|
+
// `f64(invariant i32)` — e.g. `x[i] / N` with N an i32 global/invariant: the convert is a
|
|
1965
|
+
// loop-invariant scalar, so compute once and splat (== scalar-then-splat, bit-exact). Unblocks
|
|
1966
|
+
// pure-f64 maps that scale/divide by an integer count (rfft cepstrum `cep[i] = x[i] / N`).
|
|
1967
|
+
if ((op === 'f64.convert_i32_s' || op === 'f64.convert_i32_u') && expr.length === 2) {
|
|
1968
|
+
const inner = expr[1]
|
|
1969
|
+
const inv = isArr(inner) && (inner[0] === 'global.get' || (inner[0] === 'local.get' && ctx.localKind.get(inner[1]) === 'invariant'))
|
|
1970
|
+
if (inv) return [info.splat, expr]
|
|
1971
|
+
}
|
|
1972
|
+
|
|
1973
|
+
// NaN-canonicalization wrapper (float lanes only; integer lanes never carry
|
|
1974
|
+
// it). Both the flattened `select` form and the un-flattened `block` form
|
|
1975
|
+
// lift to a per-lane v128.bitselect — canonical value in NaN lanes, X
|
|
1976
|
+
// elsewhere — exactly reproducing the scalar canonicalization lane-by-lane.
|
|
1977
|
+
if (ctx.laneType === 'f64' || ctx.laneType === 'f32') {
|
|
1978
|
+
if (op === 'select') {
|
|
1979
|
+
const m = matchCanonSelect(expr, ctx.laneType)
|
|
1980
|
+
if (!m) return liftFail(ctx, 'non-canonical select (not a NaN-canon idiom)')
|
|
1981
|
+
const coreV = liftExprV(m.val, ctx)
|
|
1982
|
+
return ctx.fail ? null : liftCanon(coreV, m.C, ctx, info)
|
|
1983
|
+
}
|
|
1984
|
+
if (op === 'block') {
|
|
1985
|
+
const m = matchCanonBlock(expr, ctx.laneType)
|
|
1986
|
+
if (!m) return liftFail(ctx, 'non-canonical value-block')
|
|
1987
|
+
const coreV = liftExprV(m.core, ctx)
|
|
1988
|
+
return ctx.fail ? null : liftCanon(coreV, m.C, ctx, info)
|
|
1989
|
+
}
|
|
1990
|
+
}
|
|
1991
|
+
|
|
1992
|
+
// Conditional select — jz lowers `cond ? X : Y` to (if (result LT) COND (then X)
|
|
1993
|
+
// (else Y)). Lift to v128.bitselect(X, Y, mask), where mask is COND as an
|
|
1994
|
+
// all-ones/all-zeros lane comparison. Both branches are lane-pure (recursion
|
|
1995
|
+
// forbids stores/sets) and trap-free (no liftable op traps — int div/rem aren't
|
|
1996
|
+
// lane-pure), so speculatively evaluating both is safe; bitselect keeps the
|
|
1997
|
+
// chosen lane. The mask is hoisted to a temp and computed FIRST: bitselect
|
|
1998
|
+
// evaluates X,Y before its 3rd operand, but any address `local.tee` lives in
|
|
1999
|
+
// COND and must run before the branches read it (matching scalar order).
|
|
2000
|
+
if (op === 'if') {
|
|
2001
|
+
if (!isArr(expr[1]) || expr[1][0] !== 'result' || expr[1][1] !== ctx.laneType) return liftFail(ctx, 'conditional without lane-typed result')
|
|
2002
|
+
const thenN = expr[3], elseN = expr[4]
|
|
2003
|
+
if (!isArr(thenN) || thenN[0] !== 'then' || thenN.length !== 2) return liftFail(ctx, 'malformed conditional then-branch')
|
|
2004
|
+
if (!isArr(elseN) || elseN[0] !== 'else' || elseN.length !== 2) return liftFail(ctx, 'malformed conditional else-branch')
|
|
2005
|
+
let cond = expr[2]
|
|
2006
|
+
if (isArr(cond) && cond[0] === 'i32.ne' && isI32Const(cond[2]) && cond[2][1] === 0) cond = cond[1] // strip `!= 0`
|
|
2007
|
+
const cmpSimd = isArr(cond) && cond.length === 3 ? LANE_COMPARE[ctx.laneType]?.[cond[0]] : null
|
|
2008
|
+
if (!cmpSimd) return liftFail(ctx, `${isArr(cond) ? cond[0] : 'condition'}: not a lane-vectorizable comparison`)
|
|
2009
|
+
const ca = liftExprV(cond[1], ctx); if (ctx.fail) return null
|
|
2010
|
+
const cb = liftExprV(cond[2], ctx); if (ctx.fail) return null
|
|
2011
|
+
const x = liftExprV(thenN[1], ctx); if (ctx.fail) return null
|
|
2012
|
+
const y = liftExprV(elseN[1], ctx); if (ctx.fail) return null
|
|
2013
|
+
const mtmp = `$__mask${ctx.freshIdRef.next++}`
|
|
2014
|
+
ctx.extraLocals.push(['local', mtmp, 'v128'])
|
|
2015
|
+
return ['block', ['result', 'v128'],
|
|
2016
|
+
['local.set', mtmp, [cmpSimd, ca, cb]],
|
|
2017
|
+
['v128.bitselect', x, y, ['local.get', mtmp]]]
|
|
2018
|
+
}
|
|
2019
|
+
|
|
2020
|
+
// Lane-pure op?
|
|
2021
|
+
const table = LANE_PURE[ctx.laneType]
|
|
2022
|
+
const entry = table?.get(op)
|
|
2023
|
+
if (entry) {
|
|
2024
|
+
const a = liftExprV(expr[1], ctx)
|
|
2025
|
+
if (ctx.fail) return null
|
|
2026
|
+
if (entry.shamtScalar) {
|
|
2027
|
+
// Second operand stays scalar i32 — must be const or invariant local.
|
|
2028
|
+
const b = expr[2]
|
|
2029
|
+
if (!isI32Const(b) && !(isArr(b) && b[0] === 'local.get' && ctx.localKind.get(b[1]) === 'invariant')) {
|
|
2030
|
+
return liftFail(ctx, `${op}: shift amount not a constant or loop-invariant`)
|
|
2031
|
+
}
|
|
2032
|
+
return [entry.simd, a, b]
|
|
2033
|
+
}
|
|
2034
|
+
if (expr.length === 2) { // unary (neg, abs, sqrt)
|
|
2035
|
+
return [entry.simd, a]
|
|
2036
|
+
}
|
|
2037
|
+
const b = liftExprV(expr[2], ctx)
|
|
2038
|
+
if (ctx.fail) return null
|
|
2039
|
+
return [entry.simd, a, b]
|
|
2040
|
+
}
|
|
2041
|
+
|
|
2042
|
+
return liftFail(ctx, `${op}: no lane-pure SIMD mapping for ${ctx.laneType}`)
|
|
2043
|
+
}
|
|
2044
|
+
|
|
2045
|
+
// ---- Induction-variable strength reduction --------------------------------
|
|
2046
|
+
|
|
2047
|
+
// Match `(i32.add (local.get $base) (i32.shl (local.get $ind) (i32.const K)))` in either
|
|
2048
|
+
// operand order, or `(i32.add (local.get $base) (local.get $ind))` (K=0). Returns
|
|
2049
|
+
// {base, k} — the address of element $ind in array $base, byte stride 1<<k — or null.
|
|
2050
|
+
function matchAffineAddr(node, ind) {
|
|
2051
|
+
if (!isArr(node) || node[0] !== 'i32.add' || node.length !== 3) return null
|
|
2052
|
+
const pair = (baseN, offN) => {
|
|
2053
|
+
if (!isLocalGet(baseN) || baseN[1] === ind) return null
|
|
2054
|
+
if (isLocalGet(offN, ind)) return { base: baseN[1], k: 0 }
|
|
2055
|
+
if (isArr(offN) && offN[0] === 'i32.shl' && offN.length === 3 && isLocalGet(offN[1], ind)) {
|
|
2056
|
+
const k = constNum(offN[2])
|
|
2057
|
+
if (k != null && k >= 0 && k <= 3) return { base: baseN[1], k }
|
|
2058
|
+
}
|
|
2059
|
+
return null
|
|
2060
|
+
}
|
|
2061
|
+
return pair(node[1], node[2]) || pair(node[2], node[1])
|
|
2062
|
+
}
|
|
2063
|
+
|
|
2064
|
+
/**
|
|
2065
|
+
* Strength-reduce induction-variable addressing in an affine loop the vectorizer
|
|
2066
|
+
* couldn't lift (an early `break`, a call, a non-lane body). For each loop-invariant
|
|
2067
|
+
* array `base` and shift `K`, every `base + (i<<K)` in the body is replaced by a strided
|
|
2068
|
+
* pointer `$p`, initialized to `base + (i<<K)` before the loop and bumped by `1<<K` in
|
|
2069
|
+
* lockstep with `i`. Drops the per-iteration shift+add — V8 does NOT strength-reduce this
|
|
2070
|
+
* itself (measured ~6% faster, the additive keep-`i` form used here). Canonical shape only
|
|
2071
|
+
* (single +1 IV, bottom increment, br_if exit). Bails if the body writes `i` or any `base`
|
|
2072
|
+
* (not invariant), or branches to the loop label (which would skip the pointer bump — a
|
|
2073
|
+
* `br_if` to the *block* label, i.e. an early break, is fine: the loop is exiting). Runs
|
|
2074
|
+
* only where the vectorizer runs (speed levels), so it never grows the size-tuned build.
|
|
2075
|
+
*/
|
|
2076
|
+
function tryStrengthReduceIV(bl, fnLocals, freshIdRef) {
|
|
2077
|
+
// Bound-shape-agnostic: strength-reduces address arithmetic, not the loop bound,
|
|
2078
|
+
// so it accepts any scaffold (no local-or-const bound check). No preamble (was
|
|
2079
|
+
// allowPreamble:false).
|
|
2080
|
+
if (!bl || bl.preamble.length) return null
|
|
2081
|
+
const { loopNode, loopLabel, incIdx, incVar } = bl
|
|
2082
|
+
|
|
2083
|
+
// Scan the body (stmts between the exit br_if and the bottom increment): collect
|
|
2084
|
+
// affine-address sites, track every written local, and bail on a loop-label branch.
|
|
2085
|
+
const sites = [] // { parent, idx, base, k }
|
|
2086
|
+
const written = new Set()
|
|
2087
|
+
let bail = false
|
|
2088
|
+
const scan = (node, parent, pi) => {
|
|
2089
|
+
if (bail || !isArr(node)) return
|
|
2090
|
+
const op = node[0]
|
|
2091
|
+
if ((op === 'br' || op === 'br_if') && node[1] === loopLabel) { bail = true; return }
|
|
2092
|
+
if ((op === 'local.set' || op === 'local.tee') && typeof node[1] === 'string') written.add(node[1])
|
|
2093
|
+
const m = matchAffineAddr(node, incVar)
|
|
2094
|
+
if (m) sites.push({ parent, idx: pi, base: m.base, k: m.k })
|
|
2095
|
+
for (let i = 1; i < node.length; i++) scan(node[i], node, i)
|
|
2096
|
+
}
|
|
2097
|
+
for (let i = 3; i < incIdx; i++) scan(loopNode[i], loopNode, i)
|
|
2098
|
+
if (bail || !sites.length || written.has(incVar)) return null
|
|
2099
|
+
|
|
2100
|
+
// Group by (base, k); keep only loop-invariant i32 bases.
|
|
2101
|
+
const groups = new Map() // `base|k` → { base, k, sites }
|
|
2102
|
+
for (const s of sites) {
|
|
2103
|
+
if (written.has(s.base) || fnLocals.get(s.base) !== 'i32') continue
|
|
2104
|
+
const key = s.base + '|' + s.k
|
|
2105
|
+
let g = groups.get(key)
|
|
2106
|
+
if (!g) groups.set(key, g = { base: s.base, k: s.k, sites: [] })
|
|
2107
|
+
g.sites.push(s)
|
|
2108
|
+
}
|
|
2109
|
+
if (!groups.size) return null
|
|
2110
|
+
|
|
2111
|
+
// One strided pointer per group: init before the block, bump after the i increment,
|
|
2112
|
+
// every matched address → (local.get $p).
|
|
2113
|
+
const id = freshIdRef.next++
|
|
2114
|
+
const preInits = [], bumps = [], newLocalDecls = []
|
|
2115
|
+
let gi = 0
|
|
2116
|
+
for (const g of groups.values()) {
|
|
2117
|
+
const p = `$__iv${id}_${gi++}`
|
|
2118
|
+
newLocalDecls.push(['local', p, 'i32'])
|
|
2119
|
+
const off = g.k === 0 ? ['local.get', incVar] : ['i32.shl', ['local.get', incVar], ['i32.const', g.k]]
|
|
2120
|
+
preInits.push(['local.set', p, ['i32.add', ['local.get', g.base], off]])
|
|
2121
|
+
bumps.push(['local.set', p, ['i32.add', ['local.get', p], ['i32.const', 1 << g.k]]])
|
|
2122
|
+
for (const s of g.sites) s.parent[s.idx] = ['local.get', p]
|
|
2123
|
+
}
|
|
2124
|
+
loopNode.splice(incIdx + 1, 0, ...bumps) // after the induction increment, before the br
|
|
2125
|
+
return { wrapper: ['block', ...preInits, bl.blockNode], newLocalDecls }
|
|
2126
|
+
}
|
|
2127
|
+
|
|
2128
|
+
// ---- memory.copy / memory.fill loop idioms ---------------------------------
|
|
2129
|
+
|
|
2130
|
+
// Same-width store←load pairs (byte-window moves) and their element stride.
|
|
2131
|
+
// Sign-variant narrow loads are interchangeable for a MOVE: load8_u/load8_s
|
|
2132
|
+
// then store8 write the same byte back.
|
|
2133
|
+
const MEMOP_STORES = {
|
|
2134
|
+
'f64.store': { k: 3, loads: new Set(['f64.load']) },
|
|
2135
|
+
'i64.store': { k: 3, loads: new Set(['i64.load']) },
|
|
2136
|
+
'f32.store': { k: 2, loads: new Set(['f32.load']) },
|
|
2137
|
+
'i32.store': { k: 2, loads: new Set(['i32.load']) },
|
|
2138
|
+
'i32.store16': { k: 1, loads: new Set(['i32.load16_u', 'i32.load16_s']) },
|
|
2139
|
+
'i32.store8': { k: 0, loads: new Set(['i32.load8_u', 'i32.load8_s']) },
|
|
2140
|
+
}
|
|
2141
|
+
|
|
2142
|
+
/**
|
|
2143
|
+
* Replace whole copy/fill loops with the engine's bulk-memory ops:
|
|
2144
|
+
*
|
|
2145
|
+
* for (i < N) a[i] = b[i] → memory.copy (overlap-guarded)
|
|
2146
|
+
* for (i < N) a[i] = 0 → memory.fill 0 (any element width)
|
|
2147
|
+
* for (i < N) u8[i] = C → memory.fill C (byte stores only)
|
|
2148
|
+
*
|
|
2149
|
+
* V8 lowers memory.copy/fill to memmove/memset — typically several times the
|
|
2150
|
+
* throughput of even a SIMD lane loop, and a handful of bytes instead of one.
|
|
2151
|
+
* Runs BEFORE the lane vectorizer so these loops never pay the lift.
|
|
2152
|
+
*
|
|
2153
|
+
* Exactness:
|
|
2154
|
+
* - COPY moves the same byte window the scalar loop wrote (same-width
|
|
2155
|
+
* store←load pairs only — see MEMOP_STORES; f64 load/store round-trips are
|
|
2156
|
+
* bit-exact per spec, so NaN payloads survive). memory.copy is memmove
|
|
2157
|
+
* (as-if-buffered); the FORWARD loop differs from that exactly when the
|
|
2158
|
+
* destination starts strictly inside the source window (dst reads bytes an
|
|
2159
|
+
* earlier iteration already overwrote), so that case keeps the original
|
|
2160
|
+
* loop behind a two-compare runtime guard — every other layout (disjoint,
|
|
2161
|
+
* dst ≤ src, same array) is bit-identical.
|
|
2162
|
+
* - FILL with 0 is width-agnostic (all-zero bytes; −0.0 is excluded — its
|
|
2163
|
+
* sign bit is not zero). Non-zero fills only for byte stores with a
|
|
2164
|
+
* loop-invariant constant ∈ [0,255].
|
|
2165
|
+
* - The induction variable ends at `bound` exactly as the loop would leave
|
|
2166
|
+
* it; a zero-trip range (i ≥ bound) leaves everything untouched.
|
|
2167
|
+
* - In-bounds for the same reason the lane vectorizer's wide loads are: the
|
|
2168
|
+
* moved window is exactly the byte range the scalar iterations touch.
|
|
2169
|
+
*/
|
|
2170
|
+
function tryMemCopyFill(bl, fnLocals, freshIdRef) {
|
|
2171
|
+
if (!bl || bl.preamble.length) return null // no-preamble policy (was allowPreamble:false)
|
|
2172
|
+
const { loopNode, incVar, bound } = bl
|
|
2173
|
+
// Shape: [loop, $l, boundExit, (set,)? store, inc, br] — 1- or 2-statement body.
|
|
2174
|
+
if (loopNode.length !== 6 && loopNode.length !== 7) return null
|
|
2175
|
+
// Bound: const or an invariant local that is not the IV itself.
|
|
2176
|
+
if (!(isI32Const(bound) || (isArr(bound) && bound[0] === 'local.get' && typeof bound[1] === 'string' && bound[1] !== incVar))) return null
|
|
2177
|
+
|
|
2178
|
+
// Body shapes (emit produces the two-statement form with a temp + shared
|
|
2179
|
+
// offset tee; fold/propagate sometimes collapse it to the bare store):
|
|
2180
|
+
// 1-stmt: (T.store DADDR CONST | (T.load SADDR))
|
|
2181
|
+
// 2-stmt: (local.set $t (T.load SADDR)) (T.store DADDR (local.get $t))
|
|
2182
|
+
const addrLocals = new Map(), offsetTees = new Map()
|
|
2183
|
+
const laneAddr = (a) => {
|
|
2184
|
+
const m = matchLaneAddr(a, incVar, addrLocals, offsetTees)
|
|
2185
|
+
if (!m || m.viaLocal) return null
|
|
2186
|
+
if (m.offsetTeeName) offsetTees.set(m.offsetTeeName, m.strideLog2)
|
|
2187
|
+
if (!(isArr(m.base) && m.base[0] === 'local.get' && typeof m.base[1] === 'string' && m.base[1] !== incVar)) return null
|
|
2188
|
+
if (fnLocals.get(m.base[1]) !== 'i32') return null
|
|
2189
|
+
return m
|
|
2190
|
+
}
|
|
2191
|
+
let storeStmt, valNode, tempName = null
|
|
2192
|
+
if (loopNode.length === 6) {
|
|
2193
|
+
storeStmt = loopNode[3]
|
|
2194
|
+
if (!isArr(storeStmt) || storeStmt.length !== 3) return null
|
|
2195
|
+
valNode = storeStmt[2]
|
|
2196
|
+
} else {
|
|
2197
|
+
const s1 = loopNode[3]
|
|
2198
|
+
storeStmt = loopNode[4]
|
|
2199
|
+
if (!isArr(s1) || s1[0] !== 'local.set' || s1.length !== 3 || typeof s1[1] !== 'string') return null
|
|
2200
|
+
if (!isArr(storeStmt) || storeStmt.length !== 3) return null
|
|
2201
|
+
if (!(isArr(storeStmt[2]) && storeStmt[2][0] === 'local.get' && storeStmt[2][1] === s1[1])) return null
|
|
2202
|
+
tempName = s1[1]
|
|
2203
|
+
if (tempName === incVar) return null
|
|
2204
|
+
valNode = s1[2]
|
|
2205
|
+
}
|
|
2206
|
+
const entry = MEMOP_STORES[storeStmt[0]]
|
|
2207
|
+
if (!entry) return null
|
|
2208
|
+
|
|
2209
|
+
// Classify VALUE first (the load side carries the offset tee in the 2-stmt form).
|
|
2210
|
+
let fillByte = null, srcM = null
|
|
2211
|
+
if (isArr(valNode) && valNode.length === 2 && (valNode[0] === 'i32.const' || valNode[0] === 'i64.const' || valNode[0] === 'f64.const' || valNode[0] === 'f32.const') && typeof valNode[1] === 'number') {
|
|
2212
|
+
if (valNode[1] === 0 && !Object.is(valNode[1], -0)) fillByte = 0
|
|
2213
|
+
else if (storeStmt[0] === 'i32.store8' && valNode[0] === 'i32.const' && Number.isInteger(valNode[1]) && valNode[1] >= 0 && valNode[1] <= 255) fillByte = valNode[1]
|
|
2214
|
+
else return null
|
|
2215
|
+
} else if (isArr(valNode) && valNode.length === 2 && entry.loads.has(valNode[0])) {
|
|
2216
|
+
srcM = laneAddr(valNode[1])
|
|
2217
|
+
if (!srcM || srcM.strideLog2 !== entry.k) return null
|
|
2218
|
+
} else return null
|
|
2219
|
+
|
|
2220
|
+
const dstM = laneAddr(storeStmt[1])
|
|
2221
|
+
if (!dstM || dstM.strideLog2 !== entry.k) return null
|
|
2222
|
+
// Soundness for any shared offset tee resolved via `(local.get $T)` (see tryVectorize).
|
|
2223
|
+
for (const [name, k] of offsetTees) {
|
|
2224
|
+
if (_offsetLocalStride([loopNode[3], storeStmt], name, incVar) !== k) return null
|
|
2225
|
+
}
|
|
2226
|
+
|
|
2227
|
+
const id = freshIdRef.next++
|
|
2228
|
+
const lenB = `$__mc${id}_len`, dstA = `$__mc${id}_dst`
|
|
2229
|
+
const newLocalDecls = [['local', lenB, 'i32'], ['local', dstA, 'i32']]
|
|
2230
|
+
const shl = (x) => entry.k === 0 ? x : ['i32.shl', x, ['i32.const', entry.k]]
|
|
2231
|
+
const boundC = () => cloneNode(bound)
|
|
2232
|
+
const setup = [
|
|
2233
|
+
['local.set', lenB, shl(['i32.sub', boundC(), ['local.get', incVar]])],
|
|
2234
|
+
['local.set', dstA, ['i32.add', ['local.get', dstM.base[1]], shl(['local.get', incVar])]],
|
|
2235
|
+
]
|
|
2236
|
+
// Exact loop-exit state: i ends at bound; a matched offset tee holds the
|
|
2237
|
+
// LAST iteration's offset; the value temp holds the last element moved.
|
|
2238
|
+
const finish = [['local.set', incVar, boundC()]]
|
|
2239
|
+
const lastOff = () => shl(['i32.sub', boundC(), ['i32.const', 1]])
|
|
2240
|
+
for (const name of offsetTees.keys()) finish.push(['local.set', name, lastOff()])
|
|
2241
|
+
if (tempName != null && srcM) finish.push(['local.set', tempName,
|
|
2242
|
+
[valNode[0], ['i32.add', ['local.get', srcM.base[1]], lastOff()]]])
|
|
2243
|
+
if (tempName != null && srcM == null) finish.push(['local.set', tempName, cloneNode(valNode)])
|
|
2244
|
+
|
|
2245
|
+
let action
|
|
2246
|
+
if (srcM == null) {
|
|
2247
|
+
action = [['memory.fill', ['local.get', dstA], ['i32.const', fillByte], ['local.get', lenB]], ...finish]
|
|
2248
|
+
} else {
|
|
2249
|
+
const srcA = `$__mc${id}_src`
|
|
2250
|
+
newLocalDecls.push(['local', srcA, 'i32'])
|
|
2251
|
+
setup.push(['local.set', srcA, ['i32.add', ['local.get', srcM.base[1]], shl(['local.get', incVar])]])
|
|
2252
|
+
// Forward-loop ≡ memmove unless dst starts strictly inside (src, src+len).
|
|
2253
|
+
action = [['if',
|
|
2254
|
+
['i32.or',
|
|
2255
|
+
['i32.le_u', ['local.get', dstA], ['local.get', srcA]],
|
|
2256
|
+
['i32.ge_u', ['local.get', dstA], ['i32.add', ['local.get', srcA], ['local.get', lenB]]]],
|
|
2257
|
+
['then',
|
|
2258
|
+
['memory.copy', ['local.get', dstA], ['local.get', srcA], ['local.get', lenB]],
|
|
2259
|
+
...finish],
|
|
2260
|
+
['else', bl.blockNode]]]
|
|
2261
|
+
}
|
|
2262
|
+
|
|
2263
|
+
const wrapper = ['block',
|
|
2264
|
+
['if', ['i32.lt_s', ['local.get', incVar], boundC()],
|
|
2265
|
+
['then', ...setup, ...action]]]
|
|
2266
|
+
return { wrapper, newLocalDecls }
|
|
2267
|
+
}
|
|
2268
|
+
|
|
2269
|
+
// ---- Byte-scan (memchr) vectorization -------------------------------------
|
|
2270
|
+
|
|
2271
|
+
// Match a single-byte compare against a constant or a loop-invariant target:
|
|
2272
|
+
// (f64.eq|ne (f64.convert_i32_u|s (i32.load8_u|s (base + i))) TARGET) [value-model form]
|
|
2273
|
+
// (i32.eq|ne (i32.load8_u|s (base + i)) TARGET) [folded i32 form]
|
|
2274
|
+
// TARGET is a const byte or a `local.get` (the `memchr(buf, delim)` runtime case).
|
|
2275
|
+
// Returns { base, eq, isF64, c, targetLocal } — exactly one of c (∈[0,255]) / targetLocal set.
|
|
2276
|
+
function matchByteCompare(node, ind) {
|
|
2277
|
+
if (!isArr(node) || node.length !== 3) return null
|
|
2278
|
+
let eq
|
|
2279
|
+
if (node[0] === 'f64.eq' || node[0] === 'i32.eq') eq = true
|
|
2280
|
+
else if (node[0] === 'f64.ne' || node[0] === 'i32.ne') eq = false
|
|
2281
|
+
else return null
|
|
2282
|
+
const isF64 = node[0][0] === 'f'
|
|
2283
|
+
const constOf = (x) => isF64
|
|
2284
|
+
? (isArr(x) && x[0] === 'f64.const' && typeof x[1] === 'number' && Number.isInteger(x[1]) ? x[1] : null)
|
|
2285
|
+
: constNum(x)
|
|
2286
|
+
// Identify which operand is the byte load and which is the target.
|
|
2287
|
+
const isLoadSide = (x) => {
|
|
2288
|
+
let l = x
|
|
2289
|
+
if (isF64) { if (!(isArr(l) && (l[0] === 'f64.convert_i32_u' || l[0] === 'f64.convert_i32_s') && l.length === 2)) return null; l = l[1] }
|
|
2290
|
+
if (!(isArr(l) && (l[0] === 'i32.load8_u' || l[0] === 'i32.load8_s') && l.length === 2)) return null
|
|
2291
|
+
const m = matchAffineAddr(l[1], ind)
|
|
2292
|
+
return m && m.k === 0 ? m.base : null // byte stride only
|
|
2293
|
+
}
|
|
2294
|
+
let base = isLoadSide(node[1]), target = node[2]
|
|
2295
|
+
if (base == null) { base = isLoadSide(node[2]); target = node[1] }
|
|
2296
|
+
if (base == null) return null
|
|
2297
|
+
const c = constOf(target)
|
|
2298
|
+
if (c != null) return c >= 0 && c <= 255 ? { base, eq, isF64, c } : null
|
|
2299
|
+
// Runtime target: a loop-invariant local (the minimal scan body writes only `i`).
|
|
2300
|
+
if (isArr(target) && target[0] === 'local.get' && typeof target[1] === 'string' && target[1] !== ind)
|
|
2301
|
+
return { base, eq, isF64, targetLocal: target[1] }
|
|
2302
|
+
return null
|
|
2303
|
+
}
|
|
2304
|
+
|
|
2305
|
+
/**
|
|
2306
|
+
* SIMD byte scan — vectorize a memchr-shaped loop the engine runs one byte at a time.
|
|
2307
|
+
* Recognizes the pure scan
|
|
2308
|
+
* (block $b (loop $l (br_if $b (eqz (i<bound))) (br_if $b (buf[i] ==/!= C)) (i := i+1) (br $l)))
|
|
2309
|
+
* — "find the first index where buf[i] (Uint8/Int8Array) ==/!= a constant byte" — and
|
|
2310
|
+
* rewrites it to scan 16 bytes per step with `i8x16.eq` + `i8x16.bitmask`, locating the
|
|
2311
|
+
* exact first match via `i32.ctz`, with the original loop kept as the <16-byte tail.
|
|
2312
|
+
* Measured ~8× over the scalar scan on V8 (which doesn't auto-vectorize it). Fails closed:
|
|
2313
|
+
* any deviation from the exact shape leaves the scalar loop. The 16-wide `v128.load` is
|
|
2314
|
+
* in-bounds because it only fires while `i+16 <= bound` and `bound` bounds the scalar
|
|
2315
|
+
* reads too. (charCodeAt over a jz string is out of scope — it lowers to per-char
|
|
2316
|
+
* bounds/SSO/heap/decode branches, not a flat byte load.)
|
|
2317
|
+
*/
|
|
2318
|
+
function tryByteScan(bl, fnLocals, freshIdRef) {
|
|
2319
|
+
if (!bl || bl.preamble.length) return null
|
|
2320
|
+
const { blockLabel, loopNode, incVar, exitInfo } = bl
|
|
2321
|
+
// Exact shape: [loop, $l, boundExit, matchExit, inc, br] — nothing else. The
|
|
2322
|
+
// scaffold already verified boundExit/inc/back-branch; only the extra match-exit
|
|
2323
|
+
// br_if at loopNode[3] and the strict length remain.
|
|
2324
|
+
if (loopNode.length !== 6) return null
|
|
2325
|
+
const matchExit = loopNode[3]
|
|
2326
|
+
if (!(isArr(matchExit) && matchExit[0] === 'br_if' && matchExit[1] === blockLabel && matchExit.length === 3)) return null
|
|
2327
|
+
const bc = matchByteCompare(matchExit[2], incVar)
|
|
2328
|
+
if (!bc) return null
|
|
2329
|
+
if (fnLocals.get(bc.base) !== 'i32' || bc.base === incVar) return null
|
|
2330
|
+
|
|
2331
|
+
const id = freshIdRef.next++
|
|
2332
|
+
const sd = `$__bscan_brk${id}`, sl = `$__bscan_loop${id}`, mask = `$__bscan_m${id}`
|
|
2333
|
+
const baseGet = ['local.get', bc.base]
|
|
2334
|
+
const iGet = ['local.get', incVar]
|
|
2335
|
+
const bound = exitInfo.bound
|
|
2336
|
+
const newLocalDecls = [['local', mask, 'i32']]
|
|
2337
|
+
|
|
2338
|
+
// The byte to splat across 16 lanes, plus a runtime guard. A constant needs none.
|
|
2339
|
+
// A runtime `delim` (f64-boxed) is only a valid SIMD target when it's an integer in
|
|
2340
|
+
// [0,255]; outside that, NO byte (0–255) equals it, so the scalar tail — which we keep —
|
|
2341
|
+
// reproduces the exact result. The guard makes that branch explicit; cb caches the byte.
|
|
2342
|
+
let splat, guard = null, cbInit = null
|
|
2343
|
+
if (bc.c != null) {
|
|
2344
|
+
splat = ['i32.const', bc.c]
|
|
2345
|
+
} else {
|
|
2346
|
+
const cb = `$__bscan_c${id}`
|
|
2347
|
+
newLocalDecls.push(['local', cb, 'i32'])
|
|
2348
|
+
const tGet = ['local.get', bc.targetLocal]
|
|
2349
|
+
const inRange = ['i32.and', ['i32.ge_s', ['local.get', cb], ['i32.const', 0]], ['i32.le_s', ['local.get', cb], ['i32.const', 255]]]
|
|
2350
|
+
if (bc.isF64) {
|
|
2351
|
+
// cb = (i32)delim; valid iff it round-trips (delim is an integer) and is in [0,255].
|
|
2352
|
+
cbInit = ['local.set', cb, ['i32.trunc_sat_f64_s', cloneNode(tGet)]]
|
|
2353
|
+
guard = ['i32.and', ['f64.eq', ['f64.convert_i32_s', ['local.get', cb]], tGet], inRange]
|
|
2354
|
+
} else {
|
|
2355
|
+
// i32 delim: valid iff already in [0,255] (splat takes the low byte regardless).
|
|
2356
|
+
cbInit = ['local.set', cb, tGet]
|
|
2357
|
+
guard = inRange
|
|
2358
|
+
}
|
|
2359
|
+
splat = ['local.get', cb]
|
|
2360
|
+
}
|
|
2361
|
+
|
|
2362
|
+
// bitmask of the 16-lane eq; for `!=` flip the low 16 bits so ctz finds the first non-match.
|
|
2363
|
+
const eqMask = ['i8x16.bitmask', ['i8x16.eq', ['v128.load', ['i32.add', baseGet, iGet]], ['i8x16.splat', splat]]]
|
|
2364
|
+
const scanMask = bc.eq ? eqMask : ['i32.xor', eqMask, ['i32.const', 0xffff]]
|
|
2365
|
+
const simdBlock = ['block', sd,
|
|
2366
|
+
['loop', sl,
|
|
2367
|
+
// Stop before a 16-wide load would pass `bound`; the scalar tail mops up the rest.
|
|
2368
|
+
['br_if', sd, ['i32.gt_s', ['i32.add', iGet, ['i32.const', 16]], cloneNode(bound)]],
|
|
2369
|
+
['local.set', mask, scanMask],
|
|
2370
|
+
['if', ['local.get', mask],
|
|
2371
|
+
['then',
|
|
2372
|
+
['local.set', incVar, ['i32.add', iGet, ['i32.ctz', ['local.get', mask]]]],
|
|
2373
|
+
['br', blockLabel]]],
|
|
2374
|
+
['local.set', incVar, ['i32.add', iGet, ['i32.const', 16]]],
|
|
2375
|
+
['br', sl]
|
|
2376
|
+
]
|
|
2377
|
+
]
|
|
2378
|
+
// Const target: SIMD then scalar tail. Runtime target: cache cb, guard the SIMD, tail.
|
|
2379
|
+
const pre = guard ? [cbInit, ['if', guard, ['then', simdBlock]]] : [simdBlock]
|
|
2380
|
+
return {
|
|
2381
|
+
wrapper: ['block', blockLabel, ...pre, loopNode],
|
|
2382
|
+
newLocalDecls,
|
|
2383
|
+
}
|
|
2384
|
+
}
|
|
2385
|
+
|
|
2386
|
+
const cloneNode = (n) => Array.isArray(n) ? n.map(cloneNode) : n
|
|
2387
|
+
|
|
2388
|
+
// ---- Channel-reduction recognizer (RGBA box-filter accumulation) -----------
|
|
2389
|
+
//
|
|
2390
|
+
// The image-convolution hot shape: 4 adjacent-byte accumulators summed over a
|
|
2391
|
+
// window, then divided + stored — `for k: sr+=src[p]; sg+=src[p+1]; …` (blur,
|
|
2392
|
+
// box filters, separable convolutions). The 4 channels are 4 i32x4 lanes, the
|
|
2393
|
+
// 4-byte load is one widening v128.load32_zero. We vectorize ONLY the inner
|
|
2394
|
+
// accumulation (integer add → associative/exact → bit-identical) and extract the
|
|
2395
|
+
// lane sums back to the scalars; the edge-clamp address math and the per-pixel
|
|
2396
|
+
// divide+store stay scalar, untouched. So no float-reduction reordering, no
|
|
2397
|
+
// lane-juggled divide — just the dense inner loop lifted, safely.
|
|
2398
|
+
//
|
|
2399
|
+
// Operates on the OUTER pixel-loop block: its body is [exit, 4×(acc=0), …setup,
|
|
2400
|
+
// (block (loop INNER)), …uses-of-acc, inc, br]. INNER's body accumulates the 4
|
|
2401
|
+
// channels off a shared base address.
|
|
2402
|
+
|
|
2403
|
+
// Match `(local.set $ACC (i32.add (local.get $ACC) (i32.load8_u ADDR)))` where
|
|
2404
|
+
// ADDR is `(local.get $base)` or `(local.tee $base EXPR)` at byte offset `off`.
|
|
2405
|
+
// Returns { acc, base, off, teeExpr? } or null.
|
|
2406
|
+
function matchChannelAccum(stmt, off) {
|
|
2407
|
+
if (!isArr(stmt) || stmt[0] !== 'local.set' || stmt.length !== 3) return null
|
|
2408
|
+
const acc = stmt[1]
|
|
2409
|
+
const add = stmt[2]
|
|
2410
|
+
if (!isArr(add) || add[0] !== 'i32.add' || add.length !== 3) return null
|
|
2411
|
+
if (!isLocalGet(add[1], acc)) return null
|
|
2412
|
+
const load = add[2]
|
|
2413
|
+
if (!isArr(load) || load[0] !== 'i32.load8_u') return null
|
|
2414
|
+
// memarg offset: bare load → off 0; `i32.load8_u offset=N addr` → off N.
|
|
2415
|
+
let addr = load[1], loadOff = 0
|
|
2416
|
+
if (typeof load[1] === 'string' && /^offset=/.test(load[1])) { loadOff = +load[1].slice(7); addr = load[2] }
|
|
2417
|
+
if (loadOff !== off) return null
|
|
2418
|
+
if (isArr(addr) && addr[0] === 'local.tee' && addr.length === 3) return { acc, base: addr[1], off, teeExpr: addr[2] }
|
|
2419
|
+
if (isLocalGet(addr)) return { acc, base: addr[1], off }
|
|
2420
|
+
return null
|
|
2421
|
+
}
|
|
2422
|
+
|
|
2423
|
+
// Recognize, inside the inner loop body, four consecutive channel accumulations
|
|
2424
|
+
// `acc0+=base[0]; acc1+=base[1]; acc2+=base[2]; acc3+=base[3]` over ONE shared
|
|
2425
|
+
// base. Returns { accs:[a0..a3], baseLocal, teeExpr, idx } (idx = position of the
|
|
2426
|
+
// first accum stmt in `body`) or null.
|
|
2427
|
+
function matchChannelGroup(body) {
|
|
2428
|
+
for (let i = 0; i + 3 < body.length; i++) {
|
|
2429
|
+
const m0 = matchChannelAccum(body[i], 0)
|
|
2430
|
+
if (!m0 || m0.teeExpr == null) continue // first load carries the address tee
|
|
2431
|
+
const ms = [m0]
|
|
2432
|
+
let ok = true
|
|
2433
|
+
for (let c = 1; c < 4; c++) {
|
|
2434
|
+
const m = matchChannelAccum(body[i + c], c)
|
|
2435
|
+
if (!m || m.base !== m0.base || m.teeExpr != null) { ok = false; break }
|
|
2436
|
+
ms.push(m)
|
|
2437
|
+
}
|
|
2438
|
+
if (!ok) continue
|
|
2439
|
+
const accs = ms.map(m => m.acc)
|
|
2440
|
+
if (new Set(accs).size !== 4) continue // four distinct accumulators
|
|
2441
|
+
return { accs, baseLocal: m0.base, teeExpr: m0.teeExpr, idx: i }
|
|
2442
|
+
}
|
|
2443
|
+
return null
|
|
2444
|
+
}
|
|
2445
|
+
|
|
2446
|
+
// ---- Byte-map recognizer (ramp + widening loads) ---------------------------
|
|
2447
|
+
//
|
|
2448
|
+
// Vectorize `for (i = 0; i < N; i++) out[i] = NARROW(f_i32(…))` where the i32
|
|
2449
|
+
// value is built from the induction variable used AS DATA (an i32x4 RAMP
|
|
2450
|
+
// `[i, i+1, i+2, i+3]`) and/or WIDENED narrow loads (`u8[i]` zero-extended to
|
|
2451
|
+
// i32x4). tryVectorize can't express either: it derives the lane type from an
|
|
2452
|
+
// input load and ties the compute width to it, so a byte LUT-free map whose
|
|
2453
|
+
// arithmetic overflows a byte (mul, shifts) — or that has no load at all —
|
|
2454
|
+
// falls through to here, where everything lifts to i32x4 and narrow-stores.
|
|
2455
|
+
// • pure ramp (no loads, i8 store) → 16-wide pack (bytebeat)
|
|
2456
|
+
// • widening u8 loads → 4-wide (alpha blend, brightness/color/threshold)
|
|
2457
|
+
//
|
|
2458
|
+
// Matches the post-strength-reduction shape (the IV strength-reducer runs
|
|
2459
|
+
// before this pass): the loop carries the logical IV (`i`, in the exit test,
|
|
2460
|
+
// += 1) plus one or more strided output pointers (`p += C`). Each increment is
|
|
2461
|
+
// scaled by LANES; the store narrows i32x4 back to the element width.
|
|
2462
|
+
//
|
|
2463
|
+
// Narrowing is truncation-exact for ANY i32 value (matching scalar store8/16):
|
|
2464
|
+
// `i8x16.shuffle` selects the low byte of each lane — never saturates — so no
|
|
2465
|
+
// value-range assumption is needed.
|
|
2466
|
+
function tryRampMap(blockNode, fnLocals, freshIdRef) {
|
|
2467
|
+
if (!isArr(blockNode) || blockNode[0] !== 'block') return null
|
|
2468
|
+
// Envelope: (block $brk (loop $L …)) — identical to tryVectorize.
|
|
2469
|
+
let blockLabel = null, loopNode = null
|
|
2470
|
+
for (let i = 1; i < blockNode.length; i++) {
|
|
2471
|
+
const c = blockNode[i]
|
|
2472
|
+
if (typeof c === 'string' && c.startsWith('$') && blockLabel == null && i === 1) { blockLabel = c; continue }
|
|
2473
|
+
if (isArr(c) && c[0] === 'loop') { if (loopNode) return null; loopNode = c }
|
|
2474
|
+
else if (isArr(c)) return null
|
|
2475
|
+
}
|
|
2476
|
+
if (!loopNode || !blockLabel) return null
|
|
2477
|
+
const loopLabel = typeof loopNode[1] === 'string' && loopNode[1].startsWith('$') ? loopNode[1] : null
|
|
2478
|
+
if (!loopLabel) return null
|
|
2479
|
+
|
|
2480
|
+
// End = (br $L); preceded by a run of trailing `x += C` increments.
|
|
2481
|
+
const endIdx = loopNode.length - 1
|
|
2482
|
+
if (!(isArr(loopNode[endIdx]) && loopNode[endIdx][0] === 'br' && loopNode[endIdx][1] === loopLabel)) return null
|
|
2483
|
+
const increments = []
|
|
2484
|
+
let bodyEnd = endIdx - 1
|
|
2485
|
+
while (bodyEnd >= 2) {
|
|
2486
|
+
const inc = matchIncN(loopNode[bodyEnd])
|
|
2487
|
+
if (!inc) break
|
|
2488
|
+
increments.unshift(inc)
|
|
2489
|
+
bodyEnd--
|
|
2490
|
+
}
|
|
2491
|
+
if (!increments.length) return null
|
|
2492
|
+
|
|
2493
|
+
// First stmt = exit guard → logical IV + bound. IV must advance by exactly 1.
|
|
2494
|
+
const exitInfo = matchExitBrIf(loopNode[2], blockLabel)
|
|
2495
|
+
if (!exitInfo) return null
|
|
2496
|
+
const ivName = exitInfo.ind
|
|
2497
|
+
const ivInc = increments.find(x => x.name === ivName)
|
|
2498
|
+
if (!ivInc || ivInc.c !== 1) return null
|
|
2499
|
+
let bound = exitInfo.bound, boundLocal = null
|
|
2500
|
+
if (isArr(bound) && bound[0] === 'local.get' && typeof bound[1] === 'string') boundLocal = bound[1]
|
|
2501
|
+
else if (!isI32Const(bound)) return null
|
|
2502
|
+
|
|
2503
|
+
const body = []
|
|
2504
|
+
for (let i = 3; i <= bodyEnd; i++) body.push(loopNode[i])
|
|
2505
|
+
if (!body.length) return null
|
|
2506
|
+
if (body.some(hasGlobalSet)) return null
|
|
2507
|
+
|
|
2508
|
+
// Find exactly one store. Its address is the inline lane address
|
|
2509
|
+
// `base + (i << K)` — the IV strength-reducer runs AFTER this pass, so the
|
|
2510
|
+
// pointer is still expressed in terms of the IV. We keep the address verbatim
|
|
2511
|
+
// (scalar i32) and advance the IV by LANES, so `base + (i<<K)` lands on the
|
|
2512
|
+
// next group's first element each SIMD step — for any element width.
|
|
2513
|
+
let storeStmt = null, storeIdx = -1
|
|
2514
|
+
for (let i = 0; i < body.length; i++) {
|
|
2515
|
+
const s = body[i]
|
|
2516
|
+
if (isArr(s) && STORE_OPS[s[0]]) {
|
|
2517
|
+
if (storeStmt) return null
|
|
2518
|
+
storeStmt = s; storeIdx = i
|
|
2519
|
+
}
|
|
2520
|
+
}
|
|
2521
|
+
if (!storeStmt) return null
|
|
2522
|
+
const storeOp = storeStmt[0]
|
|
2523
|
+
if (storeStmt.length !== 3) return null
|
|
2524
|
+
const elemLog2 = { 'i32.store8': 0, 'i32.store': 2 }[storeOp]
|
|
2525
|
+
if (elemLog2 === undefined) return null
|
|
2526
|
+
if (increments.some(x => x.name !== ivName)) return null
|
|
2527
|
+
|
|
2528
|
+
// CSE'd lane offsets: a local written ONLY as `i << K` (or bare `i`) is the
|
|
2529
|
+
// shared offset the IV stage threads across base pointers (src[i], dst[i],
|
|
2530
|
+
// out[i] all reuse one `(local.tee $p (local.get i))`). Resolve them so the
|
|
2531
|
+
// load/store address matchers accept the `(local.get $p)` reuses.
|
|
2532
|
+
const offsetTees = new Map()
|
|
2533
|
+
const allNames = new Set()
|
|
2534
|
+
const gatherNames = (n) => { if (!isArr(n)) return; if ((n[0] === 'local.set' || n[0] === 'local.tee') && typeof n[1] === 'string') allNames.add(n[1]); for (let i = 1; i < n.length; i++) gatherNames(n[i]) }
|
|
2535
|
+
for (const s of body) gatherNames(s)
|
|
2536
|
+
for (const name of allNames) { const k = _offsetLocalStride(body, name, ivName); if (k != null) offsetTees.set(name, k) }
|
|
2537
|
+
|
|
2538
|
+
const storeAddr = storeStmt[1]
|
|
2539
|
+
const addrM = matchLaneAddr(storeAddr, ivName, new Map(), offsetTees)
|
|
2540
|
+
if (!addrM || addrM.strideLog2 !== elemLog2) return null
|
|
2541
|
+
|
|
2542
|
+
// Memory loads turn this into a widening byte-map: out[i] = narrow(f(widen(a[i])…)).
|
|
2543
|
+
// Only the u8 shape is supported — every load must be a narrow UNSIGNED u8 load
|
|
2544
|
+
// of the same 4 elements the store writes (base + i; the IV strength-reducer
|
|
2545
|
+
// runs later). Full-width i32 maps are tryVectorize's job; mixed widths bail.
|
|
2546
|
+
let hasLoads = false, loadsOk = true
|
|
2547
|
+
const checkLoad = (n) => {
|
|
2548
|
+
if (!isArr(n)) return
|
|
2549
|
+
if (LOAD_OPS[n[0]]) {
|
|
2550
|
+
hasLoads = true
|
|
2551
|
+
if (storeOp !== 'i32.store8' || n[0] !== 'i32.load8_u') { loadsOk = false; return }
|
|
2552
|
+
const m = matchLaneAddr(n[1], ivName, new Map(), offsetTees)
|
|
2553
|
+
if (!m || m.strideLog2 !== 0) loadsOk = false
|
|
2554
|
+
return // address validated; the IV-strided subtree is not data
|
|
2555
|
+
}
|
|
2556
|
+
for (let i = 1; i < n.length; i++) checkLoad(n[i])
|
|
2557
|
+
}
|
|
2558
|
+
for (const s of body) checkLoad(s)
|
|
2559
|
+
if (!loadsOk) return null
|
|
2560
|
+
|
|
2561
|
+
// Every other body stmt must be `(local.set $lane EXPR)` — straight-line lane
|
|
2562
|
+
// locals feeding the store. Classify locals for the lift.
|
|
2563
|
+
const writes = new Set()
|
|
2564
|
+
for (const s of body) collectWrites(s, writes)
|
|
2565
|
+
if (boundLocal && writes.has(boundLocal)) return null
|
|
2566
|
+
const referenced = new Set()
|
|
2567
|
+
const collectRefs = (n) => {
|
|
2568
|
+
if (!isArr(n)) return
|
|
2569
|
+
if ((n[0] === 'local.get' || n[0] === 'local.set' || n[0] === 'local.tee') && typeof n[1] === 'string') referenced.add(n[1])
|
|
2570
|
+
for (let i = 1; i < n.length; i++) collectRefs(n[i])
|
|
2571
|
+
}
|
|
2572
|
+
for (const s of body) collectRefs(s)
|
|
2573
|
+
|
|
2574
|
+
const localKind = new Map()
|
|
2575
|
+
for (const name of referenced) {
|
|
2576
|
+
if (name === ivName) continue
|
|
2577
|
+
if (writes.has(name)) {
|
|
2578
|
+
let firstKind = null
|
|
2579
|
+
for (const s of body) { const kAcc = firstAccess(s, name); if (kAcc) { firstKind = kAcc; break } }
|
|
2580
|
+
if (firstKind === 'read') return null // loop-carried → reduction/stencil, not a pure map
|
|
2581
|
+
localKind.set(name, 'lane')
|
|
2582
|
+
} else {
|
|
2583
|
+
localKind.set(name, 'invariant')
|
|
2584
|
+
}
|
|
2585
|
+
}
|
|
2586
|
+
|
|
2587
|
+
// Lift. lane type is always i32 (the ramp and all narrow stores compute in i32).
|
|
2588
|
+
const newLanedLocals = new Map()
|
|
2589
|
+
const extraLocals = []
|
|
2590
|
+
const freshV128 = (tag) => { const n = `$__${tag}${freshIdRef.next++}`; extraLocals.push(['local', n, 'v128']); return n }
|
|
2591
|
+
const ctx = { laneType: 'i32', incVar: ivName, rampVar: ivName, rampTemp: null, widenLoads: true, localKind, newLanedLocals, extraLocals, freshIdRef, fail: false, failReason: null }
|
|
2592
|
+
|
|
2593
|
+
// A byte store fed by one value expression (inline, or via a single lane-local
|
|
2594
|
+
// temp `tw = EXPR; store(addr, tw)`) carries no loop-carried state, so we can
|
|
2595
|
+
// run the lane group 4× (16 samples) per iteration off four offset ramps and
|
|
2596
|
+
// pack the low bytes into ONE i8x16 v128.store — amortizing store + loop
|
|
2597
|
+
// overhead the way clang/zig's 16-wide NEON does. wideValueExpr is the
|
|
2598
|
+
// expression to lift per offset; null (i32 stores, multi-stmt bodies, or
|
|
2599
|
+
// widening loads — whose addresses would need per-offset advancing) → 4-wide.
|
|
2600
|
+
// The single byte-value expression feeding the store — inline, or via one lane-local
|
|
2601
|
+
// temp `tw = EXPR; store(addr, tw)`. Shared by both 16-wide paths below.
|
|
2602
|
+
const byteValueExpr = (() => {
|
|
2603
|
+
if (storeOp !== 'i32.store8') return null
|
|
2604
|
+
if (body.length === 1 && storeIdx === 0) return storeStmt[2]
|
|
2605
|
+
if (body.length === 2 && storeIdx === 1) {
|
|
2606
|
+
const set = body[0], sv = storeStmt[2]
|
|
2607
|
+
if (isArr(set) && set[0] === 'local.set' && set.length === 3 &&
|
|
2608
|
+
isLocalGet(sv, set[1]) && localKind.get(set[1]) === 'lane') return set[2]
|
|
2609
|
+
}
|
|
2610
|
+
return null
|
|
2611
|
+
})()
|
|
2612
|
+
// With u8 loads, the byte map can go 16-wide in i16x8 (the alpha-blend shape: out[i] =
|
|
2613
|
+
// (src[i]*A + dst[i]*B + bias) >> s) — load 16, extend_low/high, the affine arithmetic
|
|
2614
|
+
// in i16x8, narrow_u, store 16 — exactly clang's NEON. Sound ONLY when every
|
|
2615
|
+
// intermediate provably fits u16 ([0,65535], so i16x8 mod-2^16 never wraps and shr_u ==
|
|
2616
|
+
// the scalar shr_s on a non-negative value) and the result fits a byte ([0,255], so
|
|
2617
|
+
// narrow_u never saturates). `byteValueRange` returns [min,max] or null when any node
|
|
2618
|
+
// is unanalyzable, can go negative, or exceeds u16. An invariant local of unknown
|
|
2619
|
+
// magnitude (local.get) → null → falls back to the bit-exact 4-wide path.
|
|
2620
|
+
const byteValueRange = (e) => {
|
|
2621
|
+
if (!isArr(e)) return null
|
|
2622
|
+
const op = e[0]
|
|
2623
|
+
let r
|
|
2624
|
+
if (op === 'i32.const') { const v = constNum(e); r = [v, v] }
|
|
2625
|
+
else if (op === 'i32.load8_u') r = [0, 255]
|
|
2626
|
+
else if (op === 'i32.add') { const a = byteValueRange(e[1]), b = byteValueRange(e[2]); if (!a || !b) return null; r = [a[0] + b[0], a[1] + b[1]] }
|
|
2627
|
+
else if (op === 'i32.sub') { const a = byteValueRange(e[1]), b = byteValueRange(e[2]); if (!a || !b) return null; r = [a[0] - b[1], a[1] - b[0]] }
|
|
2628
|
+
else if (op === 'i32.mul') { const a = byteValueRange(e[1]), b = byteValueRange(e[2]); if (!a || !b) return null; const p = [a[0] * b[0], a[0] * b[1], a[1] * b[0], a[1] * b[1]]; r = [Math.min(...p), Math.max(...p)] }
|
|
2629
|
+
else if ((op === 'i32.shr_u' || op === 'i32.shr_s') && isI32Const(e[2])) { const a = byteValueRange(e[1]); if (!a || a[0] < 0) return null; const s = constNum(e[2]); if (s < 0 || s > 16) return null; r = [a[0] >> s, a[1] >> s] }
|
|
2630
|
+
else if (op === 'i32.and' && isI32Const(e[2])) { const a = byteValueRange(e[1]); if (!a) return null; const m = constNum(e[2]); if (m < 0) return null; r = [0, Math.min(a[1], m)] }
|
|
2631
|
+
else return null
|
|
2632
|
+
return (r[0] < 0 || r[1] > 65535) ? null : r
|
|
2633
|
+
}
|
|
2634
|
+
const wideValueExpr = (!hasLoads && byteValueExpr) ? byteValueExpr : null // pure ramp → 16-wide pack
|
|
2635
|
+
const widenRange = (hasLoads && byteValueExpr) ? byteValueRange(byteValueExpr) : null
|
|
2636
|
+
const WIDEN16 = widenRange != null && widenRange[1] <= 255 // result fits a byte ⇒ narrow_u exact
|
|
2637
|
+
const WIDE16 = wideValueExpr != null
|
|
2638
|
+
const LANES = (WIDE16 || WIDEN16) ? 16 : 4
|
|
2639
|
+
const ramp = (off) => ['i32x4.add', ['i32x4.splat', ['local.get', ivName]],
|
|
2640
|
+
['v128.const', 'i32x4', String(off), String(off + 1), String(off + 2), String(off + 3)]]
|
|
2641
|
+
|
|
2642
|
+
let lifted
|
|
2643
|
+
if (WIDEN16) {
|
|
2644
|
+
// 16-wide widening byte-map: load each u8 input once (v128.load of 16 bytes),
|
|
2645
|
+
// extend_low/high to two i16x8 halves, run the affine map in i16x8 on each half,
|
|
2646
|
+
// narrow_u the two result halves to one i8x16, store 16. Each load is hoisted to a
|
|
2647
|
+
// temp (so extend_low + extend_high share one load); the loads run in source order,
|
|
2648
|
+
// so the offset `local.tee` in the first one is set before later loads read it.
|
|
2649
|
+
const loadTemps = new Map()
|
|
2650
|
+
const loadSets = []
|
|
2651
|
+
const collectLoads = (e) => {
|
|
2652
|
+
if (!isArr(e)) return
|
|
2653
|
+
if (e[0] === 'i32.load8_u') {
|
|
2654
|
+
const k = JSON.stringify(e[1])
|
|
2655
|
+
if (!loadTemps.has(k)) { const t = freshV128('win'); loadTemps.set(k, t); loadSets.push(['local.set', t, ['v128.load', e[1]]]) }
|
|
2656
|
+
} else for (let i = 1; i < e.length; i++) collectLoads(e[i])
|
|
2657
|
+
}
|
|
2658
|
+
collectLoads(byteValueExpr)
|
|
2659
|
+
const liftW = (e, half) => {
|
|
2660
|
+
const op = e[0]
|
|
2661
|
+
if (op === 'i32.const') return ['i16x8.splat', e]
|
|
2662
|
+
if (op === 'i32.load8_u') return [`i16x8.extend_${half}_i8x16_u`, ['local.get', loadTemps.get(JSON.stringify(e[1]))]]
|
|
2663
|
+
if (op === 'i32.add') return ['i16x8.add', liftW(e[1], half), liftW(e[2], half)]
|
|
2664
|
+
if (op === 'i32.sub') return ['i16x8.sub', liftW(e[1], half), liftW(e[2], half)]
|
|
2665
|
+
if (op === 'i32.mul') return ['i16x8.mul', liftW(e[1], half), liftW(e[2], half)]
|
|
2666
|
+
if (op === 'i32.shr_u' || op === 'i32.shr_s') return ['i16x8.shr_u', liftW(e[1], half), e[2]]
|
|
2667
|
+
if (op === 'i32.and') return ['v128.and', liftW(e[1], half), ['i16x8.splat', e[2]]]
|
|
2668
|
+
return null // byteValueRange already proved every op is one of the above
|
|
2669
|
+
}
|
|
2670
|
+
lifted = [...loadSets,
|
|
2671
|
+
['v128.store', storeAddr, ['i8x16.narrow_i16x8_u', liftW(byteValueExpr, 'low'), liftW(byteValueExpr, 'high')]]]
|
|
2672
|
+
} else if (WIDE16) {
|
|
2673
|
+
lifted = []
|
|
2674
|
+
const vv = []
|
|
2675
|
+
for (let j = 0; j < 4; j++) {
|
|
2676
|
+
const rt = freshV128('ramp')
|
|
2677
|
+
lifted.push(['local.set', rt, ramp(j * 4)])
|
|
2678
|
+
ctx.rampTemp = rt
|
|
2679
|
+
const v = liftExprV(wideValueExpr, ctx)
|
|
2680
|
+
if (ctx.fail) return null
|
|
2681
|
+
const vn = freshV128('rampv')
|
|
2682
|
+
lifted.push(['local.set', vn, v])
|
|
2683
|
+
vv.push(vn)
|
|
2684
|
+
}
|
|
2685
|
+
// Pack the low byte of all 16 i32 lanes (4 vectors) into one i8x16, in order.
|
|
2686
|
+
const g = (n) => ['local.get', n]
|
|
2687
|
+
const sh = (a, b, idx) => ['i8x16.shuffle', ...idx.map(String), a, b]
|
|
2688
|
+
const lo = freshV128('ramplo'), hi = freshV128('ramphi')
|
|
2689
|
+
lifted.push(['local.set', lo, sh(g(vv[0]), g(vv[1]), [0, 4, 8, 12, 16, 20, 24, 28, 0, 0, 0, 0, 0, 0, 0, 0])])
|
|
2690
|
+
lifted.push(['local.set', hi, sh(g(vv[2]), g(vv[3]), [0, 4, 8, 12, 16, 20, 24, 28, 0, 0, 0, 0, 0, 0, 0, 0])])
|
|
2691
|
+
lifted.push(['v128.store', storeAddr, sh(g(lo), g(hi), [0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23])])
|
|
2692
|
+
} else {
|
|
2693
|
+
ctx.rampTemp = freshV128('ramp')
|
|
2694
|
+
// ramp = [i, i+1, i+2, i+3], computed once per SIMD iteration.
|
|
2695
|
+
lifted = [['local.set', ctx.rampTemp, ramp(0)]]
|
|
2696
|
+
for (let i = 0; i < body.length; i++) {
|
|
2697
|
+
if (i === storeIdx) {
|
|
2698
|
+
const vval = liftExprV(storeStmt[2], ctx)
|
|
2699
|
+
if (ctx.fail) return null
|
|
2700
|
+
lifted.push(buildRampStore(storeOp, storeAddr, vval, ctx))
|
|
2701
|
+
} else {
|
|
2702
|
+
const r = liftStmt(body[i], ctx)
|
|
2703
|
+
if (ctx.fail) return null
|
|
2704
|
+
if (r != null) { if (Array.isArray(r) && r[0] === '__seq__') lifted.push(...r.slice(1)); else lifted.push(r) }
|
|
2705
|
+
}
|
|
2706
|
+
}
|
|
2707
|
+
}
|
|
2708
|
+
if (!lifted.length) return null
|
|
2709
|
+
|
|
2710
|
+
const id = freshIdRef.next++
|
|
2711
|
+
const simdBoundName = `$__simd_bound${id}`
|
|
2712
|
+
const simdBrkLabel = `$__simd_brk${id}`
|
|
2713
|
+
const simdLoopLabel = `$__simd_loop${id}`
|
|
2714
|
+
const boundExpr = boundLocal ? ['local.get', boundLocal] : bound
|
|
2715
|
+
|
|
2716
|
+
const scaledIncs = increments.map(({ name, c }) =>
|
|
2717
|
+
['local.set', name, ['i32.add', ['local.get', name], ['i32.const', c * LANES]]])
|
|
2718
|
+
|
|
2719
|
+
const simdBlock = ['block', simdBrkLabel,
|
|
2720
|
+
['loop', simdLoopLabel,
|
|
2721
|
+
['br_if', simdBrkLabel, ['i32.eqz', ['i32.lt_s', ['local.get', ivName], ['local.get', simdBoundName]]]],
|
|
2722
|
+
...lifted,
|
|
2723
|
+
...scaledIncs,
|
|
2724
|
+
['br', simdLoopLabel]]]
|
|
2725
|
+
const boundSetup = ['local.set', simdBoundName, ['i32.and', boundExpr, ['i32.const', -LANES]]]
|
|
2726
|
+
const wrapper = ['block', boundSetup, simdBlock, blockNode]
|
|
2727
|
+
const newLocalDecls = [
|
|
2728
|
+
['local', simdBoundName, 'i32'],
|
|
2729
|
+
...[...newLanedLocals.values()].map(({ laneName }) => ['local', laneName, 'v128']),
|
|
2730
|
+
...extraLocals,
|
|
2731
|
+
]
|
|
2732
|
+
return { wrapper, newLocalDecls }
|
|
2733
|
+
}
|
|
2734
|
+
|
|
2735
|
+
// Build the store for a ramp-map iteration: i32x4 `vval` → element width of
|
|
2736
|
+
// `storeOp` at scalar address `addr`. i32.store is the full vector; i32.store8
|
|
2737
|
+
// truncates (low byte of each lane) via i8x16.shuffle — exactly matching scalar
|
|
2738
|
+
// store8, with no value-range assumption (shuffle selects, never saturates).
|
|
2739
|
+
function buildRampStore(storeOp, addr, vval, ctx) {
|
|
2740
|
+
if (storeOp === 'i32.store') return ['v128.store', addr, vval] // 4 i32 lanes → 16 bytes
|
|
2741
|
+
// i32.store8: hoist vval to a temp so the shuffle reads it once; low byte of
|
|
2742
|
+
// each of 4 lanes → bytes 0..3 → one i32.store (4 bytes). Shuffle lane indices
|
|
2743
|
+
// are string tokens for watr's binary encoder.
|
|
2744
|
+
const tmp = `$__rampv${ctx.freshIdRef.next++}`
|
|
2745
|
+
ctx.extraLocals.push(['local', tmp, 'v128'])
|
|
2746
|
+
const g = ['local.get', tmp]
|
|
2747
|
+
const packed = ['i8x16.shuffle', ...[0, 4, 8, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0].map(String), g, g]
|
|
2748
|
+
return ['block', ['local.set', tmp, vval], ['i32.store', addr, ['i32x4.extract_lane', 0, packed]]]
|
|
2749
|
+
}
|
|
2750
|
+
|
|
2751
|
+
// Pivot-stride analysis for the multi-pixel lift. The lift reads 16 source bytes
|
|
2752
|
+
// (4 RGBA pixels) with ONE v128.load at the address for output pixel `pivot`, so the
|
|
2753
|
+
// 4 outputs are correct ONLY if consecutive output pixels read consecutive source
|
|
2754
|
+
// pixels — the load address must advance by EXACTLY 4 bytes per pivot step. Build, in
|
|
2755
|
+
// program order, each local's value-delta per unit-pivot increment (`pivot` → 1, every
|
|
2756
|
+
// other local → its assigned expr's delta, unknown/outer locals → 0 = pivot-invariant).
|
|
2757
|
+
// `delta(e)` returns that constant byte-delta, or null when the dependence is non-
|
|
2758
|
+
// constant (e.g. x*k) or uses an op we don't model — both of which must bail.
|
|
2759
|
+
// Index arithmetic is often f64-lowered (JS number `*`): `(yi*ww + x)` becomes
|
|
2760
|
+
// `(f64(yi)*ww + f64(x)) |0` = trunc_sat with a NaN-guard `select`. We model those
|
|
2761
|
+
// passthrough/arith ops too, so a runtime-dimension vblur (x carried through f64)
|
|
2762
|
+
// analyses the same as a literal-dimension one (x in plain i32). The select is the
|
|
2763
|
+
// `|0` coercion (value branch when the index is finite — always so for an integer
|
|
2764
|
+
// array index); we take the value branch's delta. Multiplies need a compile-time
|
|
2765
|
+
// constant factor on the pivot-bearing side (else the stride isn't constant).
|
|
2766
|
+
function buildPivotCoeff(loopNode, pivot) {
|
|
2767
|
+
const coeff = new Map([[pivot, 1]])
|
|
2768
|
+
const cval = (n) => isI32Const(n) ? constNum(n) : (isArr(n) && n[0] === 'f64.const' && n[1] != null ? Number(n[1]) : null)
|
|
2769
|
+
const delta = (e) => {
|
|
2770
|
+
if (isI32Const(e)) return 0
|
|
2771
|
+
if (isLocalGet(e)) return coeff.has(e[1]) ? coeff.get(e[1]) : 0
|
|
2772
|
+
if (!isArr(e)) return null
|
|
2773
|
+
const [op, a, b] = e
|
|
2774
|
+
switch (op) {
|
|
2775
|
+
case 'i32.add': case 'f64.add': { const x = delta(a), y = delta(b); return x == null || y == null ? null : x + y }
|
|
2776
|
+
case 'i32.sub': case 'f64.sub': { const x = delta(a), y = delta(b); return x == null || y == null ? null : x - y }
|
|
2777
|
+
case 'i32.shl': { const c = cval(b), x = delta(a); return c == null || x == null ? null : x << c }
|
|
2778
|
+
case 'i32.mul': case 'f64.mul': {
|
|
2779
|
+
const ca = cval(a), cb = cval(b), x = delta(a), y = delta(b)
|
|
2780
|
+
if (x == null || y == null) return null
|
|
2781
|
+
if (ca != null) return ca * y // const × expr
|
|
2782
|
+
if (cb != null) return cb * x // expr × const
|
|
2783
|
+
return x === 0 && y === 0 ? 0 : null // var × var: pivot-dependent ⇒ non-constant
|
|
2784
|
+
}
|
|
2785
|
+
// value-preserving conversions: passthrough. `local.tee $v EXPR` yields EXPR.
|
|
2786
|
+
case 'f64.convert_i32_s': case 'f64.convert_i32_u': case 'i64.trunc_sat_f64_s':
|
|
2787
|
+
case 'i64.trunc_sat_f64_u': case 'i32.wrap_i64': case 'i64.extend_i32_s':
|
|
2788
|
+
return delta(a)
|
|
2789
|
+
case 'local.tee': return delta(b)
|
|
2790
|
+
// the `(expr)|0` NaN/Inf-guard `select(value, 0, finite?)`: take the value branch
|
|
2791
|
+
// (the index is a finite integer in any real blur). Only when the else-branch is
|
|
2792
|
+
// the const-0 guard — a general ternary index whose arms differ in stride bails.
|
|
2793
|
+
case 'select': return isI32Const(e[2]) && constNum(e[2]) === 0 ? delta(a) : null
|
|
2794
|
+
default: return null // any other op (and/or/div/…) ⇒ unanalyzable
|
|
2795
|
+
}
|
|
2796
|
+
}
|
|
2797
|
+
const visit = (n) => {
|
|
2798
|
+
if (!isArr(n)) return
|
|
2799
|
+
if ((n[0] === 'local.set' || n[0] === 'local.tee') && typeof n[1] === 'string') { coeff.set(n[1], delta(n[2])); visit(n[2]); return }
|
|
2800
|
+
n.forEach(visit)
|
|
2801
|
+
}
|
|
2802
|
+
visit(loopNode)
|
|
2803
|
+
return delta
|
|
2804
|
+
}
|
|
2805
|
+
|
|
2806
|
+
// Vectorize the inner accumulation of an RGBA box-filter pixel loop. See the
|
|
2807
|
+
// matchChannelGroup header. Returns { wrapper, newLocalDecls } or null.
|
|
2808
|
+
// Multi-pixel SIMD lift of a clamp-free RGBA box-filter interior (produced by the
|
|
2809
|
+
// clamp-peel pass). Processes 4 output pixels per iteration: at tap k the 4 outputs
|
|
2810
|
+
// read 4 CONSECUTIVE source pixels — one 16-byte v128.load whose lanes map DIRECTLY
|
|
2811
|
+
// to the 4 accumulators (no shuffle). Sums (≤ win·255 < 2^16) accumulate in two
|
|
2812
|
+
// i16x8 vectors; the divide+store reuses the scalar store templates per pixel.
|
|
2813
|
+
// Validated bit-exact + 4.34× vs scalar. Bails (→ tryChannelReduce 1-pixel lift)
|
|
2814
|
+
// unless the body is exactly: clamp-free `xi=x+k; p=(row+xi)<<2; acc_c += u8[base+c]`
|
|
2815
|
+
// over a unit-stride x, then a 4-byte RGBA store `dst[ab1+c] = f(acc_c)`. r≥127 →
|
|
2816
|
+
// i16 overflow → bail. Leaves a scalar remainder loop for the ≤3 trailing pixels.
|
|
2817
|
+
function tryBlurMultiPixel(blockNode, fnLocals, freshIdRef) {
|
|
2818
|
+
if (!isArr(blockNode) || blockNode[0] !== 'block') return null
|
|
2819
|
+
let blockLabel = null, loopNode = null
|
|
2820
|
+
for (let i = 1; i < blockNode.length; i++) {
|
|
2821
|
+
const c = blockNode[i]
|
|
2822
|
+
if (typeof c === 'string' && c.startsWith('$') && blockLabel == null && i === 1) { blockLabel = c; continue }
|
|
2823
|
+
if (isArr(c) && c[0] === 'loop') { if (loopNode) return null; loopNode = c }
|
|
2824
|
+
}
|
|
2825
|
+
if (!loopNode || !blockLabel) return null
|
|
2826
|
+
const loopLabel = typeof loopNode[1] === 'string' && loopNode[1].startsWith('$') ? loopNode[1] : null
|
|
2827
|
+
if (!loopLabel) return null
|
|
2828
|
+
const endIdx = loopNode.length - 1
|
|
2829
|
+
if (!(isArr(loopNode[endIdx]) && loopNode[endIdx][0] === 'br' && loopNode[endIdx][1] === loopLabel)) return null
|
|
2830
|
+
// exit guard: br_if $brk (i32.eqz (i32.lt_s x BOUND))
|
|
2831
|
+
const exit = loopNode[2]
|
|
2832
|
+
if (!(isArr(exit) && exit[0] === 'br_if' && isArr(exit[2]) && exit[2][0] === 'i32.eqz'
|
|
2833
|
+
&& isArr(exit[2][1]) && exit[2][1][0] === 'i32.lt_s' && isLocalGet(exit[2][1][1]))) return null
|
|
2834
|
+
const pivot = exit[2][1][1][1] // pixel IV $x
|
|
2835
|
+
const bound = exit[2][1][2] // interior bound (expr)
|
|
2836
|
+
const bodyEnd = endIdx - 1
|
|
2837
|
+
// increment must be `x = x + 1`
|
|
2838
|
+
const inc = loopNode[bodyEnd]
|
|
2839
|
+
if (!(isArr(inc) && inc[0] === 'local.set' && inc[1] === pivot && isArr(inc[2]) && inc[2][0] === 'i32.add'
|
|
2840
|
+
&& isLocalGet(inc[2][1], pivot) && isI32Const(inc[2][2]) && constNum(inc[2][2]) === 1)) return null
|
|
2841
|
+
|
|
2842
|
+
// four `acc_c = 0` inits
|
|
2843
|
+
let initIdx = -1, accInits = null
|
|
2844
|
+
for (let i = 3; i + 3 <= bodyEnd; i++) {
|
|
2845
|
+
const z = []
|
|
2846
|
+
for (let c = 0; c < 4; c++) { const s = loopNode[i + c]; if (isArr(s) && s[0] === 'local.set' && s.length === 3 && isI32Const(s[2]) && constNum(s[2]) === 0) z.push(s[1]); else break }
|
|
2847
|
+
if (z.length === 4 && new Set(z).size === 4) { initIdx = i; accInits = z; break }
|
|
2848
|
+
}
|
|
2849
|
+
if (initIdx < 0) return null
|
|
2850
|
+
// the tap accumulation loop
|
|
2851
|
+
let innerIdx = -1, innerBlock = null
|
|
2852
|
+
for (let i = initIdx + 4; i <= bodyEnd; i++) {
|
|
2853
|
+
const s = loopNode[i]
|
|
2854
|
+
if (isArr(s) && s[0] === 'block' && s.slice(1).some(c => isArr(c) && c[0] === 'loop')) { innerIdx = i; innerBlock = s; break }
|
|
2855
|
+
if (isArr(s) && s[0] === 'loop') { innerIdx = i; innerBlock = ['block', s]; break }
|
|
2856
|
+
}
|
|
2857
|
+
if (!innerBlock) return null
|
|
2858
|
+
const innerLoop = innerBlock.find(c => isArr(c) && c[0] === 'loop')
|
|
2859
|
+
if (!innerLoop) return null
|
|
2860
|
+
const ilEnd = innerLoop.length - 1
|
|
2861
|
+
if (!(isArr(innerLoop[ilEnd]) && innerLoop[ilEnd][0] === 'br')) return null
|
|
2862
|
+
const innerBody = innerLoop.slice(3, ilEnd - 1)
|
|
2863
|
+
const grp = matchChannelGroup(innerBody)
|
|
2864
|
+
if (!grp) return null
|
|
2865
|
+
if (grp.accs.slice().sort().join('\x00') !== accInits.slice().sort().join('\x00')) return null
|
|
2866
|
+
// CLAMP-FREE: nothing before the channel group may be an `if` (the edge clamp).
|
|
2867
|
+
for (let i = 0; i < grp.idx; i++) if (isArr(innerBody[i]) && innerBody[i][0] === 'if') return null
|
|
2868
|
+
// The tap loop must be the i32 form `k <= r`; extract the radius r (reused in the
|
|
2869
|
+
// runtime overflow guard below). f64-typed tap loops (f64 dims) bail to scalar.
|
|
2870
|
+
const ic = innerLoop[2]
|
|
2871
|
+
if (!(isArr(ic) && ic[0] === 'br_if' && isArr(ic[2]) && ic[2][0] === 'i32.eqz'
|
|
2872
|
+
&& isArr(ic[2][1]) && ic[2][1][0] === 'i32.le_s' && isLocalGet(ic[2][1][1]))) return null
|
|
2873
|
+
const rBound = ic[2][1][2]
|
|
2874
|
+
if (!(isLocalGet(rBound) || isI32Const(rBound))) return null
|
|
2875
|
+
|
|
2876
|
+
// the RGBA store: 4 consecutive i32.store8 at `ab1 + c`, ab1 tee'd in the first.
|
|
2877
|
+
let storeIdx = -1
|
|
2878
|
+
for (let i = innerIdx + 1; i + 3 <= bodyEnd; i++) {
|
|
2879
|
+
const s0 = loopNode[i]
|
|
2880
|
+
if (isArr(s0) && s0[0] === 'i32.store8' && s0.length === 3 && isArr(s0[1]) && s0[1][0] === 'local.tee') { storeIdx = i; break }
|
|
2881
|
+
}
|
|
2882
|
+
if (storeIdx < 0) return null
|
|
2883
|
+
const s0 = loopNode[storeIdx]
|
|
2884
|
+
const ab1 = s0[1][1], dstExpr = s0[1][2] // ab1 local, its address expr
|
|
2885
|
+
const storeVals = [s0[2]]
|
|
2886
|
+
for (let c = 1; c < 4; c++) {
|
|
2887
|
+
const s = loopNode[storeIdx + c]
|
|
2888
|
+
if (!(isArr(s) && s[0] === 'i32.store8' && s[1] === `offset=${c}` && isLocalGet(s[2], ab1))) return null
|
|
2889
|
+
storeVals.push(s[3])
|
|
2890
|
+
}
|
|
2891
|
+
// each store value must read its accumulator (the divided sum)
|
|
2892
|
+
for (let c = 0; c < 4; c++) if (!readsVar(storeVals[c], accInits[c])) return null
|
|
2893
|
+
|
|
2894
|
+
// ── Soundness guards for the 4-pixels-per-load lift ──────────────────────────
|
|
2895
|
+
// Scope the stride analysis to THIS pixel loop. A peeled interior is one segment ⇒
|
|
2896
|
+
// one inner tap loop, so there is no sibling same-named-`p` ambiguity, and the scope
|
|
2897
|
+
// still captures the LICM preamble where the x-dependent index term is hoisted out of
|
|
2898
|
+
// the tap loop (e.g. `__li = f64(x)` — x is invariant w.r.t. the tap IV k).
|
|
2899
|
+
const pivotDelta = buildPivotCoeff(loopNode, pivot)
|
|
2900
|
+
// (a) Source unit-stride: the v128.load address (array base + byte index) must
|
|
2901
|
+
// advance by exactly 4 bytes (one RGBA pixel) per output-pixel step, else the
|
|
2902
|
+
// 16-byte load spans the wrong source pixels. Bails on strided indices
|
|
2903
|
+
// ((xi*2+row)<<2) and non-constant ones ((yi*W + x + k*x)<<2 → x*(1+k)).
|
|
2904
|
+
if (pivotDelta(grp.teeExpr) !== 4) return null
|
|
2905
|
+
// (b) No dropped pivot-dependent setup: statements before the inits (loopNode[3..
|
|
2906
|
+
// initIdx)) are NOT carried into the 4-pixel loop, so a per-pixel value computed
|
|
2907
|
+
// there (e.g. `x2 = x*2`) would go stale. Bail if any such statement is pivot-
|
|
2908
|
+
// dependent. (Statements between the inits and the inner loop ARE carried.)
|
|
2909
|
+
for (let i = 3; i < initIdx; i++) {
|
|
2910
|
+
let bad = false
|
|
2911
|
+
const chk = (n) => { if (!isArr(n)) return; if ((n[0] === 'local.set' || n[0] === 'local.tee') && typeof n[1] === 'string' && pivotDelta(n[2]) !== 0) bad = true; n.forEach(chk) }
|
|
2912
|
+
chk(loopNode[i])
|
|
2913
|
+
if (bad) return null
|
|
2914
|
+
}
|
|
2915
|
+
// (c) Store values must read only the accumulators (set per-pixel by the epilogue)
|
|
2916
|
+
// and pivot-invariants — never a per-pixel local like the pivot itself, since the
|
|
2917
|
+
// store template is reused verbatim for all 4 pixels (`dst[o]=(sr+x)&255` would use
|
|
2918
|
+
// the group-base x for pixels 1-3).
|
|
2919
|
+
const perPixel = new Set()
|
|
2920
|
+
const collectSet = (n) => { if (!isArr(n)) return; if ((n[0] === 'local.set' || n[0] === 'local.tee') && typeof n[1] === 'string') perPixel.add(n[1]); n.forEach(collectSet) }
|
|
2921
|
+
collectSet(loopNode)
|
|
2922
|
+
for (const a of grp.accs) perPixel.delete(a) // accumulators are repopulated per pixel
|
|
2923
|
+
const readsPerPixel = (n) => isArr(n) ? ((n[0] === 'local.get' && perPixel.has(n[1])) || n.some(readsPerPixel)) : false
|
|
2924
|
+
for (let c = 0; c < 4; c++) if (readsPerPixel(storeVals[c])) return null
|
|
2925
|
+
|
|
2926
|
+
// ── Lift ───────────────────────────────────────────────────────────────────
|
|
2927
|
+
const id = freshIdRef.next++
|
|
2928
|
+
const accLo = `$__bmplo${id}`, accHi = `$__bmphi${id}`, b4 = `$__bmpb${id}`
|
|
2929
|
+
const newLocalDecls = [['local', accLo, 'v128'], ['local', accHi, 'v128'], ['local', b4, 'i32']]
|
|
2930
|
+
const Z = ['v128.const', 'i64x2', '0', '0']
|
|
2931
|
+
// inner body: 4 channel adds → 2 i16 widening adds off the same tee'd base.
|
|
2932
|
+
const newInner = []
|
|
2933
|
+
for (let i = 0; i < innerBody.length; i++) {
|
|
2934
|
+
if (i === grp.idx) {
|
|
2935
|
+
newInner.push(['local.set', accLo, ['i16x8.add', ['local.get', accLo], ['i16x8.extend_low_i8x16_u', ['v128.load', ['local.tee', grp.baseLocal, grp.teeExpr]]]]])
|
|
2936
|
+
newInner.push(['local.set', accHi, ['i16x8.add', ['local.get', accHi], ['i16x8.extend_high_i8x16_u', ['v128.load', ['local.get', grp.baseLocal]]]]])
|
|
2937
|
+
} else if (i > grp.idx && i <= grp.idx + 3) continue
|
|
2938
|
+
else newInner.push(innerBody[i])
|
|
2939
|
+
}
|
|
2940
|
+
const newInnerLoop = [...innerLoop.slice(0, 3), ...newInner, ...innerLoop.slice(ilEnd - 1)]
|
|
2941
|
+
const newInnerBlock = innerBlock.map(c => c === innerLoop ? newInnerLoop : c)
|
|
2942
|
+
// store epilogue: the o=(row+x)<<2 setup, then ab1, then per pixel j set acc_c to
|
|
2943
|
+
// its lane and reuse the scalar store template at byte offset j*4+c.
|
|
2944
|
+
const preStore = loopNode.slice(innerIdx + 1, storeIdx) // `o = (row+x)<<2` etc.
|
|
2945
|
+
const epilogue = [...preStore, ['local.set', ab1, dstExpr]]
|
|
2946
|
+
for (let j = 0; j < 4; j++) {
|
|
2947
|
+
const vec = j < 2 ? accLo : accHi, base = (j % 2) * 4
|
|
2948
|
+
// lane base+c summed source byte at offset c → grp.accs[c] (the accumulator
|
|
2949
|
+
// tied to read-offset c), NOT accInits[c] (zero-init order); see tryChannelReduce.
|
|
2950
|
+
for (let c = 0; c < 4; c++) epilogue.push(['local.set', grp.accs[c], ['i16x8.extract_lane_u', String(base + c), ['local.get', vec]]])
|
|
2951
|
+
for (let c = 0; c < 4; c++) epilogue.push(['i32.store8', `offset=${j * 4 + c}`, ['local.get', ab1], storeVals[c]])
|
|
2952
|
+
}
|
|
2953
|
+
// 4-pixel loop body: splat inits, the tap-IV init, the lifted inner loop, the
|
|
2954
|
+
// lifted store, x += 4.
|
|
2955
|
+
const v4label = `$__bmpx${id}`, v4brk = `$__bmpe${id}`
|
|
2956
|
+
const fourBody = [
|
|
2957
|
+
['br_if', v4brk, ['i32.eqz', ['i32.lt_s', ['local.get', pivot], ['local.get', b4]]]],
|
|
2958
|
+
['local.set', accLo, Z], ['local.set', accHi, Z],
|
|
2959
|
+
...loopNode.slice(initIdx + 4, innerIdx),
|
|
2960
|
+
newInnerBlock,
|
|
2961
|
+
...epilogue,
|
|
2962
|
+
['local.set', pivot, ['i32.add', ['local.get', pivot], ['i32.const', 4]]],
|
|
2963
|
+
['br', v4label],
|
|
2964
|
+
]
|
|
2965
|
+
// Deep-clone the 4-pixel loop: it reuses sub-trees (inner body, store templates,
|
|
2966
|
+
// dst expr) that also live in the scalar remainder below — sharing would let a
|
|
2967
|
+
// later pass mutating one corrupt the other.
|
|
2968
|
+
const dc = (n) => isArr(n) ? n.map(dc) : n
|
|
2969
|
+
const fourLoop = dc(['block', v4brk, ['loop', v4label, ...fourBody]])
|
|
2970
|
+
// b4 = x + (r<128 ? ((bound-x) & ~3) : 0): the 4-aligned end of the interior from
|
|
2971
|
+
// the entry x. The r<128 guard is the i16-overflow check (win·255 < 2^16 ⇔ r<128);
|
|
2972
|
+
// when r≥128 the 4-pixel loop runs zero iterations and the scalar remainder covers
|
|
2973
|
+
// the whole interior — sound for any runtime r, no duplicated loop body.
|
|
2974
|
+
const b4set = ['local.set', b4, ['i32.add', ['local.get', pivot],
|
|
2975
|
+
['select', ['i32.and', ['i32.sub', dc(bound), ['local.get', pivot]], ['i32.const', -4]], ['i32.const', 0],
|
|
2976
|
+
['i32.lt_s', dc(rBound), ['i32.const', 128]]]]]
|
|
2977
|
+
// The interior block may carry a LICM-hoisted invariant preamble (e.g. `k=-r`'s
|
|
2978
|
+
// value) that the lifted body reads but which the original sets only on block
|
|
2979
|
+
// entry — run a clone of it first so the 4-pixel loop sees it. Then b4 setup, the
|
|
2980
|
+
// 4-pixel loop, then the ORIGINAL block unchanged (the ≤3-pixel scalar remainder;
|
|
2981
|
+
// x continues from b4).
|
|
2982
|
+
const loopPos = blockNode.indexOf(loopNode)
|
|
2983
|
+
const preamble = blockNode.slice(2, loopPos).map(dc)
|
|
2984
|
+
const wrapper = ['block', ...preamble, b4set, fourLoop, blockNode]
|
|
2985
|
+
return { wrapper, newLocalDecls }
|
|
2986
|
+
}
|
|
2987
|
+
|
|
2988
|
+
function tryChannelReduce(blockNode, fnLocals, freshIdRef) {
|
|
2989
|
+
if (!isArr(blockNode) || blockNode[0] !== 'block') return null
|
|
2990
|
+
// Outer pixel loop: (block $brk [invariant preamble…] (loop $L …)). When the
|
|
2991
|
+
// pixel loop is nested in an outer row loop, LICM hoists the invariant edge-clamp
|
|
2992
|
+
// bound (`w-1`) into this block ahead of the loop; allow any such non-loop
|
|
2993
|
+
// preamble — it is preserved verbatim by the wrapper rebuild, which rewrites only
|
|
2994
|
+
// the loop. Exactly one loop is still required (a second → bail).
|
|
2995
|
+
let blockLabel = null, loopNode = null
|
|
2996
|
+
for (let i = 1; i < blockNode.length; i++) {
|
|
2997
|
+
const c = blockNode[i]
|
|
2998
|
+
if (typeof c === 'string' && c.startsWith('$') && blockLabel == null && i === 1) { blockLabel = c; continue }
|
|
2999
|
+
if (isArr(c) && c[0] === 'loop') { if (loopNode) return null; loopNode = c }
|
|
3000
|
+
}
|
|
3001
|
+
if (!loopNode || !blockLabel) return null
|
|
3002
|
+
const loopLabel = typeof loopNode[1] === 'string' && loopNode[1].startsWith('$') ? loopNode[1] : null
|
|
3003
|
+
if (!loopLabel) return null
|
|
3004
|
+
const endIdx = loopNode.length - 1
|
|
3005
|
+
if (!(isArr(loopNode[endIdx]) && loopNode[endIdx][0] === 'br' && loopNode[endIdx][1] === loopLabel)) return null
|
|
3006
|
+
|
|
3007
|
+
// Pixel-loop body (between the exit guard and the back-branch). Find: four
|
|
3008
|
+
// `acc_c = 0` inits and the inner accumulation loop that sums into them.
|
|
3009
|
+
const bodyStart = 3, bodyEnd = endIdx - 1 // [exit] at 2, [inc?][br] at the end
|
|
3010
|
+
// The four zero-inits must be consecutive `(local.set $acc (i32.const 0))`.
|
|
3011
|
+
let initIdx = -1, accInits = null
|
|
3012
|
+
for (let i = bodyStart; i + 3 <= bodyEnd; i++) {
|
|
3013
|
+
const z = []
|
|
3014
|
+
for (let c = 0; c < 4; c++) {
|
|
3015
|
+
const s = loopNode[i + c]
|
|
3016
|
+
if (isArr(s) && s[0] === 'local.set' && s.length === 3 && isI32Const(s[2]) && constNum(s[2]) === 0 && typeof s[1] === 'string') z.push(s[1])
|
|
3017
|
+
else break
|
|
3018
|
+
}
|
|
3019
|
+
if (z.length === 4 && new Set(z).size === 4) { initIdx = i; accInits = z; break }
|
|
3020
|
+
}
|
|
3021
|
+
if (initIdx < 0) return null
|
|
3022
|
+
|
|
3023
|
+
// The inner accumulation loop is the (block (loop)) appearing after the inits.
|
|
3024
|
+
let innerIdx = -1, innerBlock = null
|
|
3025
|
+
for (let i = initIdx + 4; i <= bodyEnd; i++) {
|
|
3026
|
+
const s = loopNode[i]
|
|
3027
|
+
if (isArr(s) && s[0] === 'block' && s.slice(1).some(c => isArr(c) && c[0] === 'loop')) { innerIdx = i; innerBlock = s; break }
|
|
3028
|
+
if (isArr(s) && s[0] === 'loop') { innerIdx = i; innerBlock = ['block', s]; break }
|
|
3029
|
+
}
|
|
3030
|
+
if (!innerBlock) return null
|
|
3031
|
+
// Locate the loop within the inner block and its body.
|
|
3032
|
+
const innerLoop = innerBlock.find(c => isArr(c) && c[0] === 'loop')
|
|
3033
|
+
if (!innerLoop) return null
|
|
3034
|
+
const ilEnd = innerLoop.length - 1
|
|
3035
|
+
if (!(isArr(innerLoop[ilEnd]) && innerLoop[ilEnd][0] === 'br')) return null
|
|
3036
|
+
const innerBody = innerLoop.slice(3, ilEnd - 1) // between exit guard and the (k+=1)(br)
|
|
3037
|
+
|
|
3038
|
+
const grp = matchChannelGroup(innerBody)
|
|
3039
|
+
if (!grp) return null
|
|
3040
|
+
// The four accumulators summed must be exactly the four that were zero-inited
|
|
3041
|
+
// (same set, in any order).
|
|
3042
|
+
if (grp.accs.slice().sort().join('\x00') !== accInits.slice().sort().join('\x00')) return null
|
|
3043
|
+
|
|
3044
|
+
// ── Lift. accv (v128) holds the four channel sums.
|
|
3045
|
+
const id = freshIdRef.next++
|
|
3046
|
+
const accv = `$__chv${id}`
|
|
3047
|
+
const newLocalDecls = [['local', accv, 'v128']]
|
|
3048
|
+
// Zero-init → one splat.
|
|
3049
|
+
const zeroInit = ['local.set', accv, ['v128.const', 'i32x4', '0', '0', '0', '0']]
|
|
3050
|
+
// Inner accumulation: keep the address-producing stmts, replace the 4 channel
|
|
3051
|
+
// adds with one widening add off the shared base. The base address is the
|
|
3052
|
+
// tee'd expr of the first load (re-tee'd so any later `local.get base` still
|
|
3053
|
+
// resolves — though the other channel loads are gone).
|
|
3054
|
+
const widen = ['i32x4.extend_low_i16x8_u', ['i16x8.extend_low_i8x16_u',
|
|
3055
|
+
['v128.load32_zero', ['local.tee', grp.baseLocal, grp.teeExpr]]]]
|
|
3056
|
+
const newInner = []
|
|
3057
|
+
for (let i = 0; i < innerBody.length; i++) {
|
|
3058
|
+
if (i === grp.idx) newInner.push(['local.set', accv, ['i32x4.add', ['local.get', accv], widen]])
|
|
3059
|
+
else if (i > grp.idx && i <= grp.idx + 3) continue // drop the other 3 channel adds
|
|
3060
|
+
else newInner.push(innerBody[i])
|
|
3061
|
+
}
|
|
3062
|
+
// Rebuild the inner loop with the lifted body.
|
|
3063
|
+
const newInnerLoop = [...innerLoop.slice(0, 3), ...newInner, ...innerLoop.slice(ilEnd - 1)]
|
|
3064
|
+
const newInnerBlock = innerBlock.map(c => c === innerLoop ? newInnerLoop : c)
|
|
3065
|
+
// After the inner loop, extract the four lane sums back to the scalar
|
|
3066
|
+
// accumulators the divide+store code reads. Lane c summed source byte
|
|
3067
|
+
// `base+c`, so it must land in grp.accs[c] — the accumulator matchChannelGroup
|
|
3068
|
+
// tied to read-offset c — NOT accInits[c] (zero-init order). They coincide only
|
|
3069
|
+
// when the program inits/sums in offset order (every real blur); a channel-
|
|
3070
|
+
// permuted program (sums offset 0 into a var stored at a later offset) needs the
|
|
3071
|
+
// offset-order mapping or the lanes mis-map.
|
|
3072
|
+
const extracts = grp.accs.map((acc, c) => ['local.set', acc, ['i32x4.extract_lane', c, ['local.get', accv]]])
|
|
3073
|
+
|
|
3074
|
+
// Reassemble the pixel-loop body: replace the 4 inits with the splat, the inner
|
|
3075
|
+
// block with the lifted one, and insert the extracts right after it.
|
|
3076
|
+
const newLoop = [...loopNode]
|
|
3077
|
+
newLoop.splice(innerIdx, 1, newInnerBlock, ...extracts) // inner block → lifted + extracts
|
|
3078
|
+
newLoop.splice(initIdx, 4, zeroInit) // 4 inits → 1 splat (do last; lower index)
|
|
3079
|
+
const wrapper = blockNode.map(c => c === loopNode ? newLoop : c)
|
|
3080
|
+
return { wrapper, newLocalDecls }
|
|
3081
|
+
}
|
|
3082
|
+
|
|
3083
|
+
/**
|
|
3084
|
+
* Divergent escape-time vectorizer — 2-wide f64x2, bit-exact with scalar f64.
|
|
3085
|
+
*
|
|
3086
|
+
* Recognizes the fractal escape-time nest (mandelbrot / julia / burning-ship / …):
|
|
3087
|
+
*
|
|
3088
|
+
* for (qx = 0; qx < W; qx++) { // outer pixel loop; ≥1 i32 pixel IVs
|
|
3089
|
+
* cx = <expr in qx> // per-pixel coordinate(s)
|
|
3090
|
+
* x = 0; y = 0; it = 0 // loop-carried f64 + i32 counter
|
|
3091
|
+
* while (it < MAXIT) { // inner escape loop (it-limited)
|
|
3092
|
+
* <f64 updates to x,y (+ temps); one `if (|z|² > T) break`, ANY position>
|
|
3093
|
+
* it++
|
|
3094
|
+
* }
|
|
3095
|
+
* <epilogue: store(it) OR smooth-colour(x,y,it) → store>
|
|
3096
|
+
* }
|
|
3097
|
+
*
|
|
3098
|
+
* Two adjacent pixels run in f64x2 lockstep with a per-lane active mask. A lane is
|
|
3099
|
+
* frozen (v128.bitselect) the instant it would escape or reach MAXIT, so its
|
|
3100
|
+
* it/x/y stay bit-identical to the scalar loop — f64x2 add/sub/mul/abs are IEEE-
|
|
3101
|
+
* identical to scalar f64 (no FMA fusion). The body is emitted statement-by-
|
|
3102
|
+
* statement in source order, so the escape mask lands exactly where the scalar
|
|
3103
|
+
* `break` is (before OR after the z-update), and the per-lane state freezes at the
|
|
3104
|
+
* matching point. iter is kept as f64x2 (an i32 `it` is exact in f64), so the
|
|
3105
|
+
* limit compare and the (possibly fractional) colour math both stay bitwise-exact.
|
|
3106
|
+
*
|
|
3107
|
+
* Loop-carried vars (first body access is a READ) freeze via bitselect; within-
|
|
3108
|
+
* iteration temps (first access a WRITE — inlined squares, `xt`) recompute raw.
|
|
3109
|
+
* The colour epilogue runs scalar, twice — once per lane — reading each lane's
|
|
3110
|
+
* extracted x/y/it, with the pixel IVs advanced by +0/+1. Odd widths and extra
|
|
3111
|
+
* lanes fall through to the original scalar pixel loop (the exact tail).
|
|
3112
|
+
*/
|
|
3113
|
+
const CMP_NEG = { // comparison → its logical negation (active lanes are finite → NaN-free)
|
|
3114
|
+
'f64.gt': 'f64.le', 'f64.ge': 'f64.lt', 'f64.lt': 'f64.ge', 'f64.le': 'f64.gt', 'f64.eq': 'f64.ne', 'f64.ne': 'f64.eq',
|
|
3115
|
+
'i32.lt_s': 'i32.ge_s', 'i32.ge_s': 'i32.lt_s', 'i32.gt_s': 'i32.le_s', 'i32.le_s': 'i32.gt_s',
|
|
3116
|
+
'i32.lt_u': 'i32.ge_u', 'i32.ge_u': 'i32.lt_u', 'i32.gt_u': 'i32.le_u', 'i32.le_u': 'i32.gt_u',
|
|
3117
|
+
}
|
|
3118
|
+
const CMP_LANE = { // f64/i32 scalar compare → f64x2 lane compare (iter is f64x2; z-compares are f64)
|
|
3119
|
+
'f64.gt': 'f64x2.gt', 'f64.ge': 'f64x2.ge', 'f64.lt': 'f64x2.lt', 'f64.le': 'f64x2.le', 'f64.eq': 'f64x2.eq', 'f64.ne': 'f64x2.ne',
|
|
3120
|
+
'i32.lt_s': 'f64x2.lt', 'i32.le_s': 'f64x2.le', 'i32.ge_s': 'f64x2.ge', 'i32.gt_s': 'f64x2.gt',
|
|
3121
|
+
'i32.lt_u': 'f64x2.lt', 'i32.le_u': 'f64x2.le', 'i32.ge_u': 'f64x2.ge', 'i32.gt_u': 'f64x2.gt',
|
|
3122
|
+
}
|
|
3123
|
+
const readsVar = (n, v) => isArr(n) && ((n[0] === 'local.get' && n[1] === v) || n.some(c => readsVar(c, v)))
|
|
3124
|
+
const writesName = (n, name) => isArr(n) && (((n[0] === 'local.set' || n[0] === 'local.tee' || n[0] === 'global.set') && n[1] === name) || n.some(c => writesName(c, name)))
|
|
3125
|
+
// Pixel induction variables may be i32 (const-bound loops) or f64 (param-bound loops,
|
|
3126
|
+
// e.g. `for (x=0; x<width; ++x)` with f64 `width`). Match `v += 1` and `v < bound` for both.
|
|
3127
|
+
const matchPixelInc = (stmt) => {
|
|
3128
|
+
if (!isArr(stmt) || stmt[0] !== 'local.set' || stmt.length !== 3) return null
|
|
3129
|
+
const x = stmt[1], v = stmt[2]
|
|
3130
|
+
if (!isArr(v) || v.length !== 3 || !isLocalGet(v[1], x)) return null
|
|
3131
|
+
if (v[0] === 'i32.add' && constNum(v[2]) === 1) return { name: x, type: 'i32' }
|
|
3132
|
+
if (v[0] === 'f64.add' && isArr(v[2]) && v[2][0] === 'f64.const' && Number(v[2][1]) === 1) return { name: x, type: 'f64' }
|
|
3133
|
+
return null
|
|
3134
|
+
}
|
|
3135
|
+
const matchPixelExit = (stmt, label) => {
|
|
3136
|
+
if (!isArr(stmt) || stmt[0] !== 'br_if' || stmt[1] !== label) return null
|
|
3137
|
+
const cond = stmt[2]
|
|
3138
|
+
if (!isArr(cond) || cond[0] !== 'i32.eqz') return null
|
|
3139
|
+
const cmp = cond[1]
|
|
3140
|
+
if (!isArr(cmp) || !isLocalGet(cmp[1])) return null
|
|
3141
|
+
if (cmp[0] === 'i32.lt_s' || cmp[0] === 'i32.lt_u') return { ind: cmp[1][1], bound: cmp[2], cmpOp: cmp[0], type: 'i32' }
|
|
3142
|
+
if (cmp[0] === 'f64.lt') return { ind: cmp[1][1], bound: cmp[2], cmpOp: cmp[0], type: 'f64' }
|
|
3143
|
+
return null
|
|
3144
|
+
}
|
|
3145
|
+
|
|
3146
|
+
/**
|
|
3147
|
+
* Match the OUTER per-pixel loop scaffold shared by tryDivergentEscapeVectorize
|
|
3148
|
+
* (inner escape loop) and tryPerPixelColor (straight-line body):
|
|
3149
|
+
* (block $o [preamble: pure local.set…]
|
|
3150
|
+
* (loop $l (br_if $o (i32.eqz (IV < WIDTH))) OBODY… (pxIV += 1)… (br $l)))
|
|
3151
|
+
* One-or-more trailing `v += 1` are pixel induction vars (the bound IV plus any
|
|
3152
|
+
* parallel counters like `j`); the exit bounds one of them by an invariant width.
|
|
3153
|
+
*
|
|
3154
|
+
* Returns the shared FACTS, or null:
|
|
3155
|
+
* { oLabel, loopNode, preamble, pixelIVs, pivStart, pxVar, widthBound, pivType,
|
|
3156
|
+
* obody } — obody = loopNode.slice(3, pivStart), the per-pixel work between
|
|
3157
|
+
* the exit guard and the IV bumps. Both consumers branch on `obody` afterward.
|
|
3158
|
+
* The bound is re-evaluated for the SIMD guard, so it must be invariant + pure:
|
|
3159
|
+
* a constant, or a local/global the loop nest never writes (`writesName`).
|
|
3160
|
+
*/
|
|
3161
|
+
function matchOuterPixelLoop(blockNode) {
|
|
3162
|
+
if (!isArr(blockNode) || blockNode[0] !== 'block') return null
|
|
3163
|
+
const oLabel = (typeof blockNode[1] === 'string' && blockNode[1].startsWith('$')) ? blockNode[1] : null
|
|
3164
|
+
if (!oLabel) return null
|
|
3165
|
+
let loopNode = null, loopPos = -1
|
|
3166
|
+
for (let i = 2; i < blockNode.length; i++) {
|
|
3167
|
+
const c = blockNode[i]
|
|
3168
|
+
if (!isArr(c)) return null
|
|
3169
|
+
if (c[0] === 'loop') { if (loopNode) return null; loopNode = c; loopPos = i }
|
|
3170
|
+
else if (loopNode) return null // statement after the loop → bail
|
|
3171
|
+
else if (c[0] !== 'local.set') return null // preamble must be pure local.set
|
|
3172
|
+
}
|
|
3173
|
+
if (!loopNode) return null
|
|
3174
|
+
const preamble = blockNode.slice(2, loopPos)
|
|
3175
|
+
const llabel = (typeof loopNode[1] === 'string' && loopNode[1].startsWith('$')) ? loopNode[1] : null
|
|
3176
|
+
if (!llabel) return null
|
|
3177
|
+
const oEnd = loopNode.length - 1
|
|
3178
|
+
if (!(isArr(loopNode[oEnd]) && loopNode[oEnd][0] === 'br' && loopNode[oEnd][1] === llabel)) return null
|
|
3179
|
+
const pixelIVs = [] // [{ name, type }]
|
|
3180
|
+
let pivStart = oEnd
|
|
3181
|
+
for (let i = oEnd - 1; i >= 3; i--) {
|
|
3182
|
+
const m = matchPixelInc(loopNode[i])
|
|
3183
|
+
if (!m) break
|
|
3184
|
+
pixelIVs.unshift(m); pivStart = i
|
|
3185
|
+
}
|
|
3186
|
+
if (!pixelIVs.length) return null
|
|
3187
|
+
const oExit = matchPixelExit(loopNode[2], oLabel)
|
|
3188
|
+
const pxIV = oExit && pixelIVs.find(p => p.name === oExit.ind && p.type === oExit.type)
|
|
3189
|
+
if (!pxIV) return null
|
|
3190
|
+
const widthBound = oExit.bound
|
|
3191
|
+
const pivType = new Map(pixelIVs.map(p => [p.name, p.type]))
|
|
3192
|
+
if (isI32Const(widthBound)) { /* ok */ }
|
|
3193
|
+
else if (isArr(widthBound) && (widthBound[0] === 'local.get' || widthBound[0] === 'global.get')) {
|
|
3194
|
+
if (writesName(loopNode, widthBound[1])) return null
|
|
3195
|
+
} else return null
|
|
3196
|
+
const obody = loopNode.slice(3, pivStart) // between exit guard and the pixel-IV bumps
|
|
3197
|
+
return { oLabel, loopNode, preamble, pixelIVs, pivStart, pxVar: oExit.ind, widthBound, pivType, obody, oExit }
|
|
3198
|
+
}
|
|
3199
|
+
|
|
3200
|
+
function tryDivergentEscapeVectorize(blockNode, fnLocals, freshIdRef) {
|
|
3201
|
+
// Outer per-pixel scaffold (+ LICM preamble feeding both SIMD path and tail).
|
|
3202
|
+
const outer = matchOuterPixelLoop(blockNode)
|
|
3203
|
+
if (!outer) return null
|
|
3204
|
+
const { oLabel, loopNode, preamble, pixelIVs, pxVar, widthBound, pivType, obody, oExit } = outer
|
|
3205
|
+
|
|
3206
|
+
let innerIdx = -1, innerBlock = null
|
|
3207
|
+
for (let i = 0; i < obody.length; i++) {
|
|
3208
|
+
const s = obody[i]
|
|
3209
|
+
if (isArr(s) && s[0] === 'block' && s.slice(1).some(c => isArr(c) && c[0] === 'loop')) {
|
|
3210
|
+
if (innerBlock) return null
|
|
3211
|
+
innerBlock = s; innerIdx = i
|
|
3212
|
+
}
|
|
3213
|
+
}
|
|
3214
|
+
if (!innerBlock) return null
|
|
3215
|
+
const iLabel = (typeof innerBlock[1] === 'string' && innerBlock[1].startsWith('$')) ? innerBlock[1] : null
|
|
3216
|
+
if (!iLabel || innerBlock.length < 3) return null
|
|
3217
|
+
const innerLoop = innerBlock[innerBlock.length - 1]
|
|
3218
|
+
if (!isArr(innerLoop) || innerLoop[0] !== 'loop') return null
|
|
3219
|
+
const ilLabel = (typeof innerLoop[1] === 'string' && innerLoop[1].startsWith('$')) ? innerLoop[1] : null
|
|
3220
|
+
if (!ilLabel) return null
|
|
3221
|
+
// The inner block may also carry LICM-hoisted invariants (e.g. BAILOUT) before the loop.
|
|
3222
|
+
// They must be pixel-pair-invariant (read only outer-invariants) so one copy feeds both
|
|
3223
|
+
// lanes; hoist them ahead of the SIMD loop.
|
|
3224
|
+
const innerPre = innerBlock.slice(2, innerBlock.length - 1)
|
|
3225
|
+
for (const s of innerPre) {
|
|
3226
|
+
if (!isArr(s) || s[0] !== 'local.set' || s.length !== 3) return null
|
|
3227
|
+
const inv = (n) => !isArr(n) || ((n[0] === 'local.get' || n[0] === 'global.get') ? !writesName(loopNode, n[1]) : n.every((c, i) => i === 0 || inv(c)))
|
|
3228
|
+
if (!inv(s[2])) return null
|
|
3229
|
+
}
|
|
3230
|
+
|
|
3231
|
+
// ---- inner escape loop: a top while-cond + body (f64 updates + one mid-break) + it++.
|
|
3232
|
+
// The two continue/break conditions are an `it < MAXIT` limit and a `|z|² vs T` escape,
|
|
3233
|
+
// in EITHER order (mandelbrot/ship: limit at top; example-mandelbrot: escape at top). ----
|
|
3234
|
+
const iEnd = innerLoop.length - 1
|
|
3235
|
+
if (!(isArr(innerLoop[iEnd]) && innerLoop[iEnd][0] === 'br' && innerLoop[iEnd][1] === ilLabel)) return null
|
|
3236
|
+
const itVar = matchInc1(innerLoop[iEnd - 1])
|
|
3237
|
+
if (!itVar || fnLocals.get(itVar) !== 'i32') return null
|
|
3238
|
+
|
|
3239
|
+
// A break statement (br_if, or `if BC (then (br iLabel))`) → its break-when condition.
|
|
3240
|
+
// breakCond: does this statement break the inner loop? Returns { cond, assigns } where
|
|
3241
|
+
// `assigns` is a list of { name, val } local.set stmts extracted from the then-block
|
|
3242
|
+
// BEFORE the final br. Existing single-br form → assigns=[]. Multi-stmt then form (Newton:
|
|
3243
|
+
// `if(cond)(then (local.set $root K)(br $iLabel))`) → assigns=[{name,val}].
|
|
3244
|
+
const breakCond = (s) => {
|
|
3245
|
+
if (!isArr(s)) return null
|
|
3246
|
+
if (s[0] === 'br_if' && s[1] === iLabel) return { cond: s[2], assigns: [] }
|
|
3247
|
+
if (s[0] !== 'if' || s.length !== 3 || !isArr(s[2]) || s[2][0] !== 'then') return null
|
|
3248
|
+
const then = s[2]
|
|
3249
|
+
// then-block must end with (br $iLabel); all preceding stmts must be local.set
|
|
3250
|
+
if (!isArr(then[then.length - 1]) || then[then.length - 1][0] !== 'br' || then[then.length - 1][1] !== iLabel) return null
|
|
3251
|
+
const assigns = []
|
|
3252
|
+
for (let k = 1; k < then.length - 1; k++) {
|
|
3253
|
+
const st = then[k]
|
|
3254
|
+
if (!isArr(st) || st[0] !== 'local.set' || st.length !== 3) return null
|
|
3255
|
+
assigns.push({ name: st[1], val: st[2] })
|
|
3256
|
+
}
|
|
3257
|
+
return { cond: s[1], assigns }
|
|
3258
|
+
}
|
|
3259
|
+
// keep (continue) condition vs the break-when condition. A while-guard `(i32.eqz CONT)`
|
|
3260
|
+
// keeps on CONT directly; a mid-break `if (X) break` keeps on ¬X. We must NOT lower ¬X by
|
|
3261
|
+
// flipping the comparison (f64.gt→f64.le): for NaN, scalar `NOT(NaN>T)` is TRUE but
|
|
3262
|
+
// `NaN<=T` is FALSE — they disagree, so an orbit that reaches NaN without escaping would
|
|
3263
|
+
// wrongly deactivate. Instead keep the DIRECT comparison and negate the lane mask with
|
|
3264
|
+
// v128.not (NaN-exact: gt is false on NaN, ¬false = keep, matching scalar).
|
|
3265
|
+
const keepOf = (bc) => {
|
|
3266
|
+
if (!isArr(bc)) return null
|
|
3267
|
+
if (bc[0] === 'i32.eqz' && isArr(bc[1]) && CMP_LANE[bc[1][0]]) return { cmp: bc[1], negate: false }
|
|
3268
|
+
if (CMP_NEG[bc[0]]) return { cmp: bc, negate: true }
|
|
3269
|
+
return null
|
|
3270
|
+
}
|
|
3271
|
+
const topBcR = breakCond(innerLoop[2])
|
|
3272
|
+
const topBc = topBcR?.cond // raw condition expr (backward-compat name for compound-top checks)
|
|
3273
|
+
let keepTop = topBcR && keepOf(topBcR.cond)
|
|
3274
|
+
const ibody = innerLoop.slice(3, iEnd - 1)
|
|
3275
|
+
let midIdx = -1, keepMid = null, compoundTop = false
|
|
3276
|
+
// Compound while-guard `while (it<MAX && |z|²<T)` → `eqz(and(A,B))`: two keep conditions, both
|
|
3277
|
+
// at the top (continue while A AND B). The Julia set is written this way. Split A,B into the two
|
|
3278
|
+
// keeps; the body is then pure updates (no mid-break).
|
|
3279
|
+
if (!keepTop && isArr(topBc) && topBc[0] === 'i32.eqz' && isArr(topBc[1]) && topBc[1][0] === 'i32.and'
|
|
3280
|
+
&& isArr(topBc[1][1]) && CMP_LANE[topBc[1][1][0]] && isArr(topBc[1][2]) && CMP_LANE[topBc[1][2][0]]) {
|
|
3281
|
+
keepTop = { cmp: topBc[1][1], negate: false }
|
|
3282
|
+
keepMid = { cmp: topBc[1][2], negate: false }
|
|
3283
|
+
compoundTop = true
|
|
3284
|
+
}
|
|
3285
|
+
if (!keepTop) return null
|
|
3286
|
+
// Collect mid-breaks. The existing single-break path requires exactly one.
|
|
3287
|
+
// The new multi-break path (Newton-style convergence) allows several outcome breaks
|
|
3288
|
+
// — each `if(COND)(then (local.set $X K)(br $iLabel))` — when the top guard is the
|
|
3289
|
+
// sole limit condition and ALL mid-breaks are escape-kind with i32 outcome assigns.
|
|
3290
|
+
const midBreaks = [] // [{ idx, keep, assigns }]
|
|
3291
|
+
if (compoundTop) {
|
|
3292
|
+
for (let i = 0; i < ibody.length; i++)
|
|
3293
|
+
if (!(isArr(ibody[i]) && ibody[i][0] === 'local.set' && ibody[i].length === 3 && fnLocals.get(ibody[i][1]) === 'f64')) return null
|
|
3294
|
+
} else {
|
|
3295
|
+
for (let i = 0; i < ibody.length; i++) {
|
|
3296
|
+
const bcR = breakCond(ibody[i])
|
|
3297
|
+
if (bcR) {
|
|
3298
|
+
const k = keepOf(bcR.cond)
|
|
3299
|
+
if (!k) return null
|
|
3300
|
+
midBreaks.push({ idx: i, keep: k, assigns: bcR.assigns })
|
|
3301
|
+
} else if (!(isArr(ibody[i]) && ibody[i][0] === 'local.set' && ibody[i].length === 3 && fnLocals.get(ibody[i][1]) === 'f64')) return null
|
|
3302
|
+
}
|
|
3303
|
+
if (midBreaks.length === 0) return null
|
|
3304
|
+
if (midBreaks.length === 1) {
|
|
3305
|
+
// single-break: preserve exact existing behaviour
|
|
3306
|
+
midIdx = midBreaks[0].idx; keepMid = midBreaks[0].keep
|
|
3307
|
+
}
|
|
3308
|
+
// multi-break: handled below after classification
|
|
3309
|
+
}
|
|
3310
|
+
|
|
3311
|
+
// classify f64 vars written across [top-guard, …body…]: loop-carried (first access a READ)
|
|
3312
|
+
// vs within-iteration temp (first access a WRITE — including squares tee'd in the guard).
|
|
3313
|
+
const seq = [innerLoop[2], ...ibody]
|
|
3314
|
+
const written = new Set()
|
|
3315
|
+
const collectW = (n) => { if (!isArr(n)) return; if ((n[0] === 'local.set' || n[0] === 'local.tee') && fnLocals.get(n[1]) === 'f64') written.add(n[1]); for (let i = 1; i < n.length; i++) collectW(n[i]) }
|
|
3316
|
+
seq.forEach(collectW)
|
|
3317
|
+
const carried = new Set(), temp = new Set(), seen = new Set()
|
|
3318
|
+
const classify = (n) => {
|
|
3319
|
+
if (!isArr(n)) return
|
|
3320
|
+
if (n[0] === 'local.get') { const v = n[1]; if (written.has(v) && !seen.has(v)) { seen.add(v); carried.add(v) }; return }
|
|
3321
|
+
if (n[0] === 'local.set' || n[0] === 'local.tee') { classify(n[2]); const v = n[1]; if (written.has(v) && !seen.has(v)) { seen.add(v); temp.add(v) }; return }
|
|
3322
|
+
for (let i = 1; i < n.length; i++) classify(n[i])
|
|
3323
|
+
}
|
|
3324
|
+
seq.forEach(classify)
|
|
3325
|
+
|
|
3326
|
+
// hoist tees out of the keep conditions into explicit temp computations
|
|
3327
|
+
const tees = []
|
|
3328
|
+
const detee = (n) => isArr(n) ? (n[0] === 'local.tee' ? (tees.push({ tgt: n[1], expr: detee(n[2]) }), ['local.get', n[1]]) : n.map(detee)) : n
|
|
3329
|
+
const keepTopC = { cmp: [keepTop.cmp[0], detee(keepTop.cmp[1]), detee(keepTop.cmp[2])], negate: keepTop.negate }
|
|
3330
|
+
const teeTop = tees.length
|
|
3331
|
+
const keepMidC = keepMid ? { cmp: [keepMid.cmp[0], detee(keepMid.cmp[1]), detee(keepMid.cmp[2])], negate: keepMid.negate } : null
|
|
3332
|
+
|
|
3333
|
+
// each keep is an it-limit (mentions `it` directly or via convert) or a z-escape; need one of each
|
|
3334
|
+
const isIt = (n) => isLocalGet(n, itVar) || (isArr(n) && n[0] === 'f64.convert_i32_s' && isLocalGet(n[1], itVar))
|
|
3335
|
+
const kindOf = (k) => (isIt(k.cmp[1]) || isIt(k.cmp[2])) ? 'limit' : 'escape'
|
|
3336
|
+
const kTop = kindOf(keepTopC)
|
|
3337
|
+
const limitKeep = keepTopC // initialised here; may be reassigned below for single-break
|
|
3338
|
+
let kMid = null, boundExpr, boundI32, itLeft
|
|
3339
|
+
if (midBreaks.length === 1) {
|
|
3340
|
+
kMid = kindOf(keepMidC)
|
|
3341
|
+
if (kTop === kMid) return null
|
|
3342
|
+
const lk = kTop === 'limit' ? keepTopC : keepMidC
|
|
3343
|
+
itLeft = isIt(lk.cmp[1])
|
|
3344
|
+
boundExpr = itLeft ? lk.cmp[2] : lk.cmp[1]
|
|
3345
|
+
boundI32 = lk.cmp[0].startsWith('i32.')
|
|
3346
|
+
} else if (midBreaks.length > 1) {
|
|
3347
|
+
// Multi-break path: top must be the sole limit; all mid-breaks must be escape-kind.
|
|
3348
|
+
if (kTop !== 'limit') return null
|
|
3349
|
+
for (const mb of midBreaks) if (kindOf(mb.keep) !== 'escape') return null
|
|
3350
|
+
itLeft = isIt(keepTopC.cmp[1])
|
|
3351
|
+
boundExpr = itLeft ? keepTopC.cmp[2] : keepTopC.cmp[1]
|
|
3352
|
+
boundI32 = keepTopC.cmp[0].startsWith('i32.')
|
|
3353
|
+
}
|
|
3354
|
+
// limit bound must be loop-invariant (splatted once per pair): no carried/temp ref, not written
|
|
3355
|
+
for (const v of [...carried, ...temp]) if (readsVar(boundExpr, v)) return null
|
|
3356
|
+
const boundInvariant = (n) => !isArr(n) || ((n[0] !== 'local.get' && n[0] !== 'global.get') || !writesName(loopNode, n[1])) && n.every((c, i) => i === 0 || boundInvariant(c))
|
|
3357
|
+
if (!boundInvariant(boundExpr)) return null
|
|
3358
|
+
|
|
3359
|
+
// c-vars: f64 locals read in the lifted exprs but never written there (per-pixel/invariant inputs).
|
|
3360
|
+
// Also allow loop-invariant global.get (e.g. module constants like R3 in newton) — splatted inline.
|
|
3361
|
+
const cVars = new Set()
|
|
3362
|
+
const liftable = (n) => {
|
|
3363
|
+
if (!isArr(n)) return false
|
|
3364
|
+
if (n[0] === 'local.get') {
|
|
3365
|
+
const v = n[1]
|
|
3366
|
+
if (v !== itVar && !carried.has(v) && !temp.has(v)) { if (fnLocals.get(v) !== 'f64') return false; cVars.add(v) }
|
|
3367
|
+
return true
|
|
3368
|
+
}
|
|
3369
|
+
if (n[0] === 'f64.const') return true
|
|
3370
|
+
if (n[0] === 'global.get') return !writesName(loopNode, n[1]) // loop-invariant global → splat
|
|
3371
|
+
if (LANE_PURE.f64.has(n[0])) { for (let i = 1; i < n.length; i++) if (!liftable(n[i])) return false; return true }
|
|
3372
|
+
return false
|
|
3373
|
+
}
|
|
3374
|
+
const midIdxSet = new Set(midBreaks.map(mb => mb.idx))
|
|
3375
|
+
for (const t of tees) if (!liftable(t.expr)) return null
|
|
3376
|
+
for (let i = 0; i < ibody.length; i++) if (!midIdxSet.has(i) && !liftable(ibody[i][2])) return null
|
|
3377
|
+
if (midBreaks.length === 1) {
|
|
3378
|
+
const escapeKeep = kTop === 'escape' ? keepTopC : keepMidC
|
|
3379
|
+
if (!liftable(escapeKeep.cmp[1]) || !liftable(escapeKeep.cmp[2])) return null
|
|
3380
|
+
} else {
|
|
3381
|
+
// multi-break: each break's escape condition arguments must be liftable
|
|
3382
|
+
for (const mb of midBreaks) {
|
|
3383
|
+
const mbC = { cmp: [mb.keep.cmp[0], detee(mb.keep.cmp[1]), detee(mb.keep.cmp[2])], negate: mb.keep.negate }
|
|
3384
|
+
mb.keepC = mbC // store deteed keepC on mb for use in emit
|
|
3385
|
+
if (!liftable(mbC.cmp[1]) || !liftable(mbC.cmp[2])) return null
|
|
3386
|
+
}
|
|
3387
|
+
}
|
|
3388
|
+
|
|
3389
|
+
// ---- pre-inner inits + epilogue ----
|
|
3390
|
+
// i32 outcome variables assigned by multi-break outcome stmts (e.g. $root in Newton).
|
|
3391
|
+
// Their pre-inner `local.set $root 0` default-init is valid and must not be rejected.
|
|
3392
|
+
const outcomeVarSetEarly = new Set(midBreaks.flatMap(mb => mb.assigns.map(a => a.name)))
|
|
3393
|
+
const carriedInit = new Map() // carried var → its f64.const seed (before the loop)
|
|
3394
|
+
const perPxInit = new Map() // c-var → its per-pixel init expr
|
|
3395
|
+
for (let i = 0; i < innerIdx; i++) {
|
|
3396
|
+
const s = obody[i]
|
|
3397
|
+
if (!isArr(s) || s[0] !== 'local.set' || s.length !== 3) return null
|
|
3398
|
+
const tgt = s[1]
|
|
3399
|
+
if (carried.has(tgt)) {
|
|
3400
|
+
// z₀ seed: a constant (mandelbrot/burning-ship z₀=0) OR a per-pixel expr (Julia set, where
|
|
3401
|
+
// z₀ = the pixel and c is constant — the dual of mandelbrot). liftCLane handles both at emit.
|
|
3402
|
+
carriedInit.set(tgt, s[2])
|
|
3403
|
+
} else if (temp.has(tgt)) { /* recomputed each iteration — init ignored */ }
|
|
3404
|
+
else if (tgt === itVar) { if (constNum(s[2]) !== 0) return null }
|
|
3405
|
+
else if (cVars.has(tgt)) perPxInit.set(tgt, s[2])
|
|
3406
|
+
else if (outcomeVarSetEarly.has(tgt)) { /* i32 outcome var default init — ignored, handled per-lane */ }
|
|
3407
|
+
else return null
|
|
3408
|
+
}
|
|
3409
|
+
for (const c of carried) if (!carriedInit.has(c)) return null
|
|
3410
|
+
const epilogue = obody.slice(innerIdx + 1)
|
|
3411
|
+
// The epilogue runs scalar per lane; it may only read carried/it/pixel-IV/invariant
|
|
3412
|
+
// values (each statement's reads, before that statement's writes). A read of an
|
|
3413
|
+
// inner-loop temp or a per-pixel c-var has no post-loop per-lane value → bail.
|
|
3414
|
+
{
|
|
3415
|
+
const written = new Set()
|
|
3416
|
+
for (const s of epilogue) {
|
|
3417
|
+
const r = new Set(); (function rd(n){ if(!isArr(n)) return; if(n[0]==='local.get') r.add(n[1]); else for(const c of n) rd(c) })(s)
|
|
3418
|
+
for (const v of r) if (!written.has(v) && (temp.has(v) || perPxInit.has(v))) return null
|
|
3419
|
+
;(function wr(n){ if(!isArr(n)) return; if((n[0]==='local.set'||n[0]==='local.tee')&&typeof n[1]==='string') written.add(n[1]); for(const c of n.slice(2)) wr(c) })(s)
|
|
3420
|
+
}
|
|
3421
|
+
if (!epilogue.length) return null
|
|
3422
|
+
}
|
|
3423
|
+
|
|
3424
|
+
// ============================ emit ============================
|
|
3425
|
+
const id = freshIdRef.next++
|
|
3426
|
+
const nm = (s) => `$__esc${id}_${s}`
|
|
3427
|
+
const shadow = new Map() // carried/temp f64 var → v128 shadow
|
|
3428
|
+
for (const v of [...carried, ...temp]) shadow.set(v, nm('z' + v.replace(/\W/g, '')))
|
|
3429
|
+
const cLane = new Map()
|
|
3430
|
+
for (const cv of cVars) cLane.set(cv, nm('c' + cv.replace(/\W/g, '')))
|
|
3431
|
+
const iterV = nm('iter'), activeV = nm('act'), maxitLane = nm('lim')
|
|
3432
|
+
const newLocalDecls = []
|
|
3433
|
+
for (const n of [...shadow.values(), ...cLane.values()]) newLocalDecls.push(['local', n, 'v128'])
|
|
3434
|
+
|
|
3435
|
+
const lift = (n) => {
|
|
3436
|
+
if (n[0] === 'local.get') return ['local.get', shadow.has(n[1]) ? shadow.get(n[1]) : cLane.get(n[1])]
|
|
3437
|
+
if (n[0] === 'f64.const') return ['f64x2.splat', n]
|
|
3438
|
+
if (n[0] === 'global.get') return ['f64x2.splat', n] // loop-invariant module global (e.g. R3)
|
|
3439
|
+
return [LANE_PURE.f64.get(n[0]).simd, ...n.slice(1).map(lift)]
|
|
3440
|
+
}
|
|
3441
|
+
const bump = (n, k) => k === 0 ? n // substitute every pixel-IV with (IV + k), in its type
|
|
3442
|
+
: (isArr(n) && n[0] === 'local.get' && pivType.has(n[1]))
|
|
3443
|
+
? [pivType.get(n[1]) + '.add', n, [pivType.get(n[1]) + '.const', k]]
|
|
3444
|
+
: (isArr(n) ? n.map(c => bump(c, k)) : n)
|
|
3445
|
+
|
|
3446
|
+
// Build a per-pixel c-var's two lanes by lifting its init to f64x2: the pixel IV becomes the
|
|
3447
|
+
// ramp [v, v+1], a dependency on another c-var resolves to that c-var's already-built lane
|
|
3448
|
+
// (so chains like `bail = 4 + cx*cx` get the right per-lane value — NOT the px=0 value from a
|
|
3449
|
+
// bump that can't follow `cx`), and everything else (grid constants) splats. Returns null if
|
|
3450
|
+
// the init reads inner-loop state or an op we can't lift, in which case we decline.
|
|
3451
|
+
const liftCLane = (n) => {
|
|
3452
|
+
if (!isArr(n)) return null
|
|
3453
|
+
if (n[0] === 'local.get' || n[0] === 'global.get') {
|
|
3454
|
+
const v = n[1]
|
|
3455
|
+
if (n[0] === 'local.get') {
|
|
3456
|
+
if (cLane.has(v)) return ['local.get', cLane.get(v)] // another c-var's lane
|
|
3457
|
+
if (carried.has(v) || temp.has(v)) return null // inner-loop state — must not appear
|
|
3458
|
+
if (pivType.get(v) === 'f64') return ['f64x2.replace_lane', 1, ['f64x2.splat', n], ['f64.add', n, ['f64.const', 1]]]
|
|
3459
|
+
}
|
|
3460
|
+
if (writesName(loopNode, v)) return null // a per-pixel value we can't ramp → decline
|
|
3461
|
+
return ['f64x2.splat', n] // loop-invariant local/global
|
|
3462
|
+
}
|
|
3463
|
+
if (n[0] === 'f64.const') return ['f64x2.splat', n]
|
|
3464
|
+
if (n[0] === 'f64.convert_i32_s' && isArr(n[1]) && n[1][0] === 'local.get' && pivType.get(n[1][1]) === 'i32') {
|
|
3465
|
+
const piv = n[1]
|
|
3466
|
+
return ['f64x2.replace_lane', 1, ['f64x2.splat', ['f64.convert_i32_s', piv]], ['f64.convert_i32_s', ['i32.add', piv, ['i32.const', 1]]]]
|
|
3467
|
+
}
|
|
3468
|
+
if (LANE_PURE.f64.has(n[0])) { const ks = n.slice(1).map(liftCLane); return ks.some(k => k === null) ? null : [LANE_PURE.f64.get(n[0]).simd, ...ks] }
|
|
3469
|
+
return null
|
|
3470
|
+
}
|
|
3471
|
+
// c-lanes in dependency order: invariants splat first, then per-pixel inits in source (obody) order
|
|
3472
|
+
const cOrder = [...cVars].filter(cv => !perPxInit.has(cv))
|
|
3473
|
+
for (let i = 0; i < innerIdx; i++) { const s = obody[i]; if (isArr(s) && s[0] === 'local.set' && perPxInit.has(s[1])) cOrder.push(s[1]) }
|
|
3474
|
+
const pre = []
|
|
3475
|
+
for (const cv of cOrder) {
|
|
3476
|
+
if (perPxInit.has(cv)) {
|
|
3477
|
+
const lane = liftCLane(perPxInit.get(cv))
|
|
3478
|
+
if (!lane) return null
|
|
3479
|
+
pre.push(['local.set', cLane.get(cv), lane])
|
|
3480
|
+
} else pre.push(['local.set', cLane.get(cv), ['f64x2.splat', ['local.get', cv]]])
|
|
3481
|
+
}
|
|
3482
|
+
// Seed each carried lane in dependency order. Seeds reading only non-carried values (z₀ = a
|
|
3483
|
+
// const, or a per-pixel ramp for the Julia set) build their lanes via liftCLane. Seeds reading
|
|
3484
|
+
// a carried var — the cached squares `zx2 = zx*zx` in Julia's compound guard — lift through the
|
|
3485
|
+
// already-built shadow lanes instead.
|
|
3486
|
+
const seedDeps = (e) => [...carried].some(c => readsVar(e, c))
|
|
3487
|
+
const depLiftable = (n) => !isArr(n) ? false
|
|
3488
|
+
: n[0] === 'local.get' ? shadow.has(n[1])
|
|
3489
|
+
: n[0] === 'f64.const' ? true
|
|
3490
|
+
: (LANE_PURE.f64.has(n[0]) && n.slice(1).every(depLiftable))
|
|
3491
|
+
for (const v of carried) if (!seedDeps(carriedInit.get(v))) {
|
|
3492
|
+
const lane = liftCLane(carriedInit.get(v))
|
|
3493
|
+
if (!lane) return null
|
|
3494
|
+
pre.push(['local.set', shadow.get(v), lane])
|
|
3495
|
+
}
|
|
3496
|
+
for (const v of carried) if (seedDeps(carriedInit.get(v))) {
|
|
3497
|
+
if (!depLiftable(carriedInit.get(v))) return null
|
|
3498
|
+
pre.push(['local.set', shadow.get(v), lift(carriedInit.get(v))])
|
|
3499
|
+
}
|
|
3500
|
+
const teeStmts = (range) => range.map(t => ['local.set', shadow.get(t.tgt), lift(t.expr)])
|
|
3501
|
+
const sIn = nm('ib'), sIl = nm('il'), sOut = nm('ob'), sOl = nm('ol')
|
|
3502
|
+
|
|
3503
|
+
// FAST PATH — the common escape-time shape `while (it<MAX){ …updates…; if (|z|²>T) break }`:
|
|
3504
|
+
// break the pair the instant the FIRST lane escapes (no per-iteration freeze/mask), keep `it`
|
|
3505
|
+
// a scalar i32, raw f64x2 updates. A short scalar tail then finishes whichever lane lagged (≈0
|
|
3506
|
+
// iterations when the pair is coherent, which adjacent pixels overwhelmingly are). Inside-set
|
|
3507
|
+
// pixels (both lanes to MAX) run clean 2× with zero mask overhead — exactly where the masked
|
|
3508
|
+
// loop bled its speedup. Other shapes (escape-at-top, body after the break) take the masked path.
|
|
3509
|
+
// escape-at-MID: `…updates…; if (|z|²>T) break` (burning-ship). escape-at-TOP: the escape gates
|
|
3510
|
+
// the loop head — a `while (|z|²<T)` (mandelbrot) or the second half of a compound `while (it<MAX
|
|
3511
|
+
// && |z|²<T)` (Julia). Both take the fast path; only the tail's `it` resume point differs.
|
|
3512
|
+
const escAtMid = !compoundTop && kMid === 'escape' && midIdx === ibody.length - 1
|
|
3513
|
+
const escAtTop = compoundTop || kTop === 'escape'
|
|
3514
|
+
const fastPath = escAtMid || escAtTop
|
|
3515
|
+
let simdInner, epiLane, postLoop = []
|
|
3516
|
+
|
|
3517
|
+
if (fastPath) {
|
|
3518
|
+
const shIt = nm('shit'), escF = nm('escf')
|
|
3519
|
+
newLocalDecls.push(['local', shIt, 'i32'], ['local', escF, 'i32'])
|
|
3520
|
+
pre.push(['local.set', itVar, ['i32.const', 0]], ['local.set', escF, ['i32.const', 0]])
|
|
3521
|
+
const limOf = (keepC) => keepC.negate ? [CMP_NEG[keepC.cmp[0]], keepC.cmp[1], keepC.cmp[2]] : keepC.cmp
|
|
3522
|
+
// emit a keep at its source position: a LIMIT (it is shared) → a scalar i32 guard; an ESCAPE
|
|
3523
|
+
// → an any_true break. `escF` records whether the loop exited via an escape (vs the limit) —
|
|
3524
|
+
// they can BOTH land at it=MAX (escape-at-top fires before the limit-at-mid's final update),
|
|
3525
|
+
// so `it < MAX` alone can't tell them apart; the tail must run only on an escape exit.
|
|
3526
|
+
const fastKeep = (keepC, kind, teeRange) => {
|
|
3527
|
+
const out = teeStmts(teeRange)
|
|
3528
|
+
if (kind === 'limit') out.push(['br_if', sIn, ['i32.eqz', limOf(keepC)]])
|
|
3529
|
+
else {
|
|
3530
|
+
const escL = [CMP_LANE[keepC.cmp[0]], lift(keepC.cmp[1]), lift(keepC.cmp[2])]
|
|
3531
|
+
out.push(['local.set', escF, ['i32.const', 1]])
|
|
3532
|
+
out.push(['br_if', sIn, ['v128.any_true', keepC.negate ? escL : ['v128.not', escL]]])
|
|
3533
|
+
out.push(['local.set', escF, ['i32.const', 0]])
|
|
3534
|
+
}
|
|
3535
|
+
return out
|
|
3536
|
+
}
|
|
3537
|
+
const sbody = [...fastKeep(keepTopC, kTop, tees.slice(0, teeTop))]
|
|
3538
|
+
if (compoundTop) sbody.push(...fastKeep(keepMidC, kMid, tees.slice(teeTop))) // second top keep
|
|
3539
|
+
for (let i = 0; i < ibody.length; i++) {
|
|
3540
|
+
if (!compoundTop && i === midIdx) sbody.push(...fastKeep(keepMidC, kMid, tees.slice(teeTop)))
|
|
3541
|
+
else sbody.push(['local.set', shadow.get(ibody[i][1]), lift(ibody[i][2])]) // raw update, no freeze
|
|
3542
|
+
}
|
|
3543
|
+
sbody.push(['local.set', itVar, ['i32.add', ['local.get', itVar], ['i32.const', 1]]])
|
|
3544
|
+
simdInner = ['block', sIn, ['loop', sIl, ...sbody, ['br', sIl]]]
|
|
3545
|
+
postLoop = [['local.set', shIt, ['local.get', itVar]]] // capture the break `it` AFTER the block exits
|
|
3546
|
+
|
|
3547
|
+
// a fresh-labelled copy of the inner block, to finish a lagging lane's remaining iterations
|
|
3548
|
+
let copyN = 0
|
|
3549
|
+
const relabelInner = () => {
|
|
3550
|
+
const ni = nm('tb' + copyN), nl = nm('tl' + copyN); copyN++
|
|
3551
|
+
const rl = (n) => !isArr(n) ? n
|
|
3552
|
+
: ((n[0] === 'block' || n[0] === 'loop' || n[0] === 'br' || n[0] === 'br_if') && (n[1] === iLabel || n[1] === ilLabel))
|
|
3553
|
+
? [n[0], n[1] === iLabel ? ni : nl, ...n.slice(2).map(rl)]
|
|
3554
|
+
: n.map(rl)
|
|
3555
|
+
return rl(innerBlock)
|
|
3556
|
+
}
|
|
3557
|
+
const escKeepC = compoundTop ? keepMidC : (kTop === 'escape' ? keepTopC : keepMidC)
|
|
3558
|
+
const escTees = (kTop === 'escape' ? tees.slice(0, teeTop) : tees.slice(teeTop)).map(t => ['local.set', t.tgt, t.expr])
|
|
3559
|
+
const notEsc = escKeepC.negate ? ['i32.eqz', escKeepC.cmp] : escKeepC.cmp
|
|
3560
|
+
epiLane = (k) => {
|
|
3561
|
+
const out = []
|
|
3562
|
+
// Extract carried AND temp lanes: the escape compare may read a temp (the optimizer
|
|
3563
|
+
// copy-propagates `x = xt` so `x*x` becomes `xt*xt`); the skip test below needs it.
|
|
3564
|
+
for (const v of [...carried, ...temp]) out.push(['local.set', v, ['f64x2.extract_lane', k, ['local.get', shadow.get(v)]]])
|
|
3565
|
+
for (let i = 0; i < innerIdx; i++) { const s = obody[i]; if (isArr(s) && s[0] === 'local.set' && perPxInit.has(s[1])) out.push(bump(s, k)) }
|
|
3566
|
+
out.push(['local.set', itVar, ['local.get', shIt]])
|
|
3567
|
+
out.push(['if', ['local.get', escF], ['then', // exited via escape (not the limit)…
|
|
3568
|
+
...escTees,
|
|
3569
|
+
['if', notEsc, ['then', // …and THIS lane hadn't escaped → finish it
|
|
3570
|
+
// escape-at-MID already ran this iteration's update, so step past it before resuming;
|
|
3571
|
+
// escape-at-TOP tests before the update, so resume at the same it.
|
|
3572
|
+
...(escAtMid ? [['local.set', itVar, ['i32.add', ['local.get', itVar], ['i32.const', 1]]]] : []),
|
|
3573
|
+
relabelInner()]]]])
|
|
3574
|
+
for (const s of epilogue) out.push(bump(s, k))
|
|
3575
|
+
return out
|
|
3576
|
+
}
|
|
3577
|
+
} else if (midBreaks.length > 1) {
|
|
3578
|
+
// ---- MULTI-OUTCOME masked path (Newton-style convergence loops) ----
|
|
3579
|
+
// Each mid-break `if(COND)(then (local.set $X K)(br))` converges a subset of lanes per
|
|
3580
|
+
// iteration. We track which lanes are still running in `activeV` and per-lane outcomes
|
|
3581
|
+
// in i32x4 shadow vectors (one per distinct i32 outcome variable). The break mask for
|
|
3582
|
+
// each convergence condition is: v128.and(activeV, breakMaskOf(keepC)).
|
|
3583
|
+
// Bit-exactness: i32x4 layout has 4 lanes of 32 bits. f64x2 has 2 lanes of 64 bits.
|
|
3584
|
+
// For f64 lane k, i32 lane 2*k carries the outcome (lower 32 bits of the f64 lane).
|
|
3585
|
+
// i32x4.splat(K) fills all 4 i32 lanes with K; v128.bitselect selects at bit level, so
|
|
3586
|
+
// for a converged f64 lane (bits 64k…64k+63 = -1 in the mask) both i32 halves get K.
|
|
3587
|
+
// i32x4.extract_lane(2*k, outcomeVec) recovers the scalar outcome per lane. ✓
|
|
3588
|
+
|
|
3589
|
+
// Collect distinct i32 outcome variables across all mid-breaks.
|
|
3590
|
+
const outcomeVars = [] // distinct i32 local names that get assigned in outcome breaks
|
|
3591
|
+
const outcomeVarSet = new Set()
|
|
3592
|
+
for (const mb of midBreaks) {
|
|
3593
|
+
for (const a of mb.assigns) {
|
|
3594
|
+
// itVar is tracked by iterV (f64x2) directly — not as an i32x4 outcome shadow.
|
|
3595
|
+
if (a.name !== itVar && !outcomeVarSet.has(a.name)) { outcomeVarSet.add(a.name); outcomeVars.push(a.name) }
|
|
3596
|
+
}
|
|
3597
|
+
}
|
|
3598
|
+
// Each outcome var gets an i32x4 shadow vector (initial value 0 = "no convergence" default).
|
|
3599
|
+
const outcomeVec = new Map()
|
|
3600
|
+
for (const v of outcomeVars) { const sv = nm('out' + v.replace(/\W/g, '')); outcomeVec.set(v, sv); newLocalDecls.push(['local', sv, 'v128']) }
|
|
3601
|
+
|
|
3602
|
+
newLocalDecls.push(['local', iterV, 'v128'], ['local', activeV, 'v128'], ['local', maxitLane, 'v128'])
|
|
3603
|
+
pre.push(['local.set', iterV, ['f64x2.splat', ['f64.const', 0]]])
|
|
3604
|
+
pre.push(['local.set', activeV, ['v128.const', 'i32x4', '-1', '-1', '-1', '-1']])
|
|
3605
|
+
pre.push(['local.set', maxitLane, ['f64x2.splat', boundI32 ? ['f64.convert_i32_s', boundExpr] : boundExpr]])
|
|
3606
|
+
for (const sv of outcomeVec.values()) pre.push(['local.set', sv, ['v128.const', 'i32x4', '0', '0', '0', '0']])
|
|
3607
|
+
|
|
3608
|
+
// keepMask for the limit top-guard (iter < MAXIT)
|
|
3609
|
+
const keepMask = (keep, isLimit) => {
|
|
3610
|
+
const m = isLimit
|
|
3611
|
+
? [CMP_LANE[keep.cmp[0]], itLeft ? ['local.get', iterV] : ['local.get', maxitLane], itLeft ? ['local.get', maxitLane] : ['local.get', iterV]]
|
|
3612
|
+
: [CMP_LANE[keep.cmp[0]], lift(keep.cmp[1]), lift(keep.cmp[2])]
|
|
3613
|
+
return keep.negate ? ['v128.not', m] : m
|
|
3614
|
+
}
|
|
3615
|
+
// breakMaskOf: v128 where bits = -1 for lanes whose break condition is NOW true
|
|
3616
|
+
// (opposite of keepMask: keep.negate=true → break is the positive compare).
|
|
3617
|
+
const breakMaskOf = (keepC) => {
|
|
3618
|
+
const m = [CMP_LANE[keepC.cmp[0]], lift(keepC.cmp[1]), lift(keepC.cmp[2])]
|
|
3619
|
+
return keepC.negate ? m : ['v128.not', m]
|
|
3620
|
+
}
|
|
3621
|
+
|
|
3622
|
+
const sbody = [
|
|
3623
|
+
...teeStmts(tees.slice(0, teeTop)),
|
|
3624
|
+
['local.set', activeV, ['v128.and', ['local.get', activeV], keepMask(keepTopC, true)]],
|
|
3625
|
+
['br_if', sIn, ['i32.eqz', ['v128.any_true', ['local.get', activeV]]]],
|
|
3626
|
+
]
|
|
3627
|
+
for (let i = 0; i < ibody.length; i++) {
|
|
3628
|
+
const mb = midBreaks.find(m => m.idx === i)
|
|
3629
|
+
if (mb) {
|
|
3630
|
+
// Convergence break: compute which lanes converge on THIS step.
|
|
3631
|
+
const bm = breakMaskOf(mb.keepC)
|
|
3632
|
+
const convV = nm('conv' + i)
|
|
3633
|
+
newLocalDecls.push(['local', convV, 'v128'])
|
|
3634
|
+
sbody.push(['local.set', convV, ['v128.and', ['local.get', activeV], bm]])
|
|
3635
|
+
// Apply each outcome assignment via bitselect on converged-this-step mask.
|
|
3636
|
+
for (const a of mb.assigns) {
|
|
3637
|
+
if (a.name === itVar) {
|
|
3638
|
+
// Assignment to the iteration counter (e.g. it=MAXIT before break): freeze iterV.
|
|
3639
|
+
sbody.push(['local.set', iterV, ['v128.bitselect', ['local.get', maxitLane], ['local.get', iterV], ['local.get', convV]]])
|
|
3640
|
+
} else if (outcomeVec.has(a.name)) {
|
|
3641
|
+
// i32 outcome variable (e.g. root=1/2/3): bitselect into its i32x4 shadow.
|
|
3642
|
+
// The scalar value K must be an i32 constant — splat it across all i32x4 lanes.
|
|
3643
|
+
if (!isArr(a.val) || a.val[0] !== 'i32.const') return null
|
|
3644
|
+
const sv = outcomeVec.get(a.name)
|
|
3645
|
+
sbody.push(['local.set', sv, ['v128.bitselect', ['i32x4.splat', a.val], ['local.get', sv], ['local.get', convV]]])
|
|
3646
|
+
} else return null // unexpected assign target
|
|
3647
|
+
}
|
|
3648
|
+
// Remove newly-converged lanes from active set.
|
|
3649
|
+
sbody.push(['local.set', activeV, ['v128.andnot', ['local.get', activeV], ['local.get', convV]]])
|
|
3650
|
+
} else {
|
|
3651
|
+
const v = ibody[i][1]
|
|
3652
|
+
if (carried.has(v)) sbody.push(['local.set', shadow.get(v), ['v128.bitselect', lift(ibody[i][2]), ['local.get', shadow.get(v)], ['local.get', activeV]]])
|
|
3653
|
+
else sbody.push(['local.set', shadow.get(v), lift(ibody[i][2])])
|
|
3654
|
+
}
|
|
3655
|
+
}
|
|
3656
|
+
sbody.push(['local.set', iterV, ['v128.bitselect', ['f64x2.add', ['local.get', iterV], ['f64x2.splat', ['f64.const', 1]]], ['local.get', iterV], ['local.get', activeV]]])
|
|
3657
|
+
simdInner = ['block', sIn, ['loop', sIl, ...sbody, ['br', sIl]]]
|
|
3658
|
+
const carriedInEpi = [...carried].filter(v => epilogue.some(s => readsVar(s, v)))
|
|
3659
|
+
epiLane = (k) => {
|
|
3660
|
+
const out = []
|
|
3661
|
+
for (const v of carriedInEpi) out.push(['local.set', v, ['f64x2.extract_lane', k, ['local.get', shadow.get(v)]]])
|
|
3662
|
+
out.push(['local.set', itVar, ['i32.trunc_f64_s', ['f64x2.extract_lane', k, ['local.get', iterV]]]])
|
|
3663
|
+
// Extract per-lane outcome variables from their i32x4 shadows (i32 lane = 2*k).
|
|
3664
|
+
for (const [v, sv] of outcomeVec) out.push(['local.set', v, ['i32x4.extract_lane', 2 * k, ['local.get', sv]]])
|
|
3665
|
+
for (const s of epilogue) out.push(bump(s, k))
|
|
3666
|
+
return out
|
|
3667
|
+
}
|
|
3668
|
+
} else {
|
|
3669
|
+
newLocalDecls.push(['local', iterV, 'v128'], ['local', activeV, 'v128'], ['local', maxitLane, 'v128'])
|
|
3670
|
+
pre.push(['local.set', iterV, ['f64x2.splat', ['f64.const', 0]]])
|
|
3671
|
+
pre.push(['local.set', activeV, ['v128.const', 'i32x4', '-1', '-1', '-1', '-1']])
|
|
3672
|
+
pre.push(['local.set', maxitLane, ['f64x2.splat', boundI32 ? ['f64.convert_i32_s', boundExpr] : boundExpr]])
|
|
3673
|
+
// AND a keep into the active mask: limit compares iter (f64x2) to the bound; escape lifts its
|
|
3674
|
+
// z-comparison; a mid-break's ¬X is the DIRECT compare negated by v128.not (NaN-exact).
|
|
3675
|
+
const keepMask = (keep, isLimit) => {
|
|
3676
|
+
const m = isLimit
|
|
3677
|
+
? [CMP_LANE[keep.cmp[0]], itLeft ? ['local.get', iterV] : ['local.get', maxitLane], itLeft ? ['local.get', maxitLane] : ['local.get', iterV]]
|
|
3678
|
+
: [CMP_LANE[keep.cmp[0]], lift(keep.cmp[1]), lift(keep.cmp[2])]
|
|
3679
|
+
return keep.negate ? ['v128.not', m] : m
|
|
3680
|
+
}
|
|
3681
|
+
const sbody = [
|
|
3682
|
+
...teeStmts(tees.slice(0, teeTop)),
|
|
3683
|
+
['local.set', activeV, ['v128.and', ['local.get', activeV], keepMask(keepTopC, kTop === 'limit')]],
|
|
3684
|
+
// compound `while (A && B)`: the second keep (B) also gates at the top, before the body
|
|
3685
|
+
...(compoundTop ? teeStmts(tees.slice(teeTop)).concat([['local.set', activeV, ['v128.and', ['local.get', activeV], keepMask(keepMidC, kMid === 'limit')]]]) : []),
|
|
3686
|
+
['br_if', sIn, ['i32.eqz', ['v128.any_true', ['local.get', activeV]]]],
|
|
3687
|
+
]
|
|
3688
|
+
for (let i = 0; i < ibody.length; i++) {
|
|
3689
|
+
if (!compoundTop && i === midIdx) {
|
|
3690
|
+
sbody.push(...teeStmts(tees.slice(teeTop)))
|
|
3691
|
+
sbody.push(['local.set', activeV, ['v128.and', ['local.get', activeV], keepMask(keepMidC, kMid === 'limit')]])
|
|
3692
|
+
} else {
|
|
3693
|
+
const v = ibody[i][1]
|
|
3694
|
+
if (carried.has(v)) sbody.push(['local.set', shadow.get(v), ['v128.bitselect', lift(ibody[i][2]), ['local.get', shadow.get(v)], ['local.get', activeV]]])
|
|
3695
|
+
else sbody.push(['local.set', shadow.get(v), lift(ibody[i][2])])
|
|
3696
|
+
}
|
|
3697
|
+
}
|
|
3698
|
+
sbody.push(['local.set', iterV, ['v128.bitselect', ['f64x2.add', ['local.get', iterV], ['f64x2.splat', ['f64.const', 1]]], ['local.get', iterV], ['local.get', activeV]]])
|
|
3699
|
+
simdInner = ['block', sIn, ['loop', sIl, ...sbody, ['br', sIl]]]
|
|
3700
|
+
const carriedInEpi = [...carried].filter(v => epilogue.some(s => readsVar(s, v)))
|
|
3701
|
+
epiLane = (k) => {
|
|
3702
|
+
const out = []
|
|
3703
|
+
for (const v of carriedInEpi) out.push(['local.set', v, ['f64x2.extract_lane', k, ['local.get', shadow.get(v)]]])
|
|
3704
|
+
out.push(['local.set', itVar, ['i32.trunc_f64_s', ['f64x2.extract_lane', k, ['local.get', iterV]]]])
|
|
3705
|
+
for (const s of epilogue) out.push(bump(s, k))
|
|
3706
|
+
return out
|
|
3707
|
+
}
|
|
3708
|
+
}
|
|
3709
|
+
|
|
3710
|
+
// SIMD outer loop: process a pair while x+1 is still a valid pixel, then advance every pixel IV
|
|
3711
|
+
// by 2. The scalar tail (original block) finishes any last odd pixel.
|
|
3712
|
+
const simdOuter = ['block', sOut, ['loop', sOl,
|
|
3713
|
+
['br_if', sOut, ['i32.eqz', [oExit.cmpOp, bump(['local.get', pxVar], 1), widthBound]]],
|
|
3714
|
+
...pre, simdInner, ...postLoop, ...epiLane(0), ...epiLane(1),
|
|
3715
|
+
...pixelIVs.map(p => ['local.set', p.name, [p.type + '.add', ['local.get', p.name], [p.type + '.const', 2]]]),
|
|
3716
|
+
['br', sOl]]]
|
|
3717
|
+
|
|
3718
|
+
// wrapper: hoisted preambles once (outer + inner-block invariants like BAILOUT), then the
|
|
3719
|
+
// SIMD even-pixel loop, then the original scalar block handles the odd-pixel tail.
|
|
3720
|
+
const tailBlock = ['block', oLabel, loopNode]
|
|
3721
|
+
const wrapper = ['block', nm('w'), ...preamble, ...innerPre, simdOuter, tailBlock]
|
|
3722
|
+
return { wrapper, newLocalDecls }
|
|
3723
|
+
}
|
|
3724
|
+
|
|
3725
|
+
// Math.sin/cos lower to `call $math.{sin,cos}_core` (the emit-time fast path, math.js:67); the
|
|
3726
|
+
// public `$math.{sin,cos}` wrap the same core. Their f64x2 mirrors $math.sin2/$math.cos2 (the
|
|
3727
|
+
// vectorized reduce+horner, module/math.js:543) are BIT-EXACT per lane to the scalar core — so we
|
|
3728
|
+
// can lift the call straight to the *2 helper. Phase-2 adds pow/log/atan2 here (see PPC_CALL2).
|
|
3729
|
+
// NOTE: scalar targets here must be kept out of inlineOnce's single-caller inlining (SIMD_PROTECTED
|
|
3730
|
+
// in src/wat/optimize.js) — else the call node is gone before this lift runs.
|
|
3731
|
+
const PPC_CALL2 = {
|
|
3732
|
+
'$math.sin_core': '$math.sin2', '$math.cos_core': '$math.cos2',
|
|
3733
|
+
'$math.sin': '$math.sin2', '$math.cos': '$math.cos2',
|
|
3734
|
+
'$math.pow': '$math.pow2', // 2-arg; bit-exact per-lane scalar (cancellation-sensitive — see module/math.js)
|
|
3735
|
+
'$math.atan2': '$math.atan2_2', '$math.hypot': '$math.hypot_2', // 2-arg; bit-exact extract/repack
|
|
3736
|
+
// log/exp/exp2: TRUE f64x2 polys — both lanes one evaluation (≈2×, beats V8 native log). Bit-exact
|
|
3737
|
+
// via hot-path-vectorized + scalar-edge-fallback ($math.log_v/exp_v/exp2_v, module/math.js).
|
|
3738
|
+
'$math.log': '$math.log_v', '$math.exp': '$math.exp_v', '$math.exp2': '$math.exp2_v',
|
|
3739
|
+
}
|
|
3740
|
+
|
|
3741
|
+
// Per-pixel-color vectorizer. The dual of tryDivergentEscapeVectorize for kernels with NO inner
|
|
3742
|
+
// escape loop: an outer pixel loop whose body computes an f64 value from the pixel index (via
|
|
3743
|
+
// cos/sin/sqrt/…), packs it to a u32 colour, and stores it — every pixel independent. We lift the
|
|
3744
|
+
// liftable f64 PREFIX of the body to f64x2 (two adjacent pixels per lane: the index becomes the
|
|
3745
|
+
// ramp [x, x+1]; transcendentals map to the bit-exact $math.*2 helpers; conditionals to bitselect),
|
|
3746
|
+
// then run the SCALAR pack+store once per lane (extract_lane → the original f64 local → the
|
|
3747
|
+
// untouched integer pack). The expensive transcendentals run 2-wide; the cheap pack stays scalar.
|
|
3748
|
+
// Bit-exact by construction: f64x2 arithmetic is per-lane IEEE-identical and extract_lane is exact.
|
|
3749
|
+
// A call we can't yet vectorize (pow in Phase 1) just ends the SIMD prefix — its lane local and the
|
|
3750
|
+
// rest fall to the scalar epilogue, so the kernel still partially vectorizes. The original scalar
|
|
3751
|
+
// loop, re-run as the tail, finishes the odd last pixel for free (its own `x < W` guard).
|
|
3752
|
+
function tryPerPixelColor(blockNode, fnLocals, freshIdRef, pureFuncMap) {
|
|
3753
|
+
// Outer per-pixel scaffold — shared with tryDivergentEscapeVectorize; this pass
|
|
3754
|
+
// takes the straight-line-body branch (no inner escape loop) below.
|
|
3755
|
+
const outer = matchOuterPixelLoop(blockNode)
|
|
3756
|
+
if (!outer) return null
|
|
3757
|
+
const { oLabel, loopNode, preamble, pixelIVs, pxVar, widthBound, pivType, obody, oExit } = outer
|
|
3758
|
+
|
|
3759
|
+
// ---- body: straight-line (no inner escape loop), no impure call ----
|
|
3760
|
+
for (const s of obody)
|
|
3761
|
+
if (isArr(s) && s[0] === 'block' && s.slice(1).some(c => isArr(c) && c[0] === 'loop')) return null // inner escape loop → tryDivergentEscapeVectorize's job
|
|
3762
|
+
// A non-pure call (e.g. a ray-march helper that writes a scratch global / memory) can mutate state
|
|
3763
|
+
// that a lane local — computed ONCE, before the per-lane epilogue runs the call — would then read
|
|
3764
|
+
// stale, breaking bit-exactness. $math.* helpers are pure (no global/memory writes), so allow them.
|
|
3765
|
+
// Pure user-defined functions in pureFuncMap are also safe: they have no side effects (verified
|
|
3766
|
+
// when the map is built) and liftPPC inlines them expression-level rather than emitting a call.
|
|
3767
|
+
const impureCall = (n) => isArr(n) && ((n[0] === 'call' && typeof n[1] === 'string' && !n[1].startsWith('$math.') && !(pureFuncMap && pureFuncMap.has(n[1]))) || n.some(impureCall))
|
|
3768
|
+
if (obody.some(impureCall)) return null
|
|
3769
|
+
|
|
3770
|
+
// bump: substitute every pixel IV with (IV + k) in its own type — for the lane-k epilogue.
|
|
3771
|
+
const bump = (n, k) => k === 0 ? n
|
|
3772
|
+
: (isArr(n) && n[0] === 'local.get' && pivType.has(n[1]))
|
|
3773
|
+
? [pivType.get(n[1]) + '.add', n, [pivType.get(n[1]) + '.const', k]]
|
|
3774
|
+
: (isArr(n) ? n.map(c => bump(c, k)) : n)
|
|
3775
|
+
|
|
3776
|
+
// Pixel-coordinate aliases: a local consistently CSE'd to `convert_i32_s(pixelIV)` (jz tees the f64
|
|
3777
|
+
// pixel-x once — reused for the store address AND the per-pixel math — so it lives inside the i32
|
|
3778
|
+
// offset stmt, out of reach as a lane local). Treat its reads as the ramp, recomputed per lane.
|
|
3779
|
+
const pxAlias = new Map()
|
|
3780
|
+
{
|
|
3781
|
+
const defs = new Map()
|
|
3782
|
+
const scan = (n) => { if (!isArr(n)) return; if ((n[0] === 'local.set' || n[0] === 'local.tee') && typeof n[1] === 'string') { (defs.get(n[1]) || defs.set(n[1], []).get(n[1])).push(n[2]) } for (let i = 1; i < n.length; i++) scan(n[i]) }
|
|
3783
|
+
obody.forEach(scan)
|
|
3784
|
+
for (const [name, rhss] of defs) {
|
|
3785
|
+
const j = JSON.stringify(rhss[0])
|
|
3786
|
+
if (!rhss.every(r => JSON.stringify(r) === j)) continue // multiple distinct defs → not a stable alias
|
|
3787
|
+
const r = rhss[0]
|
|
3788
|
+
if (isArr(r) && r[0] === 'f64.convert_i32_s' && isArr(r[1]) && r[1][0] === 'local.get' && pivType.get(r[1][1]) === 'i32') pxAlias.set(name, r[1][1])
|
|
3789
|
+
else if (isArr(r) && r[0] === 'local.get' && pivType.get(r[1]) === 'f64') pxAlias.set(name, r[1])
|
|
3790
|
+
}
|
|
3791
|
+
}
|
|
3792
|
+
// The two lanes of a pixel IV (or its alias): [v, v+1], in f64 (an i32 IV is converted per lane).
|
|
3793
|
+
const rampOf = (piv) => pivType.get(piv) === 'f64'
|
|
3794
|
+
? ['f64x2.replace_lane', 1, ['f64x2.splat', ['local.get', piv]], ['f64.add', ['local.get', piv], ['f64.const', 1]]]
|
|
3795
|
+
: ['f64x2.replace_lane', 1, ['f64x2.splat', ['f64.convert_i32_s', ['local.get', piv]]], ['f64.convert_i32_s', ['i32.add', ['local.get', piv], ['i32.const', 1]]]]
|
|
3796
|
+
|
|
3797
|
+
const id = freshIdRef.next++
|
|
3798
|
+
const nm = (s) => `$__ppc${id}_${s}`
|
|
3799
|
+
const laneMap = new Map() // f64 lane-local name → its v128 shadow
|
|
3800
|
+
const laneLifted = new Map() // f64 lane-local name → its lifted f64x2 expr
|
|
3801
|
+
|
|
3802
|
+
// Inline a pure user function call into a lifted f64x2 expression.
|
|
3803
|
+
// `callNode` is ['call', '$name', arg0, arg1, ...]; `outerLift` is the liftPPC fn.
|
|
3804
|
+
// Walks the callee's body, substituting params with lifted args and inlined-local
|
|
3805
|
+
// intermediates. Returns null if any step fails (bail → scalar epilogue).
|
|
3806
|
+
// SOUND: only called when callee is in pureFuncMap (no stores/global.sets/impure
|
|
3807
|
+
// calls); param names are read-only in the inlinee body (verified below).
|
|
3808
|
+
const liftPPCInline = (callNode, outerLift) => {
|
|
3809
|
+
const callee = pureFuncMap.get(callNode[1])
|
|
3810
|
+
if (!callee) return null
|
|
3811
|
+
const calleeBodyStart = findBodyStart(callee)
|
|
3812
|
+
if (calleeBodyStart < 0) return null
|
|
3813
|
+
|
|
3814
|
+
// Collect callee params in order.
|
|
3815
|
+
const calleeParams = []
|
|
3816
|
+
for (let i = 2; i < calleeBodyStart; i++) {
|
|
3817
|
+
const d = callee[i]
|
|
3818
|
+
if (isArr(d) && d[0] === 'param' && typeof d[1] === 'string') calleeParams.push(d[1])
|
|
3819
|
+
}
|
|
3820
|
+
// Args supplied by the call site (call node children after the name).
|
|
3821
|
+
const callArgs = callNode.slice(2)
|
|
3822
|
+
if (callArgs.length !== calleeParams.length) return null
|
|
3823
|
+
|
|
3824
|
+
// Lift each arg with the outer liftPPC.
|
|
3825
|
+
const substMap = new Map()
|
|
3826
|
+
for (let i = 0; i < calleeParams.length; i++) {
|
|
3827
|
+
const lifted = outerLift(callArgs[i])
|
|
3828
|
+
if (lifted === null) return null
|
|
3829
|
+
substMap.set(calleeParams[i], lifted)
|
|
3830
|
+
}
|
|
3831
|
+
|
|
3832
|
+
// Verify no local.set on a param name inside the callee body (params are read-only).
|
|
3833
|
+
for (const pname of calleeParams) {
|
|
3834
|
+
if (writesName(callee.slice(calleeBodyStart), pname)) return null
|
|
3835
|
+
}
|
|
3836
|
+
|
|
3837
|
+
// liftInline: lift a callee-body expression using substMap (params + inlined locals).
|
|
3838
|
+
// Handles f64.const, local.get from substMap, LANE_PURE.f64 ops, and PPC_CALL2 calls.
|
|
3839
|
+
// Returns null on any unsupported node.
|
|
3840
|
+
const liftInline = (n) => {
|
|
3841
|
+
if (!isArr(n)) return null
|
|
3842
|
+
const op = n[0]
|
|
3843
|
+
if (op === 'f64.const') return ['f64x2.splat', n]
|
|
3844
|
+
if (op === 'local.get' && typeof n[1] === 'string') {
|
|
3845
|
+
return substMap.has(n[1]) ? substMap.get(n[1]) : null
|
|
3846
|
+
}
|
|
3847
|
+
if (op === 'call') {
|
|
3848
|
+
const v2 = PPC_CALL2[n[1]]
|
|
3849
|
+
if (v2 && n.length === 3) { const a = liftInline(n[2]); return a && ['call', v2, a] }
|
|
3850
|
+
if (v2 && n.length === 4) { const a = liftInline(n[2]), b = liftInline(n[3]); return (a && b) ? ['call', v2, a, b] : null }
|
|
3851
|
+
return null
|
|
3852
|
+
}
|
|
3853
|
+
if (LANE_PURE.f64.has(op)) {
|
|
3854
|
+
const ks = n.slice(1).map(liftInline)
|
|
3855
|
+
return ks.some(k => k === null) ? null : [LANE_PURE.f64.get(op).simd, ...ks]
|
|
3856
|
+
}
|
|
3857
|
+
return null
|
|
3858
|
+
}
|
|
3859
|
+
|
|
3860
|
+
// Walk callee body: local.set stmts define inlined locals; return stmt gives result.
|
|
3861
|
+
for (let i = calleeBodyStart; i < callee.length; i++) {
|
|
3862
|
+
const stmt = callee[i]
|
|
3863
|
+
if (!isArr(stmt)) return null
|
|
3864
|
+
if (stmt[0] === 'local.set' && typeof stmt[1] === 'string') {
|
|
3865
|
+
const lifted = liftInline(stmt[2])
|
|
3866
|
+
if (lifted === null) return null
|
|
3867
|
+
substMap.set(stmt[1], lifted)
|
|
3868
|
+
continue
|
|
3869
|
+
}
|
|
3870
|
+
if (stmt[0] === 'return') {
|
|
3871
|
+
return liftInline(stmt[1])
|
|
3872
|
+
}
|
|
3873
|
+
// Any other statement type → bail (impure or unsupported structure).
|
|
3874
|
+
return null
|
|
3875
|
+
}
|
|
3876
|
+
return null // no return stmt found
|
|
3877
|
+
}
|
|
3878
|
+
|
|
3879
|
+
// Lift a scalar f64 expression to f64x2: pixel IV → ramp [v, v+1]; an earlier lane local → its
|
|
3880
|
+
// shadow; an invariant (local/global the loop never writes) → splat; transcendental call → the
|
|
3881
|
+
// *2 helper; conditional → bitselect; LANE_PURE.f64 op → recurse. null = not liftable (the lift
|
|
3882
|
+
// stops here and the rest becomes the scalar epilogue).
|
|
3883
|
+
const liftPPC = (n) => {
|
|
3884
|
+
if (!isArr(n)) return null
|
|
3885
|
+
const op = n[0]
|
|
3886
|
+
if (op === 'f64.const') return ['f64x2.splat', n]
|
|
3887
|
+
if (op === 'local.get') {
|
|
3888
|
+
const v = n[1]
|
|
3889
|
+
if (laneMap.has(v)) return ['local.get', laneMap.get(v)]
|
|
3890
|
+
if (pxAlias.has(v)) return rampOf(pxAlias.get(v))
|
|
3891
|
+
if (pivType.get(v) === 'f64') return rampOf(v)
|
|
3892
|
+
if (writesName(loopNode, v)) return null
|
|
3893
|
+
return ['f64x2.splat', n]
|
|
3894
|
+
}
|
|
3895
|
+
if (op === 'local.tee') { // CSE temp inside a lane expr (e.g. `dx` reused as dx*dx) → a v128 tee
|
|
3896
|
+
const lifted = liftPPC(n[2])
|
|
3897
|
+
if (lifted === null) return null
|
|
3898
|
+
const lane = laneMap.get(n[1]) || nm('t' + n[1].replace(/\W/g, ''))
|
|
3899
|
+
laneMap.set(n[1], lane) // later local.get $v in this expr resolves to the tee's lane
|
|
3900
|
+
return ['local.tee', lane, lifted]
|
|
3901
|
+
}
|
|
3902
|
+
if (op === 'global.get') return writesName(loopNode, n[1]) ? null : ['f64x2.splat', n]
|
|
3903
|
+
if (op === 'f64.convert_i32_s' && isArr(n[1]) && n[1][0] === 'local.get' && pivType.get(n[1][1]) === 'i32') return rampOf(n[1][1])
|
|
3904
|
+
if (op === 'call') {
|
|
3905
|
+
const v2 = PPC_CALL2[n[1]]
|
|
3906
|
+
if (v2 && n.length === 3) { const a = liftPPC(n[2]); return a && ['call', v2, a] }
|
|
3907
|
+
if (v2 && n.length === 4) { const a = liftPPC(n[2]), b = liftPPC(n[3]); return (a && b) ? ['call', v2, a, b] : null }
|
|
3908
|
+
// Pure user-function inline: substitute params with lifted args, walk body.
|
|
3909
|
+
if (pureFuncMap && pureFuncMap.has(n[1])) return liftPPCInline(n, liftPPC)
|
|
3910
|
+
return null
|
|
3911
|
+
}
|
|
3912
|
+
if (op === 'if') { // `cond ? X : Y` (jz lowers to (if (result f64) COND (then X)(else Y))) → bitselect
|
|
3913
|
+
if (!isArr(n[1]) || n[1][0] !== 'result' || n[1][1] !== 'f64') return null
|
|
3914
|
+
const thenN = n[3], elseN = n[4]
|
|
3915
|
+
if (!isArr(thenN) || thenN[0] !== 'then' || thenN.length !== 2) return null
|
|
3916
|
+
if (!isArr(elseN) || elseN[0] !== 'else' || elseN.length !== 2) return null
|
|
3917
|
+
let cond = n[2]
|
|
3918
|
+
if (isArr(cond) && cond[0] === 'i32.ne' && isI32Const(cond[2]) && cond[2][1] === 0) cond = cond[1]
|
|
3919
|
+
const cmp = isArr(cond) && cond.length === 3 ? CMP_LANE[cond[0]] : null
|
|
3920
|
+
if (!cmp) return null
|
|
3921
|
+
const ca = liftPPC(cond[1]), cb = liftPPC(cond[2]), x = liftPPC(thenN[1]), y = liftPPC(elseN[1])
|
|
3922
|
+
if (!ca || !cb || !x || !y) return null
|
|
3923
|
+
return ['v128.bitselect', x, y, [cmp, ca, cb]]
|
|
3924
|
+
}
|
|
3925
|
+
if (LANE_PURE.f64.has(op)) {
|
|
3926
|
+
const ks = n.slice(1).map(liftPPC)
|
|
3927
|
+
return ks.some(k => k === null) ? null : [LANE_PURE.f64.get(op).simd, ...ks]
|
|
3928
|
+
}
|
|
3929
|
+
return null
|
|
3930
|
+
}
|
|
3931
|
+
|
|
3932
|
+
// Classify each body statement: a `local.set $v EXPR` with v an f64 whose EXPR fully lifts is a
|
|
3933
|
+
// SIMD lane local (computed once per pair); everything else (the integer pack, the store, an
|
|
3934
|
+
// un-liftable call like pow in Phase 1, a recomputed i32 offset) is a scalar EPILOGUE statement,
|
|
3935
|
+
// re-run per lane. Lane locals need NOT be a contiguous prefix — `offset = w*y+x` (i32) commonly
|
|
3936
|
+
// precedes the f64 work. Processed in source order so a lane local can reference an earlier one;
|
|
3937
|
+
// liftPPC returns null on a read of any in-loop value that isn't already a lane local (incl. a
|
|
3938
|
+
// later or epilogue local), so the classification self-enforces "lane locals depend only on
|
|
3939
|
+
// IVs/invariants/earlier lane locals" — reordering all lane computes ahead of the epilogue is safe.
|
|
3940
|
+
const epilogue = []
|
|
3941
|
+
for (const s of obody) {
|
|
3942
|
+
if (isArr(s) && s[0] === 'local.set' && s.length === 3 && fnLocals.get(s[1]) === 'f64') {
|
|
3943
|
+
const before = new Set(laneMap.keys())
|
|
3944
|
+
const lifted = liftPPC(s[2])
|
|
3945
|
+
if (lifted !== null) { laneMap.set(s[1], nm('l' + s[1].replace(/\W/g, ''))); laneLifted.set(s[1], lifted); continue }
|
|
3946
|
+
for (const k of [...laneMap.keys()]) if (!before.has(k)) laneMap.delete(k) // roll back tee pollution from a failed lift
|
|
3947
|
+
}
|
|
3948
|
+
epilogue.push(s)
|
|
3949
|
+
}
|
|
3950
|
+
if (!laneMap.size) return null // nothing lifted to f64x2
|
|
3951
|
+
// HAZARD: a lane local re-written by an epilogue statement leaves its f64x2 shadow STALE.
|
|
3952
|
+
// e.g. `let fx=0; if(denom>ε){fx=…}; let mag=hypot(fx,…)` — `fx=0` lifts to a lane local
|
|
3953
|
+
// splat(0); the statement-form `if` lands in the scalar epilogue (updates only the SCALAR local),
|
|
3954
|
+
// so the lifted `hypot(fx,…)` — emitted BEFORE the epilogue — reads the stale splat(0) → all-zero.
|
|
3955
|
+
// Bail ONLY when the stale shadow actually feeds another LANE compute: a lane local whose shadow
|
|
3956
|
+
// is CONSUMED by some laneLifted expr and is ALSO reassigned in the epilogue. (A lane local merely
|
|
3957
|
+
// extracted for the scalar epilogue — e.g. `gv`, clamped by `if(gv<0)gv=0` then packed — is safe:
|
|
3958
|
+
// the clamp runs per-lane after extraction, corrupting no other lane.)
|
|
3959
|
+
{
|
|
3960
|
+
const consumedShadows = new Set()
|
|
3961
|
+
const scanShadow = (n) => { if (!isArr(n)) return; if (n[0] === 'local.get' && typeof n[1] === 'string') consumedShadows.add(n[1]); for (const c of n.slice(1)) scanShadow(c) }
|
|
3962
|
+
for (const expr of laneLifted.values()) scanShadow(expr)
|
|
3963
|
+
const hazard = (n) => isArr(n) && (((n[0] === 'local.set' || n[0] === 'local.tee') && laneMap.has(n[1]) && consumedShadows.has(laneMap.get(n[1]))) || n.slice(1).some(hazard))
|
|
3964
|
+
if (epilogue.some(hazard)) return null
|
|
3965
|
+
}
|
|
3966
|
+
// Only worth the extract overhead if a costly op (a *2 transcendental or f64x2.sqrt) got lifted.
|
|
3967
|
+
const heavy = (n) => isArr(n) && ((n[0] === 'call' && /\$math\.(sin2|cos2|pow2|log_v|atan2_2|hypot_2|exp2_2|tan2)/.test(n[1])) || n[0] === 'f64x2.sqrt' || n.some(heavy))
|
|
3968
|
+
if (![...laneLifted.values()].some(heavy)) return null // only cheap arithmetic lifted — not worth it
|
|
3969
|
+
|
|
3970
|
+
// Exactly one i32.store, found anywhere in the epilogue (jz wraps `mem[off]=…` in a `(block …)`).
|
|
3971
|
+
let storeStmt = null
|
|
3972
|
+
const findStore = (n) => { if (!isArr(n)) return; if (STORE_OPS[n[0]]) { if (storeStmt) storeStmt = false; else if (storeStmt !== false) storeStmt = n } for (const c of n) findStore(c) }
|
|
3973
|
+
epilogue.forEach(findStore)
|
|
3974
|
+
if (!storeStmt || storeStmt[0] !== 'i32.store') return null // not exactly one u32 colour store
|
|
3975
|
+
// The store cell must differ per lane, i.e. its address must depend on a pixel IV — directly
|
|
3976
|
+
// (chladni's `px[j]`) or transitively through an epilogue local (interference's `mem[offset]`,
|
|
3977
|
+
// offset=w*y+x). Follow the address's reads through epilogue local definitions to a pixel IV.
|
|
3978
|
+
// Walk every child except a set's name slot (a def nested under an `(if … then)` must still be
|
|
3979
|
+
// recorded — the n.slice(2)-everywhere form would miss it, conservatively bailing a valid kernel).
|
|
3980
|
+
const epiDef = new Map()
|
|
3981
|
+
const collect = (n) => { if (!isArr(n)) return; if (n[0] === 'local.set' && typeof n[1] === 'string' && !epiDef.has(n[1])) epiDef.set(n[1], n[2]); for (const c of (n[0] === 'local.set' ? n.slice(2) : n.slice(1))) collect(c) }
|
|
3982
|
+
epilogue.forEach(collect)
|
|
3983
|
+
const feedsIV = (n, seen = new Set()) => isArr(n) && (n[0] === 'local.get'
|
|
3984
|
+
? (pivType.has(n[1]) || (epiDef.has(n[1]) && !seen.has(n[1]) && (seen.add(n[1]), feedsIV(epiDef.get(n[1]), seen))))
|
|
3985
|
+
: n.some(c => feedsIV(c, seen)))
|
|
3986
|
+
if (!feedsIV(storeStmt[1])) return null // store cell wouldn't vary per lane → can't pair
|
|
3987
|
+
|
|
3988
|
+
// ---- epilogue safety: runs scalar per lane (each statement bumped to pixel j+k). It may read a
|
|
3989
|
+
// lane local (extracted below), an invariant/pixel-IV, or a value the epilogue itself computes —
|
|
3990
|
+
// incl. within-statement tees (e.g. the Infinity-guard temp inside an `(if … |0)` pack). Straight-
|
|
3991
|
+
// line source guarantees write-before-read, so it suffices that every read of an in-loop local is
|
|
3992
|
+
// a lane local or written somewhere in the epilogue (a lane local read is satisfied by extraction). ----
|
|
3993
|
+
{
|
|
3994
|
+
const epiWritten = new Set()
|
|
3995
|
+
const wr = (n) => { if (!isArr(n)) return; const st = (n[0] === 'local.set' || n[0] === 'local.tee') && typeof n[1] === 'string'; if (st) epiWritten.add(n[1]); for (const c of (st ? n.slice(2) : n.slice(1))) wr(c) }
|
|
3996
|
+
epilogue.forEach(wr)
|
|
3997
|
+
const reads = new Set(); const rd = (n) => { if (!isArr(n)) return; if (n[0] === 'local.get') reads.add(n[1]); else for (const c of n) rd(c) }
|
|
3998
|
+
epilogue.forEach(rd)
|
|
3999
|
+
for (const v of reads) if (writesName(loopNode, v) && !laneMap.has(v) && !epiWritten.has(v) && !pivType.has(v)) return null // reads an in-loop value with no per-lane source
|
|
4000
|
+
}
|
|
4001
|
+
const epiReads = [...laneMap.keys()].filter(v => epilogue.some(s => readsVar(s, v)))
|
|
4002
|
+
|
|
4003
|
+
// ============================ emit ============================
|
|
4004
|
+
const newLocalDecls = [...laneMap.values()].map(n => ['local', n, 'v128'])
|
|
4005
|
+
const laneCompute = [...laneLifted.keys()].map(v => ['local.set', laneMap.get(v), laneLifted.get(v)])
|
|
4006
|
+
const epiLane = (k) => [
|
|
4007
|
+
...epiReads.map(v => ['local.set', v, ['f64x2.extract_lane', k, ['local.get', laneMap.get(v)]]]),
|
|
4008
|
+
...epilogue.map(s => bump(s, k)),
|
|
4009
|
+
]
|
|
4010
|
+
const sOut = nm('ob'), sOl = nm('ol')
|
|
4011
|
+
const simdOuter = ['block', sOut, ['loop', sOl,
|
|
4012
|
+
['br_if', sOut, ['i32.eqz', [oExit.cmpOp, bump(['local.get', pxVar], 1), widthBound]]],
|
|
4013
|
+
...laneCompute, ...epiLane(0), ...epiLane(1),
|
|
4014
|
+
...pixelIVs.map(p => ['local.set', p.name, [p.type + '.add', ['local.get', p.name], [p.type + '.const', 2]]]),
|
|
4015
|
+
['br', sOl]]]
|
|
4016
|
+
const wrapper = ['block', nm('w'), ...preamble, simdOuter, ['block', oLabel, loopNode]]
|
|
4017
|
+
return { wrapper, newLocalDecls }
|
|
4018
|
+
}
|
|
4019
|
+
|
|
4020
|
+
// ---- Outer-loop strip-mine over an inner reduction (tryOuterStrip, experimental) ----
|
|
4021
|
+
//
|
|
4022
|
+
// The dual of tryPerPixelColor for pixel loops whose per-pixel value comes from an
|
|
4023
|
+
// INNER REDUCTION over invariant data — metaballs `sum += r²/((cx-bx[b])²+(cy-by[b])²+ε)`,
|
|
4024
|
+
// voronoi/lyapunov shapes. Strip-mine the OUTER pixel loop 2-wide: pixels (xi, xi+1) →
|
|
4025
|
+
// f64x2 lanes. The per-pixel coordinate (`cx = xi/W`) becomes a ramp `[cx, cx+1/W]`; the
|
|
4026
|
+
// inner loop's loads `bx[b]` are indexed by the INNER IV (same for both pixels) → splat;
|
|
4027
|
+
// the accumulator `sum` becomes an f64x2 carrying both lanes' running sums. After the inner
|
|
4028
|
+
// loop, each lane's sum is extracted and the scalar pack+store runs per lane (xi, xi+1).
|
|
4029
|
+
//
|
|
4030
|
+
// BIT-EXACT: each lane accumulates in the SAME scalar order as the original (f64x2.add is
|
|
4031
|
+
// per-lane IEEE-754-identical) — a per-lane reduction reorders nothing, unlike a horizontal
|
|
4032
|
+
// fold. The inner loop's trip count (b < count) is invariant, so its scaffold stays scalar;
|
|
4033
|
+
// only the f64 body lifts. Distinct base subtrees assumed non-aliasing (the standing model).
|
|
4034
|
+
// Gated behind cfg.experimentalOuterStrip until proven across the corpus.
|
|
4035
|
+
function tryOuterStrip(blockNode, fnLocals, freshIdRef, enabled) {
|
|
4036
|
+
if (!enabled) return null
|
|
4037
|
+
const outer = matchOuterPixelLoop(blockNode)
|
|
4038
|
+
if (!outer) return null
|
|
4039
|
+
const { oLabel, loopNode, preamble, pixelIVs, pxVar, widthBound, pivType, obody, oExit } = outer
|
|
4040
|
+
|
|
4041
|
+
// Exactly one inner loop in obody; it is the per-pixel reduction.
|
|
4042
|
+
let innerIdx = -1, innerBlock = null
|
|
4043
|
+
for (let i = 0; i < obody.length; i++) {
|
|
4044
|
+
const s = obody[i]
|
|
4045
|
+
if (isArr(s) && s[0] === 'block' && s.slice(1).some(c => isArr(c) && c[0] === 'loop')) {
|
|
4046
|
+
if (innerBlock) return null
|
|
4047
|
+
innerBlock = s; innerIdx = i
|
|
4048
|
+
}
|
|
4049
|
+
}
|
|
4050
|
+
if (!innerBlock) return null
|
|
4051
|
+
const ibl = matchBlockLoop(innerBlock, { allowPreamble: true })
|
|
4052
|
+
if (!ibl) return null
|
|
4053
|
+
if (ibl.preamble.length) return null
|
|
4054
|
+
const innerIV = ibl.incVar, ibody = ibl.body
|
|
4055
|
+
// No impure calls (a non-pure call would read stale state in the per-lane epilogue). $math.* pure.
|
|
4056
|
+
const impureCall = (n) => isArr(n) && ((n[0] === 'call' && typeof n[1] === 'string' && !n[1].startsWith('$math.')) || n.some(impureCall))
|
|
4057
|
+
if (obody.some(impureCall)) return null
|
|
4058
|
+
|
|
4059
|
+
const id = freshIdRef.next++
|
|
4060
|
+
const nm = (s) => `$__os${id}_${s}`
|
|
4061
|
+
const bump = (n, k) => k === 0 ? n
|
|
4062
|
+
: (isArr(n) && n[0] === 'local.get' && pivType.has(n[1])) ? [pivType.get(n[1]) + '.add', n, [pivType.get(n[1]) + '.const', k]]
|
|
4063
|
+
: (isArr(n) ? n.map(c => bump(c, k)) : n)
|
|
4064
|
+
const rampOf = (piv) => pivType.get(piv) === 'f64'
|
|
4065
|
+
? ['f64x2.replace_lane', 1, ['f64x2.splat', ['local.get', piv]], ['f64.add', ['local.get', piv], ['f64.const', 1]]]
|
|
4066
|
+
: ['f64x2.replace_lane', 1, ['f64x2.splat', ['f64.convert_i32_s', ['local.get', piv]]], ['f64.convert_i32_s', ['i32.add', ['local.get', piv], ['i32.const', 1]]]]
|
|
4067
|
+
const readsName = (n, name) => isArr(n) && ((n[0] === 'local.get' && n[1] === name) || n.some(c => readsName(c, name)))
|
|
4068
|
+
|
|
4069
|
+
const laneMap = new Map() // f64 lane-local (per-pixel-varying) name → its v128 shadow
|
|
4070
|
+
// Lift a scalar f64 expr to f64x2 (null = not liftable). pxVar → ramp; lane local → shadow;
|
|
4071
|
+
// pixel-invariant local/global → splat; pixel-invariant f64.load → splat(scalar load);
|
|
4072
|
+
// $math.*2 transcendental; cond → bitselect; LANE_PURE.f64 op → recurse.
|
|
4073
|
+
const liftOS = (n) => {
|
|
4074
|
+
if (!isArr(n)) return null
|
|
4075
|
+
const op = n[0]
|
|
4076
|
+
if (op === 'f64.const') return ['f64x2.splat', n]
|
|
4077
|
+
if (op === 'local.get') {
|
|
4078
|
+
const v = n[1]
|
|
4079
|
+
if (laneMap.has(v)) return ['local.get', laneMap.get(v)]
|
|
4080
|
+
if (pivType.get(v) === 'f64') return rampOf(v)
|
|
4081
|
+
if (writesName(loopNode, v)) return null
|
|
4082
|
+
return ['f64x2.splat', n]
|
|
4083
|
+
}
|
|
4084
|
+
if (op === 'f64.convert_i32_s' && isArr(n[1]) && n[1][0] === 'local.get' && pivType.get(n[1][1]) === 'i32') return rampOf(n[1][1])
|
|
4085
|
+
if (op === 'global.get') return writesName(loopNode, n[1]) ? null : ['f64x2.splat', n]
|
|
4086
|
+
if (LOAD_OPS[op] === 'f64') {
|
|
4087
|
+
// pixel-invariant load (address reads neither the pixel IV nor any per-pixel lane) is the
|
|
4088
|
+
// same value for both lanes → load once, splat. A per-pixel gather is not supported.
|
|
4089
|
+
const addr = typeof n[1] === 'string' && n[1].startsWith('offset=') ? n[2] : n[1]
|
|
4090
|
+
if (readsName(addr, pxVar) || [...laneMap.keys()].some(lv => readsName(addr, lv))) return null
|
|
4091
|
+
return ['f64x2.splat', n]
|
|
4092
|
+
}
|
|
4093
|
+
if (op === 'call') {
|
|
4094
|
+
const v2 = PPC_CALL2[n[1]]
|
|
4095
|
+
if (v2 && n.length === 3) { const a = liftOS(n[2]); return a && ['call', v2, a] }
|
|
4096
|
+
if (v2 && n.length === 4) { const a = liftOS(n[2]), b = liftOS(n[3]); return (a && b) ? ['call', v2, a, b] : null }
|
|
4097
|
+
return null
|
|
4098
|
+
}
|
|
4099
|
+
if (op === 'if') {
|
|
4100
|
+
if (!isArr(n[1]) || n[1][0] !== 'result' || n[1][1] !== 'f64') return null
|
|
4101
|
+
const thenN = n[3], elseN = n[4]
|
|
4102
|
+
if (!isArr(thenN) || thenN[0] !== 'then' || thenN.length !== 2) return null
|
|
4103
|
+
if (!isArr(elseN) || elseN[0] !== 'else' || elseN.length !== 2) return null
|
|
4104
|
+
let cond = n[2]
|
|
4105
|
+
if (isArr(cond) && cond[0] === 'i32.ne' && isI32Const(cond[2]) && cond[2][1] === 0) cond = cond[1]
|
|
4106
|
+
const cmp = isArr(cond) && cond.length === 3 ? CMP_LANE[cond[0]] : null
|
|
4107
|
+
if (!cmp) return null
|
|
4108
|
+
const ca = liftOS(cond[1]), cb = liftOS(cond[2]), x = liftOS(thenN[1]), y = liftOS(elseN[1])
|
|
4109
|
+
if (!ca || !cb || !x || !y) return null
|
|
4110
|
+
return ['v128.bitselect', x, y, [cmp, ca, cb]]
|
|
4111
|
+
}
|
|
4112
|
+
if (LANE_PURE.f64.has(op)) {
|
|
4113
|
+
const ks = n.slice(1).map(liftOS)
|
|
4114
|
+
return ks.some(k => k === null) ? null : [LANE_PURE.f64.get(op).simd, ...ks]
|
|
4115
|
+
}
|
|
4116
|
+
return null
|
|
4117
|
+
}
|
|
4118
|
+
|
|
4119
|
+
// ---- lift the inner loop body: temp f64 lane locals + f64 accumulator(s) `acc = acc + EXPR`,
|
|
4120
|
+
// the inner IV bump stays scalar. Anything else (or an unliftable expr) → bail. ----
|
|
4121
|
+
// Pre-scan: f64 accumulators `acc = acc + EXPR`. Assign their f64x2 shadows up front so laneInit
|
|
4122
|
+
// can seed them and the lift can accumulate into them (order-independent of the inner body).
|
|
4123
|
+
const accNames = new Set()
|
|
4124
|
+
for (const s of ibody) {
|
|
4125
|
+
if (!(isArr(s) && s[0] === 'local.set' && typeof s[1] === 'string' && s.length === 3)) continue
|
|
4126
|
+
const name = s[1], rhs = s[2]
|
|
4127
|
+
if (fnLocals.get(name) !== 'f64' || !isArr(rhs) || rhs[0] !== 'f64.add') continue
|
|
4128
|
+
const addend = isLocalGet(rhs[1], name) ? rhs[2] : isLocalGet(rhs[2], name) ? rhs[1] : null
|
|
4129
|
+
if (addend != null && !readsName(addend, name)) { accNames.add(name); laneMap.set(name, nm('acc' + name.replace(/\W/g, ''))) }
|
|
4130
|
+
}
|
|
4131
|
+
if (!accNames.size) return null
|
|
4132
|
+
|
|
4133
|
+
// ---- pre-inner-loop stmts (obody[<innerIdx]): per-pixel coord lanes (cx = f(xi) → ramp),
|
|
4134
|
+
// accumulator seeds (→ splat), scalar inner-IV init. MUST run before the inner-body lift so the
|
|
4135
|
+
// per-pixel coord lanes are registered when the inner loop references them. ----
|
|
4136
|
+
const laneInit = []
|
|
4137
|
+
for (let i = 0; i < innerIdx; i++) {
|
|
4138
|
+
const s = obody[i]
|
|
4139
|
+
if (!(isArr(s) && s[0] === 'local.set' && typeof s[1] === 'string' && s.length === 3)) { laneInit.push(s); continue }
|
|
4140
|
+
const name = s[1]
|
|
4141
|
+
if (accNames.has(name)) { // accumulator seed → splat
|
|
4142
|
+
const seed = liftOS(s[2])
|
|
4143
|
+
if (!seed) return null
|
|
4144
|
+
laneInit.push(['local.set', laneMap.get(name), seed]); continue
|
|
4145
|
+
}
|
|
4146
|
+
if (fnLocals.get(name) === 'f64' && readsName(s[2], pxVar)) { // per-pixel coord (cx = xi/W) → ramp lane
|
|
4147
|
+
const lane = liftOS(s[2])
|
|
4148
|
+
if (!lane) return null
|
|
4149
|
+
const sh = nm('p' + name.replace(/\W/g, ''))
|
|
4150
|
+
laneMap.set(name, sh)
|
|
4151
|
+
laneInit.push(['local.set', sh, lane]); continue
|
|
4152
|
+
}
|
|
4153
|
+
laneInit.push(s) // scalar (inner IV init `b=0`, invariant setup)
|
|
4154
|
+
}
|
|
4155
|
+
|
|
4156
|
+
// ---- lift the inner-loop body: temp f64 lane locals + accumulate into the acc shadows; the
|
|
4157
|
+
// inner IV bump stays scalar. Per-pixel coords now resolve via laneMap. ----
|
|
4158
|
+
const liftedInner = []
|
|
4159
|
+
for (const s of ibody) {
|
|
4160
|
+
if (matchInc1(s) === innerIV || matchIncN(s)?.name === innerIV) { liftedInner.push(s); continue }
|
|
4161
|
+
if (!(isArr(s) && s[0] === 'local.set' && typeof s[1] === 'string' && s.length === 3)) return null
|
|
4162
|
+
const name = s[1], rhs = s[2]
|
|
4163
|
+
if (fnLocals.get(name) !== 'f64') return null
|
|
4164
|
+
if (accNames.has(name)) {
|
|
4165
|
+
const addend = isLocalGet(rhs[1], name) ? rhs[2] : rhs[1]
|
|
4166
|
+
const lifted = liftOS(addend)
|
|
4167
|
+
if (!lifted) return null
|
|
4168
|
+
liftedInner.push(['local.set', laneMap.get(name), ['f64x2.add', ['local.get', laneMap.get(name)], lifted]]); continue
|
|
4169
|
+
}
|
|
4170
|
+
if (readsName(rhs, name)) return null // loop-carried non-accumulator → bail
|
|
4171
|
+
const lifted = liftOS(rhs)
|
|
4172
|
+
if (!lifted) return null
|
|
4173
|
+
const sh = laneMap.get(name) || nm('t' + name.replace(/\W/g, ''))
|
|
4174
|
+
laneMap.set(name, sh)
|
|
4175
|
+
liftedInner.push(['local.set', sh, lifted])
|
|
4176
|
+
}
|
|
4177
|
+
|
|
4178
|
+
// ---- epilogue (obody[>innerIdx]): the per-pixel pack+store, run scalar per lane (bumped to xi+k),
|
|
4179
|
+
// reading the extracted accumulator/lane values. Safety: every in-loop read must be a lane local,
|
|
4180
|
+
// a pixel IV, or written within the epilogue itself. ----
|
|
4181
|
+
const epilogue = obody.slice(innerIdx + 1)
|
|
4182
|
+
{
|
|
4183
|
+
const epiWritten = new Set()
|
|
4184
|
+
const wr = (n) => { if (!isArr(n)) return; const st = (n[0] === 'local.set' || n[0] === 'local.tee') && typeof n[1] === 'string'; if (st) epiWritten.add(n[1]); for (const c of (st ? n.slice(2) : n.slice(1))) wr(c) }
|
|
4185
|
+
epilogue.forEach(wr)
|
|
4186
|
+
const reads = new Set(); const rd = (n) => { if (!isArr(n)) return; if (n[0] === 'local.get') reads.add(n[1]); else for (const c of n) rd(c) }
|
|
4187
|
+
epilogue.forEach(rd)
|
|
4188
|
+
for (const v of reads) if (writesName(loopNode, v) && !laneMap.has(v) && !epiWritten.has(v) && !pivType.has(v)) return null
|
|
4189
|
+
}
|
|
4190
|
+
// store must exist + vary per lane
|
|
4191
|
+
let hasStore = false
|
|
4192
|
+
const findStore = (n) => { if (!isArr(n)) return; if (STORE_OPS[n[0]]) hasStore = true; n.forEach(findStore) }
|
|
4193
|
+
epilogue.forEach(findStore)
|
|
4194
|
+
if (!hasStore) return null
|
|
4195
|
+
|
|
4196
|
+
// ============================ emit ============================
|
|
4197
|
+
const newLocalDecls = [...new Set(laneMap.values())].map(n => ['local', n, 'v128'])
|
|
4198
|
+
const epiReads = [...laneMap.keys()].filter(v => epilogue.some(s => readsVar(s, v)))
|
|
4199
|
+
// Rebuild the inner loop with its scalar scaffold (exit + the bottom IV bump, which lives at
|
|
4200
|
+
// loopNode[incIdx] — NOT in `body`) and the lifted f64x2 body in between.
|
|
4201
|
+
const innerLoopNode = ibl.loopNode
|
|
4202
|
+
const iExit = innerLoopNode[2] // (br_if iBrk (eqz (b < count)))
|
|
4203
|
+
const iInc = innerLoopNode[ibl.incIdx] // (local.set b (i32.add b 1)) — scalar, kept
|
|
4204
|
+
const iLabelB = innerBlock[1], iLabelL = innerLoopNode[1]
|
|
4205
|
+
const innerSimd = ['block', iLabelB, ['loop', iLabelL, iExit, ...liftedInner, iInc, ['br', iLabelL]]]
|
|
4206
|
+
const laneCompute = [...laneInit, innerSimd]
|
|
4207
|
+
const epiLane = (k) => [
|
|
4208
|
+
...epiReads.map(v => ['local.set', v, ['f64x2.extract_lane', k, ['local.get', laneMap.get(v)]]]),
|
|
4209
|
+
...epilogue.map(s => bump(s, k)),
|
|
4210
|
+
]
|
|
4211
|
+
const sOut = nm('ob'), sOl = nm('ol')
|
|
4212
|
+
const simdOuter = ['block', sOut, ['loop', sOl,
|
|
4213
|
+
['br_if', sOut, ['i32.eqz', [oExit.cmpOp, bump(['local.get', pxVar], 1), widthBound]]],
|
|
4214
|
+
...laneCompute, ...epiLane(0), ...epiLane(1),
|
|
4215
|
+
...pixelIVs.map(p => ['local.set', p.name, [p.type + '.add', ['local.get', p.name], [p.type + '.const', 2]]]),
|
|
4216
|
+
['br', sOl]]]
|
|
4217
|
+
const wrapper = ['block', nm('w'), ...preamble, simdOuter, ['block', oLabel, loopNode]]
|
|
4218
|
+
return { wrapper, newLocalDecls }
|
|
4219
|
+
}
|
|
4220
|
+
|
|
4221
|
+
// ---- Mixed-lane tone-map (tryToneMap, experimental) ------------------------
|
|
4222
|
+
//
|
|
4223
|
+
// Vectorizes the log-tonemap TAIL shared by fern / bifurcation / attractors:
|
|
4224
|
+
// while (i<n){ let v=dens[i]; if(v>0){ g = trunc(min(log(v+1)*S, 255)) }
|
|
4225
|
+
// px[i] = (255<<24)|(g<<16)|(g<<8)|g }
|
|
4226
|
+
// A flat 1-D loop that loads an i32 density, lifts it to f64 for a log, truncates
|
|
4227
|
+
// back to i32, packs an ARGB word, and stores it — i32 lanes wrapping an f64 ISLAND.
|
|
4228
|
+
// The single-lane-type lift can't carry an f64 intermediate inside an i32 store
|
|
4229
|
+
// (tryVectorize bails on `f64.mul: no lane-pure SIMD mapping for i32`), so this is a
|
|
4230
|
+
// dedicated 2-wide (f64x2) hybrid: load 2 u32 (`v128.load64_zero` → i32x4 low lanes),
|
|
4231
|
+
// `f64x2.convert_low_i32x4_s` into the island, `$math.log_v` + f64x2 arith + clamp,
|
|
4232
|
+
// `i32x4.trunc_sat_f64x2_s_zero` back out, the i32 pack, then a masked
|
|
4233
|
+
// `i64.store` of `i64x2.extract_lane 0` (the low 2 lanes = 2 pixels). 2 pixels/iter.
|
|
4234
|
+
//
|
|
4235
|
+
// BIT-EXACT by construction: each lane runs the scalar op (log_v is the per-lane
|
|
4236
|
+
// extract/repack mirror; the clamp keeps L finite & in [0,255] so `trunc_sat == |0`,
|
|
4237
|
+
// the ±Inf canon is a no-op and is dropped; the pack is element-wise). The conditional
|
|
4238
|
+
// masks are emitted in the SAME lane width as the data they select (`v>0` is i32 and
|
|
4239
|
+
// gates i32 stores/values; the `L>255` clamp is f64 and gates f64) — a width mismatch
|
|
4240
|
+
// bails. No cross-lane reordering, so no ulp drift. Speculatively-evaluated arms are
|
|
4241
|
+
// trap-free (log/convert/mul/min/trunc never trap; there is no div/rem). Gated until
|
|
4242
|
+
// proven across the corpus, then promoted like the stencil/outer-strip wins.
|
|
4243
|
+
|
|
4244
|
+
const _toneStripTee = (n) => isArr(n) && n[0] === 'local.tee' && n.length === 3 ? n[2] : n
|
|
4245
|
+
|
|
4246
|
+
// `(i32.wrap_i64 (i64.trunc_sat_f64_{s,u} X))` or `(i32.trunc_sat_f64_{s,u} X)` — the
|
|
4247
|
+
// f64→i32 `|0` bridge. Returns { inner, signed } (tee on X stripped) or null.
|
|
4248
|
+
function matchTruncF64(expr) {
|
|
4249
|
+
if (!isArr(expr)) return null
|
|
4250
|
+
if (expr[0] === 'i32.wrap_i64' && isArr(expr[1])) {
|
|
4251
|
+
const t = expr[1]
|
|
4252
|
+
if (t[0] === 'i64.trunc_sat_f64_s') return { inner: _toneStripTee(t[1]), signed: true }
|
|
4253
|
+
if (t[0] === 'i64.trunc_sat_f64_u') return { inner: _toneStripTee(t[1]), signed: false }
|
|
4254
|
+
}
|
|
4255
|
+
if (expr[0] === 'i32.trunc_sat_f64_s') return { inner: _toneStripTee(expr[1]), signed: true }
|
|
4256
|
+
if (expr[0] === 'i32.trunc_sat_f64_u') return { inner: _toneStripTee(expr[1]), signed: false }
|
|
4257
|
+
return null
|
|
4258
|
+
}
|
|
4259
|
+
|
|
4260
|
+
// The `|0` of a known-finite f64: `(select (trunc X) (i32.const 0) (f64.ne X' ±Inf))`.
|
|
4261
|
+
// Since the tonemap clamps L into [0,255] before the trunc, the `≠Inf` guard is always
|
|
4262
|
+
// true, so this lowers to a plain `trunc_sat` — returns the inner f64 to truncate.
|
|
4263
|
+
function matchInfCanonTone(sel) {
|
|
4264
|
+
if (!isArr(sel) || sel[0] !== 'select' || sel.length !== 4) return null
|
|
4265
|
+
if (!(isI32Const(sel[2]) && constNum(sel[2]) === 0)) return null
|
|
4266
|
+
const c = sel[3]
|
|
4267
|
+
if (!(isArr(c) && c[0] === 'f64.ne' && isArr(c[2]) && c[2][0] === 'f64.const' && /inf/i.test(String(c[2][1])))) return null
|
|
4268
|
+
const tr = matchTruncF64(sel[1])
|
|
4269
|
+
return tr ? tr.inner : null
|
|
4270
|
+
}
|
|
4271
|
+
|
|
4272
|
+
// `base + (i<<2)` with `base` a loop-invariant array pointer (local or global). Returns
|
|
4273
|
+
// the base node (stride-4 ⇒ load64_zero/i64.store cover exactly 2 consecutive u32) or null.
|
|
4274
|
+
function matchToneAddr(addr, ind) {
|
|
4275
|
+
if (!isArr(addr) || addr[0] !== 'i32.add' || addr.length !== 3) return null
|
|
4276
|
+
const pair = (baseN, offN) => {
|
|
4277
|
+
if (!isArr(baseN) || (baseN[0] !== 'local.get' && baseN[0] !== 'global.get')) return null
|
|
4278
|
+
if (baseN[0] === 'local.get' && baseN[1] === ind) return null
|
|
4279
|
+
if (isArr(offN) && offN[0] === 'i32.shl' && offN.length === 3 && isLocalGet(offN[1], ind) && constNum(offN[2]) === 2) return baseN
|
|
4280
|
+
return null
|
|
4281
|
+
}
|
|
4282
|
+
return pair(addr[1], addr[2]) || pair(addr[2], addr[1])
|
|
4283
|
+
}
|
|
4284
|
+
|
|
4285
|
+
const _toneUnwrapArm = (arm) => { // arm = ['then'|'else', ...stmts]; unwrap a single nested block
|
|
4286
|
+
let body = arm.slice(1)
|
|
4287
|
+
if (body.length === 1 && isArr(body[0]) && body[0][0] === 'block') {
|
|
4288
|
+
const b = body[0]; let i = 1
|
|
4289
|
+
if (typeof b[i] === 'string' && b[i].startsWith('$')) i++
|
|
4290
|
+
if (isArr(b[i]) && b[i][0] === 'result') i++
|
|
4291
|
+
body = b.slice(i)
|
|
4292
|
+
}
|
|
4293
|
+
return body
|
|
4294
|
+
}
|
|
4295
|
+
|
|
4296
|
+
const _toneAppears = (n, name) => isArr(n) && (((n[0] === 'local.get' || n[0] === 'local.set' || n[0] === 'local.tee') && n[1] === name) || n.some(c => _toneAppears(c, name)))
|
|
4297
|
+
|
|
4298
|
+
// First WRITE of `name`: returns { stmtIdx, nested } (nested = inside an `if`) or null.
|
|
4299
|
+
function _toneFirstWrite(body, name) {
|
|
4300
|
+
for (let i = 0; i < body.length; i++) {
|
|
4301
|
+
let found = false, nested = false
|
|
4302
|
+
const w = (n, depth) => {
|
|
4303
|
+
if (found || !isArr(n)) return
|
|
4304
|
+
if ((n[0] === 'local.set' || n[0] === 'local.tee') && n[1] === name) { found = true; nested = depth > 0; return }
|
|
4305
|
+
const d = depth + (n[0] === 'if' ? 1 : 0)
|
|
4306
|
+
for (let j = 1; j < n.length; j++) w(n[j], d)
|
|
4307
|
+
}
|
|
4308
|
+
w(body[i], 0)
|
|
4309
|
+
if (found) return { stmtIdx: i, nested }
|
|
4310
|
+
}
|
|
4311
|
+
return null
|
|
4312
|
+
}
|
|
4313
|
+
|
|
4314
|
+
// Mixed-lane log-tonemap: i32 dens[i] → f64 log → i32 pack → px[i]. See the block comment above.
|
|
4315
|
+
function tryToneMap(bl, fnLocals, freshIdRef, enabled) {
|
|
4316
|
+
if (!enabled || !bl) return null
|
|
4317
|
+
const { incVar, bound, boundLocal, body, preamble } = bl
|
|
4318
|
+
if (!boundLocal && !isI32Const(bound)) return null // bound must be loop-invariant
|
|
4319
|
+
|
|
4320
|
+
// Shape gate: exactly one i32.store + ≥1 i32.load, all at `base+(i<<2)`, plus the f64
|
|
4321
|
+
// island signature (`f64.convert_i32_*`). Any other-width load/store declines. The
|
|
4322
|
+
// f64-convert requirement is what distinguishes this from a plain i32 map (tryVectorize,
|
|
4323
|
+
// which runs earlier and already owns those).
|
|
4324
|
+
let hasConvert = false, storeCount = 0, loadCount = 0, ok = true
|
|
4325
|
+
const scan = (n) => {
|
|
4326
|
+
if (!ok || !isArr(n)) return
|
|
4327
|
+
const o = n[0]
|
|
4328
|
+
if (o === 'f64.convert_i32_s' || o === 'f64.convert_i32_u') hasConvert = true
|
|
4329
|
+
if (o === 'i32.store') { storeCount++; if (!matchToneAddr(n[1], incVar)) ok = false; scan(n[2]); return }
|
|
4330
|
+
if (o === 'i32.load') { loadCount++; if (!matchToneAddr(n[1], incVar)) ok = false; return }
|
|
4331
|
+
if (LOAD_OPS[o] || STORE_OPS[o]) { ok = false; return } // any wider/narrower memop → not this shape
|
|
4332
|
+
for (let i = 1; i < n.length; i++) scan(n[i])
|
|
4333
|
+
}
|
|
4334
|
+
for (const s of body) scan(s)
|
|
4335
|
+
// 1 store (unconditional, attractors) or 2 (the then/else arms of a conditional store,
|
|
4336
|
+
// bifurcation/fern — both write the same pixel and collapse to one masked store).
|
|
4337
|
+
if (!ok || !hasConvert || storeCount < 1 || storeCount > 2 || loadCount < 1) return null
|
|
4338
|
+
if (body.some(hasGlobalSet)) return null
|
|
4339
|
+
|
|
4340
|
+
// Classify locals (mirrors tryVectorize): written ⇒ lane (first access must be a write,
|
|
4341
|
+
// else loop-carried), unwritten ⇒ invariant.
|
|
4342
|
+
const writes = new Set()
|
|
4343
|
+
for (const s of body) collectWrites(s, writes)
|
|
4344
|
+
if (boundLocal && writes.has(boundLocal)) return null
|
|
4345
|
+
const referenced = new Set()
|
|
4346
|
+
const collectRefs = (n) => {
|
|
4347
|
+
if (!isArr(n)) return
|
|
4348
|
+
if ((n[0] === 'local.get' || n[0] === 'local.set' || n[0] === 'local.tee') && typeof n[1] === 'string') referenced.add(n[1])
|
|
4349
|
+
for (let i = 1; i < n.length; i++) collectRefs(n[i])
|
|
4350
|
+
}
|
|
4351
|
+
for (const s of body) collectRefs(s)
|
|
4352
|
+
const localKind = new Map()
|
|
4353
|
+
for (const name of referenced) {
|
|
4354
|
+
if (name === incVar) continue
|
|
4355
|
+
if (writes.has(name)) {
|
|
4356
|
+
let firstKind = null
|
|
4357
|
+
for (const s of body) { const k = firstAccess(s, name); if (k) { firstKind = k; break } }
|
|
4358
|
+
if (firstKind === 'read') return null // loop-carried
|
|
4359
|
+
localKind.set(name, 'lane')
|
|
4360
|
+
} else localKind.set(name, 'invariant')
|
|
4361
|
+
}
|
|
4362
|
+
|
|
4363
|
+
// Liveness gate: a lane local first ASSIGNED inside an `if` is set speculatively
|
|
4364
|
+
// (unconditionally) — sound only if it never leaks past that statement (else a false
|
|
4365
|
+
// lane would read a value scalar never produced). Bail otherwise.
|
|
4366
|
+
for (const [name, kind] of localKind) {
|
|
4367
|
+
if (kind !== 'lane') continue
|
|
4368
|
+
const fw = _toneFirstWrite(body, name)
|
|
4369
|
+
if (fw && fw.nested) {
|
|
4370
|
+
for (let i = 0; i < body.length; i++) if (i !== fw.stmtIdx && _toneAppears(body[i], name)) return null
|
|
4371
|
+
}
|
|
4372
|
+
}
|
|
4373
|
+
|
|
4374
|
+
const ctx = { extraLocals: [], fail: false, failReason: null } // only liftFail (fail/failReason) + freshMask (extraLocals) read it
|
|
4375
|
+
const newLanedLocals = new Map() // origName → { laneName }
|
|
4376
|
+
const toneSetBefore = new Set() // lane locals already assigned (conditional-merge gate)
|
|
4377
|
+
const laned = (name) => { let r = newLanedLocals.get(name); if (!r) { r = { laneName: `${name}__v` }; newLanedLocals.set(name, r) } return r.laneName }
|
|
4378
|
+
const freshMask = () => { const mt = `$__mask${freshIdRef.next++}`; ctx.extraLocals.push(['local', mt, 'v128']); return mt }
|
|
4379
|
+
|
|
4380
|
+
// The ctx-using lifters are NESTED function declarations that CAPTURE the state above (like
|
|
4381
|
+
// scanForLoadsStores) — taking `ctx` as a param instead would make jz's self-host inference
|
|
4382
|
+
// mistype the recursive lifter's `ctx` (the recursive call site can't agree on i32, so it
|
|
4383
|
+
// stays boxed f64 and its callers emit a bad i64.reinterpret_f64). Capturing sidesteps that.
|
|
4384
|
+
|
|
4385
|
+
// Result lane width of a value expr ('i32' | 'f64' | 'x') — keeps a conditional's mask the
|
|
4386
|
+
// SAME width as the data it selects (a mismatch bails).
|
|
4387
|
+
function toneWidth(e) {
|
|
4388
|
+
if (!isArr(e)) return 'x'
|
|
4389
|
+
const o = e[0]
|
|
4390
|
+
if (o === 'f64.const' || o === 'f64.convert_i32_s' || o === 'f64.convert_i32_u' || o === 'call') return 'f64'
|
|
4391
|
+
if (o === 'i32.const' || o === 'i32.wrap_i64' || o === 'i32.trunc_sat_f64_s' || o === 'i32.trunc_sat_f64_u') return 'i32'
|
|
4392
|
+
if (o === 'local.get') return fnLocals.get(e[1]) === 'f64' ? 'f64' : 'i32'
|
|
4393
|
+
if (o === 'select') return toneWidth(e[1])
|
|
4394
|
+
if (o === 'if') return isArr(e[1]) && e[1][1] === 'f64' ? 'f64' : 'i32'
|
|
4395
|
+
if (o.startsWith('f64.')) return 'f64'
|
|
4396
|
+
if (o.startsWith('i32.')) return 'i32'
|
|
4397
|
+
return 'x'
|
|
4398
|
+
}
|
|
4399
|
+
|
|
4400
|
+
// Lift one value expression to v128. Result lane type comes from the op (f64.* → f64x2,
|
|
4401
|
+
// i32.* → i32x4; convert/trunc bridge between them). Bit-exact per lane.
|
|
4402
|
+
function liftV(expr) {
|
|
4403
|
+
if (!isArr(expr)) return liftFail(ctx, 'tonemap: non-expression operand')
|
|
4404
|
+
const op = expr[0]
|
|
4405
|
+
if ((op === 'f64.convert_i32_s' || op === 'f64.convert_i32_u') && expr.length === 2) {
|
|
4406
|
+
const inner = expr[1]
|
|
4407
|
+
if (isArr(inner) && (inner[0] === 'global.get' || inner[0] === 'i32.const' ||
|
|
4408
|
+
(inner[0] === 'local.get' && localKind.get(inner[1]) === 'invariant')))
|
|
4409
|
+
return ['f64x2.splat', expr] // loop-invariant convert: scalar-then-splat, bit-exact
|
|
4410
|
+
const a = liftV(inner); if (ctx.fail) return null
|
|
4411
|
+
return [op === 'f64.convert_i32_s' ? 'f64x2.convert_low_i32x4_s' : 'f64x2.convert_low_i32x4_u', a]
|
|
4412
|
+
}
|
|
4413
|
+
const tr = matchTruncF64(expr) // f64 → i32 `|0` bridge
|
|
4414
|
+
if (tr) { const a = liftV(tr.inner); if (ctx.fail) return null; return [tr.signed ? 'i32x4.trunc_sat_f64x2_s_zero' : 'i32x4.trunc_sat_f64x2_u_zero', a] }
|
|
4415
|
+
if (op === 'i32.load' && expr.length === 2) return ['v128.load64_zero', expr[1]] // address kept scalar
|
|
4416
|
+
if (op === 'i32.const') return ['i32x4.splat', expr]
|
|
4417
|
+
if (op === 'f64.const') return ['f64x2.splat', expr]
|
|
4418
|
+
if (op === 'local.get' && typeof expr[1] === 'string') {
|
|
4419
|
+
const name = expr[1], kind = localKind.get(name)
|
|
4420
|
+
if (kind === 'lane') return ['local.get', laned(name)]
|
|
4421
|
+
if (kind === 'invariant') return [fnLocals.get(name) === 'f64' ? 'f64x2.splat' : 'i32x4.splat', expr]
|
|
4422
|
+
return liftFail(ctx, `tonemap: ${name} address/induction var used as lane data`)
|
|
4423
|
+
}
|
|
4424
|
+
if (op === 'call' && PPC_CALL2[expr[1]]) { // transcendental → its 2-wide mirror
|
|
4425
|
+
const args = []
|
|
4426
|
+
for (let i = 2; i < expr.length; i++) { const a = liftV(expr[i]); if (ctx.fail) return null; args.push(a) }
|
|
4427
|
+
return ['call', PPC_CALL2[expr[1]], ...args]
|
|
4428
|
+
}
|
|
4429
|
+
if (op === 'select' && expr.length === 4) {
|
|
4430
|
+
const inf = matchInfCanonTone(expr)
|
|
4431
|
+
if (inf) { const a = liftV(inf); if (ctx.fail) return null; return ['i32x4.trunc_sat_f64x2_s_zero', a] }
|
|
4432
|
+
return liftSel(expr[1], expr[2], expr[3])
|
|
4433
|
+
}
|
|
4434
|
+
if (op === 'if' && isArr(expr[1]) && expr[1][0] === 'result' && isArr(expr[3]) && expr[3][0] === 'then' && isArr(expr[4]) && expr[4][0] === 'else')
|
|
4435
|
+
return liftSel(expr[3][1], expr[4][1], expr[2])
|
|
4436
|
+
const insl = op.startsWith('f64.') ? 'f64' : (op.startsWith('i32.') ? 'i32' : 'x')
|
|
4437
|
+
const entry = LANE_PURE[insl]?.get(op)
|
|
4438
|
+
if (entry) {
|
|
4439
|
+
const a = liftV(expr[1]); if (ctx.fail) return null
|
|
4440
|
+
if (entry.shamtScalar) {
|
|
4441
|
+
const b = expr[2]
|
|
4442
|
+
if (!isI32Const(b) && !(isArr(b) && b[0] === 'local.get' && localKind.get(b[1]) === 'invariant'))
|
|
4443
|
+
return liftFail(ctx, `tonemap: ${op}: shift amount not constant/invariant`)
|
|
4444
|
+
return [entry.simd, a, b]
|
|
4445
|
+
}
|
|
4446
|
+
if (expr.length === 2) return [entry.simd, a]
|
|
4447
|
+
const b = liftV(expr[2]); if (ctx.fail) return null
|
|
4448
|
+
return [entry.simd, a, b]
|
|
4449
|
+
}
|
|
4450
|
+
return liftFail(ctx, `tonemap: ${op}: no lane mapping`)
|
|
4451
|
+
}
|
|
4452
|
+
|
|
4453
|
+
// Lane-comparison mask, required to match the selected data width (a LOCAL `c` — never
|
|
4454
|
+
// reassign the param). Returns the mask expr or null on bail.
|
|
4455
|
+
function liftMask(cond, dataTy) {
|
|
4456
|
+
let c = cond
|
|
4457
|
+
if (isArr(c) && c[0] === 'i32.ne' && isI32Const(c[2]) && c[2][1] === 0) c = c[1]
|
|
4458
|
+
if (!isArr(c) || c.length !== 3) return liftFail(ctx, 'tonemap: condition is not a comparison')
|
|
4459
|
+
const condTy = c[0].startsWith('f64.') ? 'f64' : (c[0].startsWith('i32.') ? 'i32' : 'x')
|
|
4460
|
+
if (condTy !== dataTy) return liftFail(ctx, `tonemap: mask width ${condTy} ≠ data width ${dataTy}`)
|
|
4461
|
+
const cmp = LANE_COMPARE[condTy]?.[c[0]]
|
|
4462
|
+
if (!cmp) return liftFail(ctx, `tonemap: ${c[0]}: not a lane comparison`)
|
|
4463
|
+
const ca = liftV(c[1]); if (ctx.fail) return null
|
|
4464
|
+
const cb = liftV(c[2]); if (ctx.fail) return null
|
|
4465
|
+
return [cmp, ca, cb]
|
|
4466
|
+
}
|
|
4467
|
+
|
|
4468
|
+
// `cond ? a : b` → bitselect(a, b, mask(cond)); mask in the branch's lane width.
|
|
4469
|
+
function liftSel(a, b, cond) {
|
|
4470
|
+
const m = liftMask(cond, toneWidth(a)); if (ctx.fail) return null
|
|
4471
|
+
const av = liftV(a); if (ctx.fail) return null
|
|
4472
|
+
const bv = liftV(b); if (ctx.fail) return null
|
|
4473
|
+
const mt = freshMask()
|
|
4474
|
+
return ['block', ['result', 'v128'], ['local.set', mt, m], ['v128.bitselect', av, bv, ['local.get', mt]]]
|
|
4475
|
+
}
|
|
4476
|
+
|
|
4477
|
+
// Lift one statement, pushing v128 stmts into `out`. Sets ctx.fail on any bail.
|
|
4478
|
+
function liftS(stmt, out) {
|
|
4479
|
+
if (!isArr(stmt)) { liftFail(ctx, 'tonemap: non-array statement'); return }
|
|
4480
|
+
const op = stmt[0]
|
|
4481
|
+
if (op === 'block') {
|
|
4482
|
+
let i = 1
|
|
4483
|
+
if (typeof stmt[i] === 'string' && stmt[i].startsWith('$')) i++
|
|
4484
|
+
if (isArr(stmt[i]) && stmt[i][0] === 'result') i++
|
|
4485
|
+
for (const s of stmt.slice(i)) { liftS(s, out); if (ctx.fail) return }
|
|
4486
|
+
return
|
|
4487
|
+
}
|
|
4488
|
+
if (op === 'local.set' && typeof stmt[1] === 'string' && stmt.length === 3) {
|
|
4489
|
+
const name = stmt[1]
|
|
4490
|
+
if (localKind.get(name) !== 'lane') { liftFail(ctx, `tonemap: set of non-lane ${name}`); return }
|
|
4491
|
+
const v = liftV(stmt[2]); if (ctx.fail) return
|
|
4492
|
+
out.push(['local.set', laned(name), v]); toneSetBefore.add(name); return
|
|
4493
|
+
}
|
|
4494
|
+
if (STORE_OPS[op]) { // i32.store ADDR VAL → masked i64.store of the low 2 lanes (2 pixels)
|
|
4495
|
+
if (op !== 'i32.store' || stmt.length !== 3) { liftFail(ctx, `tonemap: unsupported store ${op}`); return }
|
|
4496
|
+
const v = liftV(stmt[2]); if (ctx.fail) return
|
|
4497
|
+
out.push(['i64.store', stmt[1], ['i64x2.extract_lane', 0, v]]); return
|
|
4498
|
+
}
|
|
4499
|
+
if (op === 'if' && isArr(stmt[2]) && stmt[2][0] === 'then') {
|
|
4500
|
+
const hasElse = isArr(stmt[3]) && stmt[3][0] === 'else'
|
|
4501
|
+
const thenStmts = _toneUnwrapArm(stmt[2])
|
|
4502
|
+
const elseStmts = hasElse ? _toneUnwrapArm(stmt[3]) : null
|
|
4503
|
+
const thenLast = thenStmts[thenStmts.length - 1]
|
|
4504
|
+
const elseLast = elseStmts && elseStmts[elseStmts.length - 1]
|
|
4505
|
+
const thenStore = isArr(thenLast) && STORE_OPS[thenLast[0]] && thenLast.length === 3
|
|
4506
|
+
const elseStore = elseLast && isArr(elseLast) && STORE_OPS[elseLast[0]] && elseLast.length === 3
|
|
4507
|
+
// (a) Conditional STORE — both arms (or then-only) end in a store to the same address.
|
|
4508
|
+
if (thenStore && (elseStore || !hasElse)) {
|
|
4509
|
+
if (elseStore && JSON.stringify(thenLast[1]) !== JSON.stringify(elseLast[1])) { liftFail(ctx, 'tonemap: arms store to different addresses'); return }
|
|
4510
|
+
for (const s of thenStmts.slice(0, -1)) { liftS(s, out); if (ctx.fail) return }
|
|
4511
|
+
if (hasElse) for (const s of elseStmts.slice(0, -1)) { liftS(s, out); if (ctx.fail) return }
|
|
4512
|
+
const thenVal = liftV(thenLast[2]); if (ctx.fail) return
|
|
4513
|
+
const elseVal = elseStore ? liftV(elseLast[2]) : ['v128.load64_zero', thenLast[1]]
|
|
4514
|
+
if (ctx.fail) return
|
|
4515
|
+
const m = liftMask(stmt[1], 'i32'); if (ctx.fail) return
|
|
4516
|
+
const mt = freshMask()
|
|
4517
|
+
out.push(['local.set', mt, m],
|
|
4518
|
+
['i64.store', thenLast[1], ['i64x2.extract_lane', 0, ['v128.bitselect', thenVal, elseVal, ['local.get', mt]]]])
|
|
4519
|
+
return
|
|
4520
|
+
}
|
|
4521
|
+
// (b) Conditional VALUE update — `if (cond) { L = …; … }` (no else) updating lane locals.
|
|
4522
|
+
if (!hasElse) {
|
|
4523
|
+
for (const s of thenStmts) {
|
|
4524
|
+
if (!(isArr(s) && s[0] === 'local.set' && typeof s[1] === 'string' && s.length === 3 && localKind.get(s[1]) === 'lane')) {
|
|
4525
|
+
liftFail(ctx, 'tonemap: no-else arm is not a lane update'); return
|
|
4526
|
+
}
|
|
4527
|
+
const name = s[1], ty = fnLocals.get(name) === 'f64' ? 'f64' : 'i32'
|
|
4528
|
+
const xv = liftV(s[2]); if (ctx.fail) return
|
|
4529
|
+
const ln = laned(name)
|
|
4530
|
+
if (toneSetBefore.has(name)) {
|
|
4531
|
+
const m = liftMask(stmt[1], ty); if (ctx.fail) return // conditional merge
|
|
4532
|
+
const mt = freshMask()
|
|
4533
|
+
out.push(['local.set', mt, m], ['local.set', ln, ['v128.bitselect', xv, ['local.get', ln], ['local.get', mt]]])
|
|
4534
|
+
} else {
|
|
4535
|
+
out.push(['local.set', ln, xv]); toneSetBefore.add(name) // first, unconditional (liveness-gated)
|
|
4536
|
+
}
|
|
4537
|
+
}
|
|
4538
|
+
return
|
|
4539
|
+
}
|
|
4540
|
+
liftFail(ctx, 'tonemap: unsupported if shape'); return
|
|
4541
|
+
}
|
|
4542
|
+
liftFail(ctx, `tonemap: unsupported statement ${op}`)
|
|
4543
|
+
}
|
|
4544
|
+
|
|
4545
|
+
const lifted = []
|
|
4546
|
+
for (const s of body) { liftS(s, lifted); if (ctx.fail) return null }
|
|
4547
|
+
if (!lifted.length) return null
|
|
4548
|
+
|
|
4549
|
+
// 2-wide SIMD wrapper (LANES=2, the f64x2 island's width). Scalar tail = original block.
|
|
4550
|
+
const LANES = 2
|
|
4551
|
+
const id = freshIdRef.next++
|
|
4552
|
+
const simdBoundName = `$__simd_bound${id}`, simdBrk = `$__simd_brk${id}`, simdLoop = `$__simd_loop${id}`
|
|
4553
|
+
const boundExpr = boundLocal ? ['local.get', boundLocal] : bound
|
|
4554
|
+
const simdBlock = ['block', simdBrk,
|
|
4555
|
+
['loop', simdLoop,
|
|
4556
|
+
['br_if', simdBrk, ['i32.eqz', ['i32.lt_s', ['local.get', incVar], ['local.get', simdBoundName]]]],
|
|
4557
|
+
...lifted,
|
|
4558
|
+
['local.set', incVar, ['i32.add', ['local.get', incVar], ['i32.const', LANES]]],
|
|
4559
|
+
['br', simdLoop]]]
|
|
4560
|
+
const boundSetup = ['local.set', simdBoundName, ['i32.and', boundExpr, ['i32.const', -LANES]]]
|
|
4561
|
+
const wrapper = ['block', ...preamble.map(cloneNode), boundSetup, simdBlock, bl.blockNode]
|
|
4562
|
+
const newLocalDecls = [
|
|
4563
|
+
['local', simdBoundName, 'i32'],
|
|
4564
|
+
...[...newLanedLocals.values()].map(({ laneName }) => ['local', laneName, 'v128']),
|
|
4565
|
+
...ctx.extraLocals,
|
|
4566
|
+
]
|
|
4567
|
+
return { wrapper, newLocalDecls }
|
|
4568
|
+
}
|
|
4569
|
+
|
|
4570
|
+
// ---- Pass entry ------------------------------------------------------------
|
|
4571
|
+
|
|
4572
|
+
/**
|
|
4573
|
+
* Walk a function looking for vectorizable (block (loop)) pairs, in-place.
|
|
4574
|
+
* Adds new locals to the function header.
|
|
4575
|
+
*/
|
|
4576
|
+
export function vectorizeLaneLocal(fn, multiAcc = false, relaxedFma = false, blurMP = true, whyNot = false, stencil = false, outerStrip = false, pureFuncMap = null, toneMap = false) {
|
|
4577
|
+
if (!isArr(fn) || fn[0] !== 'func') return
|
|
4578
|
+
const bodyStart = findBodyStart(fn)
|
|
4579
|
+
if (bodyStart < 0) return
|
|
4580
|
+
const fnName = typeof fn[1] === 'string' ? fn[1] : '(anon)'
|
|
4581
|
+
let whyNotN = 0
|
|
4582
|
+
|
|
4583
|
+
// Build local-name → wasm-type map.
|
|
4584
|
+
const fnLocals = new Map()
|
|
4585
|
+
for (let i = 2; i < bodyStart; i++) {
|
|
4586
|
+
const d = fn[i]
|
|
4587
|
+
if (isArr(d) && d[0] === 'local' && typeof d[1] === 'string' && typeof d[2] === 'string') {
|
|
4588
|
+
fnLocals.set(d[1], d[2])
|
|
4589
|
+
} else if (isArr(d) && d[0] === 'param' && typeof d[1] === 'string' && typeof d[2] === 'string') {
|
|
4590
|
+
fnLocals.set(d[1], d[2])
|
|
4591
|
+
}
|
|
4592
|
+
}
|
|
4593
|
+
|
|
4594
|
+
const freshIdRef = { next: 0 }
|
|
4595
|
+
const newLocalDeclsAll = []
|
|
4596
|
+
|
|
4597
|
+
vectorizeStraightLineF64DotPairsIn(fn, fnLocals, freshIdRef, newLocalDeclsAll, relaxedFma)
|
|
4598
|
+
|
|
4599
|
+
// Walk body recursively. Process inner-most matches first (post-order)
|
|
4600
|
+
// so we don't try to vectorize an outer loop whose inner is the lane-local one.
|
|
4601
|
+
function walk(parent, idx) {
|
|
4602
|
+
const node = parent[idx]
|
|
4603
|
+
if (!isArr(node)) return
|
|
4604
|
+
for (let i = 0; i < node.length; i++) {
|
|
4605
|
+
if (isArr(node[i])) walk(node, i)
|
|
4606
|
+
}
|
|
4607
|
+
if (node[0] === 'block') {
|
|
4608
|
+
if (_whyNotActive) _whyNotReason = null
|
|
4609
|
+
// Recognition layer: match the canonical (block (loop)) scaffold ONCE; the
|
|
4610
|
+
// inner-scaffold lifters (memcpy/map/reduce/map-reduce/byte-scan) consume the
|
|
4611
|
+
// descriptor instead of each re-matching. The outer-pixel + special-shape
|
|
4612
|
+
// recognizers (divergent-escape, ramp-map, blur, channel-reduce, per-pixel)
|
|
4613
|
+
// do their own matching on the raw node. Order is preserved exactly — it is
|
|
4614
|
+
// load-bearing (first match wins).
|
|
4615
|
+
const bl = matchBlockLoop(node, { allowPreamble: true })
|
|
4616
|
+
let r = tryDivergentEscapeVectorize(node, fnLocals, freshIdRef)
|
|
4617
|
+
?? tryMemCopyFill(bl, fnLocals, freshIdRef)
|
|
4618
|
+
?? tryVectorize(bl, fnLocals, freshIdRef)
|
|
4619
|
+
?? tryReduceVectorize(bl, fnLocals, freshIdRef, multiAcc)
|
|
4620
|
+
?? tryMapReduceVectorize(bl, fnLocals, freshIdRef)
|
|
4621
|
+
?? tryStencil(node, fnLocals, freshIdRef, stencil)
|
|
4622
|
+
?? tryRampMap(node, fnLocals, freshIdRef)
|
|
4623
|
+
?? (blurMP ? tryBlurMultiPixel(node, fnLocals, freshIdRef) : null)
|
|
4624
|
+
?? tryChannelReduce(node, fnLocals, freshIdRef)
|
|
4625
|
+
?? tryByteScan(bl, fnLocals, freshIdRef)
|
|
4626
|
+
?? tryPerPixelColor(node, fnLocals, freshIdRef, pureFuncMap)
|
|
4627
|
+
?? tryOuterStrip(node, fnLocals, freshIdRef, outerStrip)
|
|
4628
|
+
?? tryToneMap(bl, fnLocals, freshIdRef, toneMap)
|
|
4629
|
+
// --why-not-simd: a canonical loop-shaped candidate that no SIMD pass took.
|
|
4630
|
+
// Reported BEFORE the scalar strength-reduce fallback (which fires on most
|
|
4631
|
+
// affine loops and would otherwise mask "didn't vectorize"). Diagnostic only.
|
|
4632
|
+
if (!r && _whyNotActive && (bl || matchOuterPixelLoop(node))) {
|
|
4633
|
+
whyNotN++
|
|
4634
|
+
warn('simd-why-not',
|
|
4635
|
+
`${fnName}: loop #${whyNotN} not vectorized — ${_whyNotReason || 'no SIMD-liftable shape (loop-carried dependency, non-affine address, or unsupported control flow)'}`,
|
|
4636
|
+
{ fn: `${fnName}#${whyNotN}` })
|
|
4637
|
+
}
|
|
4638
|
+
// Scalar IV strength-reduction is a non-SIMD fallback; it may still fire.
|
|
4639
|
+
r = r ?? tryStrengthReduceIV(bl, fnLocals, freshIdRef)
|
|
4640
|
+
if (r) {
|
|
4641
|
+
parent[idx] = r.wrapper
|
|
4642
|
+
newLocalDeclsAll.push(...r.newLocalDecls)
|
|
4643
|
+
}
|
|
4644
|
+
}
|
|
4645
|
+
}
|
|
4646
|
+
_whyNotActive = whyNot
|
|
4647
|
+
for (let i = bodyStart; i < fn.length; i++) walk(fn, i)
|
|
4648
|
+
_whyNotActive = false
|
|
4649
|
+
|
|
4650
|
+
if (newLocalDeclsAll.length) {
|
|
4651
|
+
// Sibling loops (and the straight-line dot pass) can each lift the SAME source
|
|
4652
|
+
// local to an identically-named `$name__v` v128 scratch. Post-order vectorizes
|
|
4653
|
+
// innermost-first and an outer loop bails once its inner became a wrapper block,
|
|
4654
|
+
// so no two NESTED loops ever share a lift — every collision is between
|
|
4655
|
+
// SEQUENTIAL loops, where one shared scratch is correct (each writes its lanes
|
|
4656
|
+
// before reading). Declaring a local twice is invalid wasm ("duplicate local"),
|
|
4657
|
+
// so keep one decl per name (all dups are the identical `['local', name, 'v128']`).
|
|
4658
|
+
fn.splice(bodyStart, 0, ...new Map(newLocalDeclsAll.map(d => [d[1], d])).values())
|
|
4659
|
+
}
|
|
4660
|
+
}
|