jz 0.7.0 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -33
- package/bench/README.md +176 -73
- package/bench/bench.svg +58 -71
- package/cli.js +12 -5
- package/dist/interop.js +1 -1
- package/dist/jz.js +1366 -1101
- package/index.js +40 -9
- package/interop.js +193 -138
- package/layout.js +29 -18
- package/module/array.js +49 -73
- package/module/collection.js +83 -25
- package/module/console.js +1 -1
- package/module/core.js +161 -15
- package/module/json.js +3 -3
- package/module/math.js +167 -117
- package/module/number.js +247 -13
- package/module/object.js +11 -5
- package/module/regex.js +8 -7
- package/module/string.js +295 -171
- package/module/typedarray.js +169 -105
- package/package.json +7 -3
- package/src/abi/string.js +40 -35
- package/src/ast.js +19 -2
- package/src/compile/analyze.js +64 -2
- package/src/compile/cse-load.js +200 -0
- package/src/compile/emit-assign.js +73 -14
- package/src/compile/emit.js +324 -34
- package/src/compile/index.js +204 -61
- package/src/compile/infer.js +8 -1
- package/src/compile/loop-divmod.js +12 -58
- package/src/compile/loop-model.js +91 -0
- package/src/compile/loop-recurrence.js +167 -0
- package/src/compile/loop-square.js +102 -0
- package/src/compile/narrow.js +180 -34
- package/src/compile/peel-stencil.js +18 -64
- package/src/compile/plan/common.js +29 -0
- package/src/compile/plan/index.js +4 -1
- package/src/compile/plan/inline.js +176 -21
- package/src/compile/plan/literals.js +93 -19
- package/src/ctx.js +51 -12
- package/src/helper-counters.js +137 -0
- package/src/ir.js +102 -13
- package/src/kind-traits.js +7 -3
- package/src/kind.js +14 -1
- package/src/op-policy.js +5 -2
- package/src/ops.js +119 -0
- package/src/optimize/index.js +1125 -136
- package/src/optimize/recurse.js +182 -0
- package/src/optimize/vectorize.js +1302 -144
- package/src/prepare/index.js +29 -12
- package/src/reps.js +4 -1
- package/src/type.js +53 -45
- package/src/wat/assemble.js +92 -9
- package/src/widen.js +21 -0
- package/src/wat/optimize.js +0 -3938
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { findBodyStart } from '../ir.js'
|
|
2
|
-
import { warn } from '../ctx.js'
|
|
2
|
+
import { warn, ctx } from '../ctx.js'
|
|
3
|
+
import { nodeEqual as exprEq } from '../ast.js'
|
|
3
4
|
|
|
4
5
|
/**
|
|
5
6
|
* Lane-local SIMD-128 vectorizer.
|
|
@@ -48,7 +49,9 @@ import { warn } from '../ctx.js'
|
|
|
48
49
|
|
|
49
50
|
const isArr = n => Array.isArray(n)
|
|
50
51
|
|
|
51
|
-
|
|
52
|
+
// Structural node equality — must be non-finite- AND bigint-safe: plain
|
|
53
|
+
// JSON.stringify maps Infinity/-Infinity/NaN→null and -0→0, so it would equate a
|
|
54
|
+
// `[Inf,-Inf]` lane pair and splat it (dropping -Inf). nodeEqual tags those.
|
|
52
55
|
const localGetName = n => isArr(n) && n[0] === 'local.get' && typeof n[1] === 'string' ? n[1] : null
|
|
53
56
|
const f64Zero = n => isArr(n) && n[0] === 'f64.const' && Number(n[1]) === 0
|
|
54
57
|
|
|
@@ -67,9 +70,16 @@ const isSplatConst = (n, constOp) =>
|
|
|
67
70
|
function matchCanonSelect(sel, laneType) {
|
|
68
71
|
if (!isArr(sel) || sel[0] !== 'select') return null
|
|
69
72
|
const C = sel[1], val = sel[2], cond = sel[3]
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
+
// f32 lane: jz computes the value in f64, so `Math.min/max` (and any NaN-canon'd
|
|
74
|
+
// f32 result) emit the canon with `f64.ne` + an f64 NaN const. Accept that
|
|
75
|
+
// alongside the native f32 form; liftCanon splats the const as f32.
|
|
76
|
+
const f64Canon = laneType === 'f32' && isArr(C) && C[0] === 'f64.const' && isArr(cond) && cond[0] === 'f64.ne'
|
|
77
|
+
if (!f64Canon) {
|
|
78
|
+
const neOp = laneType === 'f32' ? 'f32.ne' : 'f64.ne'
|
|
79
|
+
if (!isSplatConst(C, LANE_INFO[laneType].constOp)) return null
|
|
80
|
+
if (!(isArr(cond) && cond[0] === neOp)) return null
|
|
81
|
+
}
|
|
82
|
+
if (!(exprEq(cond[1], val) && exprEq(cond[2], val))) return null
|
|
73
83
|
return { val, C }
|
|
74
84
|
}
|
|
75
85
|
|
|
@@ -93,10 +103,11 @@ function normTee(n) {
|
|
|
93
103
|
// RESULT — equal operands tie to the same value — so only the direction axis matters.
|
|
94
104
|
function matchIntMinMaxReduce(rhs, accName) {
|
|
95
105
|
if (!isArr(rhs)) return null
|
|
96
|
-
let cond, T, E
|
|
106
|
+
let cond, T, E, resTy = null
|
|
97
107
|
if (rhs[0] === 'if') {
|
|
98
108
|
let i = 1
|
|
99
|
-
if (!(isArr(rhs[i]) && rhs[i][0] === 'result' && rhs[i][1] === 'i32')) return null
|
|
109
|
+
if (!(isArr(rhs[i]) && rhs[i][0] === 'result' && (rhs[i][1] === 'i32' || rhs[i][1] === 'f64'))) return null
|
|
110
|
+
resTy = rhs[i][1]
|
|
100
111
|
i++
|
|
101
112
|
if (rhs.length !== i + 3) return null
|
|
102
113
|
cond = rhs[i]
|
|
@@ -116,8 +127,12 @@ function matchIntMinMaxReduce(rhs, accName) {
|
|
|
116
127
|
let cmp = cond
|
|
117
128
|
if (isArr(cmp) && cmp[0] === 'i32.ne' && isI32Const(cmp[2]) && cmp[2][1] === 0) cmp = cmp[1]
|
|
118
129
|
if (!isArr(cmp) || cmp.length !== 3) return null
|
|
119
|
-
|
|
130
|
+
// Integer (i32x4.max_s, exact) or float (f64x2.pmax, exact per-element incl NaN/±0) compare.
|
|
131
|
+
const dir = { 'i32.gt_s': 'gt', 'i32.ge_s': 'gt', 'i32.lt_s': 'lt', 'i32.le_s': 'lt',
|
|
132
|
+
'f64.gt': 'gt', 'f64.ge': 'gt', 'f64.lt': 'lt', 'f64.le': 'lt' }[cmp[0]]
|
|
120
133
|
if (!dir) return null
|
|
134
|
+
const laneType = cmp[0].startsWith('f64.') ? 'f64' : 'i32'
|
|
135
|
+
if (resTy != null && resTy !== laneType) return null // if-form result type must agree with the compare
|
|
121
136
|
// Comparison operands must be {acc, EXPR}; take the non-acc side as the canonical lane
|
|
122
137
|
// expr (it carries the address tee). exprIsLeftOfCmp records its position.
|
|
123
138
|
let condExpr, exprIsLeftOfCmp
|
|
@@ -128,7 +143,7 @@ function matchIntMinMaxReduce(rhs, accName) {
|
|
|
128
143
|
if (!exprEq(normTee(condExpr), normTee(exprBr))) return null
|
|
129
144
|
// cond true ⟺ EXPR > acc ⇒ picking EXPR-when-true is a max; picking-when-false a min.
|
|
130
145
|
const predExprGreater = dir === 'gt' ? exprIsLeftOfCmp : !exprIsLeftOfCmp
|
|
131
|
-
return { exprNode: condExpr, isMax: takeExprWhenTrue === predExprGreater }
|
|
146
|
+
return { exprNode: condExpr, isMax: takeExprWhenTrue === predExprGreater, laneType }
|
|
132
147
|
}
|
|
133
148
|
|
|
134
149
|
// Match the un-flattened canon, emitted when a Math.* result feeds another op
|
|
@@ -175,19 +190,24 @@ const matchDotStore = (n, acc) => {
|
|
|
175
190
|
return null
|
|
176
191
|
}
|
|
177
192
|
|
|
193
|
+
// Unroll width this dot-product recognizer expects: a `acc=0` reset, exactly this
|
|
194
|
+
// many `acc += L[k]*R[k]` steps, then the store. Tied to the emitter's 4-wide dot
|
|
195
|
+
// unroll — matchDotStore / f64x2Pair / dotPairExpr below are hardwired to it.
|
|
196
|
+
const DOT_UNROLL = 4
|
|
197
|
+
|
|
178
198
|
const matchF64DotSeq = (stmts, i) => {
|
|
179
199
|
const reset = stmts[i]
|
|
180
200
|
if (!isArr(reset) || reset[0] !== 'local.set' || typeof reset[1] !== 'string' || !f64Zero(reset[2])) return null
|
|
181
201
|
const acc = reset[1]
|
|
182
202
|
const left = [], right = []
|
|
183
|
-
for (let k = 0; k <
|
|
203
|
+
for (let k = 0; k < DOT_UNROLL; k++) {
|
|
184
204
|
const pair = matchAccumStep(stmts[i + 1 + k], acc)
|
|
185
205
|
if (!pair) return null
|
|
186
206
|
left.push(pair[0])
|
|
187
207
|
right.push(pair[1])
|
|
188
208
|
}
|
|
189
|
-
const store = matchDotStore(stmts[i +
|
|
190
|
-
return store ? { end: i +
|
|
209
|
+
const store = matchDotStore(stmts[i + 1 + DOT_UNROLL], acc)
|
|
210
|
+
return store ? { end: i + 2 + DOT_UNROLL, acc, left, right, ...store } : null
|
|
191
211
|
}
|
|
192
212
|
|
|
193
213
|
const f64x2Pair = (lo, hi) => ['f64x2.replace_lane', 1, ['f64x2.splat', ['local.get', lo]], ['local.get', hi]]
|
|
@@ -243,7 +263,7 @@ const vectorizeStraightLineF64DotPairsIn = (node, fnLocals, freshIdRef, newLocal
|
|
|
243
263
|
addend = ['local.get', tmp]
|
|
244
264
|
}
|
|
245
265
|
const pairs = []
|
|
246
|
-
for (let k = 0; k <
|
|
266
|
+
for (let k = 0; k < DOT_UNROLL; k++) {
|
|
247
267
|
const key = `${a.right[k]}\0${b.right[k]}`
|
|
248
268
|
let tmp = pairTemps.get(key)
|
|
249
269
|
if (!tmp) {
|
|
@@ -267,6 +287,270 @@ const vectorizeStraightLineF64DotPairsIn = (node, fnLocals, freshIdRef, newLocal
|
|
|
267
287
|
}
|
|
268
288
|
}
|
|
269
289
|
|
|
290
|
+
// =============================================================================
|
|
291
|
+
// Loop-invariant partial-product hoist for unrolled f64 dot reductions.
|
|
292
|
+
//
|
|
293
|
+
// A fully-unrolled inner reduction over scalar-replaced array cells (mat4's
|
|
294
|
+
// `out[r][c] = Σ a[r][k]·b[k][c]`) lives in the body of an OUTER loop that
|
|
295
|
+
// mutates only a few of those cells (mat4: a[0],a[5],b[0],b[5]). Every product
|
|
296
|
+
// whose two operands are both outer-loop-invariant is therefore the SAME every
|
|
297
|
+
// iteration — yet the body recomputes all of them. rust/LLVM precomputes those
|
|
298
|
+
// invariant partials in a loop prologue (mat4.rs → ~294 lines before its loop);
|
|
299
|
+
// V8/wasmtime/JSC cannot, because at the wasm level they can't prove the cells
|
|
300
|
+
// are loop-invariant (no aliasing model). So jz must hoist them itself.
|
|
301
|
+
//
|
|
302
|
+
// Splitting `s = t0+t1+t2+t3` into `INV = Σ(invariant tk)` (hoisted) + `Σ(variant
|
|
303
|
+
// tk)` (kept) REASSOCIATES the float sum — invariant terms are summed first,
|
|
304
|
+
// regardless of original position — so results differ by ULPs from the strict
|
|
305
|
+
// left-to-right order. That is the SAME class of reorder jz already ships for
|
|
306
|
+
// horizontal/multi-accumulator reductions (policy at lines ~620 and ~1584), and
|
|
307
|
+
// rust itself does it at -O3 without fast-math. Gated to the relaxedFma/speed
|
|
308
|
+
// tier exactly like those, so strict opts keep bit-exact order.
|
|
309
|
+
//
|
|
310
|
+
// Surgical by construction: fires only on a dot that MIXES invariant and variant
|
|
311
|
+
// terms inside a loop. A pure-variant dot (a real matmul kernel, every operand
|
|
312
|
+
// streaming) has no invariant term → untouched. Runs BEFORE the dot-pair
|
|
313
|
+
// vectorizer; a hoisted dot has < DOT_UNROLL accumulate steps so matchF64DotSeq
|
|
314
|
+
// no longer matches it — it stays the (faster here) scalar form, like rust.
|
|
315
|
+
const hoistDotInvariant = (loop, parent, idx, fnLocals, freshIdRef, newLocalDecls) => {
|
|
316
|
+
const writeSet = new Set()
|
|
317
|
+
collectWrites(loop, writeSet)
|
|
318
|
+
const isInv = name => typeof name === 'string' && !writeSet.has(name) && fnLocals.get(name) === 'f64'
|
|
319
|
+
const invInits = []
|
|
320
|
+
const processList = (list) => {
|
|
321
|
+
for (let i = 0; i < list.length;) {
|
|
322
|
+
const seq = matchF64DotSeq(list, i)
|
|
323
|
+
if (!seq) { i++; continue }
|
|
324
|
+
const invKs = [], varKs = []
|
|
325
|
+
for (let k = 0; k < DOT_UNROLL; k++) (isInv(seq.left[k]) && isInv(seq.right[k]) ? invKs : varKs).push(k)
|
|
326
|
+
if (invKs.length === 0) { i = seq.end; continue } // nothing loop-invariant — leave for the vectorizer
|
|
327
|
+
// INV = Σ invariant products, in original k-order, computed once before the loop.
|
|
328
|
+
let inv = ['f64.const', '0']
|
|
329
|
+
for (const k of invKs) inv = ['f64.add', inv, ['f64.mul', ['local.get', seq.left[k]], ['local.get', seq.right[k]]]]
|
|
330
|
+
const invName = `$__rinv_${freshIdRef.next++}`
|
|
331
|
+
newLocalDecls.push(['local', invName, 'f64']); fnLocals.set(invName, 'f64')
|
|
332
|
+
invInits.push(['local.set', invName, inv])
|
|
333
|
+
// In-loop: seed acc with INV, add only the variant products, then the unchanged store.
|
|
334
|
+
const repl = [['local.set', seq.acc, ['local.get', invName]]]
|
|
335
|
+
for (const k of varKs) repl.push(['local.set', seq.acc, ['f64.add', ['local.get', seq.acc], ['f64.mul', ['local.get', seq.left[k]], ['local.get', seq.right[k]]]]])
|
|
336
|
+
repl.push(['local.set', seq.out, seq.addend ? ['f64.add', ['local.get', seq.acc], seq.addend] : ['local.get', seq.acc]])
|
|
337
|
+
list.splice(i, seq.end - i, ...repl)
|
|
338
|
+
i += repl.length
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
const scan = (n) => { if (!isArr(n)) return; processList(n); for (let j = 0; j < n.length; j++) if (isArr(n[j])) scan(n[j]) }
|
|
342
|
+
scan(loop)
|
|
343
|
+
if (invInits.length) parent.splice(idx, 0, ...invInits)
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
// Walk a function, hoisting invariant reduction partials out of each loop. Inner
|
|
347
|
+
// loops first (post-order) so a dot is hoisted relative to its tightest enclosing
|
|
348
|
+
// loop, and an already-rewritten dot can't re-match in an outer pass.
|
|
349
|
+
const hoistReductionInvariantsIn = (fn, fnLocals, freshIdRef, newLocalDecls) => {
|
|
350
|
+
const walk = (node, parent, idx) => {
|
|
351
|
+
if (!isArr(node)) return
|
|
352
|
+
for (let i = 0; i < node.length; i++) if (isArr(node[i])) walk(node[i], node, i)
|
|
353
|
+
if (node[0] === 'loop' && isArr(parent)) hoistDotInvariant(node, parent, idx, fnLocals, freshIdRef, newLocalDecls)
|
|
354
|
+
}
|
|
355
|
+
for (let i = 0; i < fn.length; i++) if (isArr(fn[i])) walk(fn[i], fn, i)
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
// =============================================================================
|
|
359
|
+
// SLP (superword-level parallelism): pack two ADJACENT isomorphic f64 element
|
|
360
|
+
// stores into one f64x2 store — the WITHIN-iteration 2-lane class the loop
|
|
361
|
+
// vectorizer (which packs ACROSS iterations) structurally cannot reach.
|
|
362
|
+
//
|
|
363
|
+
// Soundness has TWO obligations, because the pack reorders memory: it materializes
|
|
364
|
+
// BOTH lane values BEFORE either store, turning [read0, write0, read1, write1] into
|
|
365
|
+
// [read0, read1, write0, write1].
|
|
366
|
+
// 1. CROSS-base aliasing — guarded by one module fact: no typed-array VIEW exists
|
|
367
|
+
// (`ctx.features.typedView` false, checked at the dispatch). A view (subarray /
|
|
368
|
+
// buffer-backed ctor) is the only way two DISTINCT typed bases can overlap;
|
|
369
|
+
// without one, distinct bases own disjoint allocations and can't alias.
|
|
370
|
+
// 2. WITHIN-base read-after-write — the high value (read1) must not load the low
|
|
371
|
+
// store's target (write0), or the pack reads write0's PRE-store value. This is a
|
|
372
|
+
// same-base hazard a view gate can't see (`o[k+1]=o[k]; o[k+2]=o[k+1]` forward
|
|
373
|
+
// shift); slpReadsOffset rejects it. The sound own-index map reads its OWN offset,
|
|
374
|
+
// never the sibling's, so it survives.
|
|
375
|
+
// The pack is admitted ONLY when overhead-free (adjacent loads → v128.load,
|
|
376
|
+
// identical pure scalar → splat, matching op → recurse); anything that would
|
|
377
|
+
// need a per-lane `replace_lane` build bails, which makes the rewrite both
|
|
378
|
+
// PROFITABLE and unable to grow code. Every f64x2 lane op is bit-identical to its
|
|
379
|
+
// scalar f64 op (IEEE element-wise), so the result is byte-equal to the scalar form.
|
|
380
|
+
// =============================================================================
|
|
381
|
+
const F64X2_BIN = { 'f64.add': 'f64x2.add', 'f64.sub': 'f64x2.sub', 'f64.mul': 'f64x2.mul', 'f64.div': 'f64x2.div', 'f64.min': 'f64x2.min', 'f64.max': 'f64x2.max' }
|
|
382
|
+
const F64X2_UN = { 'f64.neg': 'f64x2.neg', 'f64.abs': 'f64x2.abs', 'f64.sqrt': 'f64x2.sqrt' }
|
|
383
|
+
|
|
384
|
+
// The subtree's value is the SAME evaluated once (splat) or twice (the two source
|
|
385
|
+
// statements, which are adjacent — no store/reassign between): a pure, side-effect-free,
|
|
386
|
+
// DETERMINISTIC expression. Rejects calls (a `new TypedArray()` alloc returns a fresh
|
|
387
|
+
// pointer per call — splatting it would make the two lanes ALIAS, the array-literal
|
|
388
|
+
// scatter miscompile), loads, and any store/set/memory op.
|
|
389
|
+
const slpSplatSafe = (n) => {
|
|
390
|
+
if (!isArr(n)) return true
|
|
391
|
+
const op = n[0]
|
|
392
|
+
if (typeof op !== 'string') return false
|
|
393
|
+
if (op.startsWith('call') || op.includes('.load') || op.includes('.store')
|
|
394
|
+
|| op === 'local.set' || op === 'local.tee' || op === 'global.set'
|
|
395
|
+
|| op.startsWith('memory.') || op.includes('.atomic.')) return false
|
|
396
|
+
for (let i = 1; i < n.length; i++) if (!slpSplatSafe(n[i])) return false
|
|
397
|
+
return true
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
// Decompose a load/store node, normalizing the optional `offset=K` attribute jz
|
|
401
|
+
// folds adjacent accesses into: `(op addr …)` → off 0, `(op offset=K addr …)` → K.
|
|
402
|
+
const slpMem = (n) => {
|
|
403
|
+
if (!isArr(n)) return null
|
|
404
|
+
if (typeof n[1] === 'string' && n[1].startsWith('offset=')) return { off: +n[1].slice(7), addr: n[2], val: n[3] }
|
|
405
|
+
return { off: 0, addr: n[1], val: n[2] }
|
|
406
|
+
}
|
|
407
|
+
// The two accesses (x = low/first, y = high/second) provably address the SAME base
|
|
408
|
+
// pointer. Sound shapes only — `y` must read what `x` produced, never redefine it:
|
|
409
|
+
// • x = (local.tee $X e), y = (local.get $X) — x defines the shared ptr, y reuses it
|
|
410
|
+
// • x = (local.get $X), y = (local.get $X) — both read the same already-set local
|
|
411
|
+
// • exprEq(x, y) with NEITHER a tee — identical side-effect-free addresses
|
|
412
|
+
// REJECTS `(local.tee $X eA), (local.tee $X eB)` (y redefines $X to a different address
|
|
413
|
+
// → the high lane would write the wrong place) and `(get $X), (tee $X e)` — the watr.js
|
|
414
|
+
// self-host miscompile came from accepting those by name alone.
|
|
415
|
+
const slpSameBase = (x, y) => {
|
|
416
|
+
if (!isArr(x) || !isArr(y)) return false
|
|
417
|
+
if (x[0] === 'local.tee' && y[0] === 'local.get' && typeof x[1] === 'string' && x[1] === y[1]) return true
|
|
418
|
+
if (x[0] === 'local.get' && y[0] === 'local.get' && typeof x[1] === 'string' && x[1] === y[1]) return true
|
|
419
|
+
return x[0] !== 'local.tee' && y[0] !== 'local.tee' && exprEq(x, y)
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
// Two address expressions name the SAME pointer base. Symmetric, tee/get-normalized
|
|
423
|
+
// (a `local.tee $X` defines what a `local.get $X` reads, so they're one base); else
|
|
424
|
+
// structural exprEq. Used by the RAW guard below — under the no-view gate, a different
|
|
425
|
+
// base is a different allocation, so "not same base" ⇒ provably disjoint memory.
|
|
426
|
+
const slpSameMem = (a, b) => {
|
|
427
|
+
if (!isArr(a) || !isArr(b)) return false
|
|
428
|
+
const an = (a[0] === 'local.tee' || a[0] === 'local.get') && typeof a[1] === 'string' ? a[1] : null
|
|
429
|
+
const bn = (b[0] === 'local.tee' || b[0] === 'local.get') && typeof b[1] === 'string' ? b[1] : null
|
|
430
|
+
if (an !== null || bn !== null) return an === bn
|
|
431
|
+
return exprEq(a, b)
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
// Does `value` load the element at (addr, off) — the slot u0 stores to? SLP materializes
|
|
435
|
+
// BOTH packed values BEFORE either store, so if the high store's VALUE reads the low
|
|
436
|
+
// store's TARGET, the original (which stored low first) and the pack (which reads low's
|
|
437
|
+
// OLD value) diverge — a within-iteration read-after-write hazard. `o[k+1]=o[k]; o[k+2]=
|
|
438
|
+
// o[k+1]` is the canonical miscompile: the second value reads o[k+1], which the first
|
|
439
|
+
// store just wrote. (The sound own-index map `o[i]=…; o[i+1]=…` reads u1's OWN offset,
|
|
440
|
+
// never u0's, so it never trips this.) f64 accesses are 8-byte and 8-aligned, so two
|
|
441
|
+
// overlap iff their offsets are equal.
|
|
442
|
+
const slpReadsOffset = (value, addr, off) => {
|
|
443
|
+
let hit = false
|
|
444
|
+
const walk = (n) => {
|
|
445
|
+
if (hit || !isArr(n)) return
|
|
446
|
+
if (n[0] === 'f64.load') {
|
|
447
|
+
const m = slpMem(n)
|
|
448
|
+
if (m && m.off === off && slpSameMem(m.addr, addr)) { hit = true; return }
|
|
449
|
+
}
|
|
450
|
+
for (let i = 1; i < n.length; i++) walk(n[i])
|
|
451
|
+
}
|
|
452
|
+
walk(value)
|
|
453
|
+
return hit
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
// Pack two isomorphic f64 trees [lo, hi] into an f64x2 value, or null if it isn't
|
|
457
|
+
// overhead-free (adjacent loads → v128.load, identical pure scalar → splat, matching
|
|
458
|
+
// op → recurse). The overhead-free restriction is what makes it both profitable and
|
|
459
|
+
// unable to grow code; every f64x2 lane op is bit-identical to its scalar f64 op.
|
|
460
|
+
const slpPackF64x2 = (lo, hi) => {
|
|
461
|
+
if (!isArr(lo) || !isArr(hi)) return null
|
|
462
|
+
if (lo[0] === 'f64.load' && hi[0] === 'f64.load') {
|
|
463
|
+
const a = slpMem(lo), b = slpMem(hi)
|
|
464
|
+
if (b.off - a.off !== 8 || !slpSameBase(a.addr, b.addr)) return null
|
|
465
|
+
return a.off ? ['v128.load', `offset=${a.off}`, a.addr] : ['v128.load', a.addr]
|
|
466
|
+
}
|
|
467
|
+
if (exprEq(lo, hi) && slpSplatSafe(lo)) return ['f64x2.splat', lo]
|
|
468
|
+
if (lo[0] === hi[0]) {
|
|
469
|
+
const bin = F64X2_BIN[lo[0]]
|
|
470
|
+
if (bin && lo.length === 3 && hi.length === 3) {
|
|
471
|
+
const x = slpPackF64x2(lo[1], hi[1]); if (!x) return null
|
|
472
|
+
const y = slpPackF64x2(lo[2], hi[2]); return y ? [bin, x, y] : null
|
|
473
|
+
}
|
|
474
|
+
const un = F64X2_UN[lo[0]]
|
|
475
|
+
if (un && lo.length === 2 && hi.length === 2) {
|
|
476
|
+
const x = slpPackF64x2(lo[1], hi[1]); return x ? [un, x] : null
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
return null
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
// Resolve the element store at `stmts[i]` to { off, addr, value, lo, hi } — the f64
|
|
483
|
+
// value to pack and the inclusive statement span it occupies. jz emits an element
|
|
484
|
+
// store in three shapes, all handled here so SLP fires both pre- and post-watr:
|
|
485
|
+
// • inline (f64.store addr V) → span [i,i]
|
|
486
|
+
// • flat tee'd (local.set $t V) ; (f64.store addr (local.get $t)) → span [i-1,i]
|
|
487
|
+
// • block-wrapped (block (local.set $t V) (f64.store addr (local.get $t)))
|
|
488
|
+
// The tee'd value to pack is V (the definition), not the `(local.get $t)`.
|
|
489
|
+
const slpUnitAt = (stmts, i, getCounts) => {
|
|
490
|
+
const s = stmts[i]
|
|
491
|
+
if (!isArr(s)) return null
|
|
492
|
+
if (s[0] === 'block' && s.length === 3
|
|
493
|
+
&& isArr(s[1]) && s[1][0] === 'local.set' && isArr(s[2]) && s[2][0] === 'f64.store') {
|
|
494
|
+
const m = slpMem(s[2])
|
|
495
|
+
if (m && isArr(m.val) && m.val[0] === 'local.get' && m.val[1] === s[1][1]) return { off: m.off, addr: m.addr, value: s[1][2], lo: i, hi: i }
|
|
496
|
+
return null
|
|
497
|
+
}
|
|
498
|
+
if (s[0] !== 'f64.store') return null
|
|
499
|
+
const m = slpMem(s)
|
|
500
|
+
if (!m) return null
|
|
501
|
+
// Flat tee'd: `(local.set $t V) ; (f64.store … (local.get $t))`. Resolving the value
|
|
502
|
+
// to V and dropping the set is sound ONLY if $t is used nowhere else — otherwise a
|
|
503
|
+
// later `(local.get $t)` reads a value we deleted (the watr.js self-host miscompile).
|
|
504
|
+
if (isArr(m.val) && m.val[0] === 'local.get' && typeof m.val[1] === 'string'
|
|
505
|
+
&& i > 0 && isArr(stmts[i - 1]) && stmts[i - 1][0] === 'local.set' && stmts[i - 1][1] === m.val[1]
|
|
506
|
+
&& getCounts.get(m.val[1]) === 1)
|
|
507
|
+
return { off: m.off, addr: m.addr, value: stmts[i - 1][2], lo: i - 1, hi: i }
|
|
508
|
+
return { off: m.off, addr: m.addr, value: m.val, lo: i, hi: i }
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
// Count `(local.get NAME)` occurrences across the function, so the flat-tee'd
|
|
512
|
+
// resolution above can confirm a store value's temp is single-use before removing it.
|
|
513
|
+
const slpGetCounts = (fn) => {
|
|
514
|
+
const counts = new Map()
|
|
515
|
+
const walk = (n) => {
|
|
516
|
+
if (!isArr(n)) return
|
|
517
|
+
if (n[0] === 'local.get' && typeof n[1] === 'string') counts.set(n[1], (counts.get(n[1]) || 0) + 1)
|
|
518
|
+
for (let i = 1; i < n.length; i++) walk(n[i])
|
|
519
|
+
}
|
|
520
|
+
walk(fn)
|
|
521
|
+
return counts
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
// Rewrite two back-to-back element stores one f64 apart with isomorphic values into a
|
|
525
|
+
// single v128 store. The packed value is computed into a fresh v128 local FIRST, then
|
|
526
|
+
// stored — preserving jz's value-before-address evaluation order (the store address can
|
|
527
|
+
// read a `local.tee` the value defines, e.g. the shared `i<<3` offset). base is the LOW
|
|
528
|
+
// store's address (its tee that defines the shared pointer is kept); the high store +
|
|
529
|
+
// its value dissolve into the high lane. Sound only under the no-view gate at dispatch.
|
|
530
|
+
const slpStorePairsIn = (node, fnLocals, freshIdRef, newLocalDecls, getCounts) => {
|
|
531
|
+
if (!isArr(node)) return
|
|
532
|
+
for (let i = 0; i < node.length; i++) if (isArr(node[i])) slpStorePairsIn(node[i], fnLocals, freshIdRef, newLocalDecls, getCounts)
|
|
533
|
+
for (let i = 0; i < node.length; i++) {
|
|
534
|
+
const u0 = slpUnitAt(node, i, getCounts)
|
|
535
|
+
if (!u0) continue
|
|
536
|
+
const u1 = slpUnitAt(node, u0.hi + 1, getCounts)
|
|
537
|
+
if (!u1 || u1.lo !== u0.hi + 1) continue
|
|
538
|
+
if (u1.off - u0.off !== 8 || !slpSameBase(u0.addr, u1.addr)) continue
|
|
539
|
+
// RAW hazard: the high store's value must not read the low store's target — the pack
|
|
540
|
+
// would read its pre-store value. (u0 writes u0.off; reject if u1.value loads it.)
|
|
541
|
+
if (slpReadsOffset(u1.value, u0.addr, u0.off)) continue
|
|
542
|
+
const packed = slpPackF64x2(u0.value, u1.value)
|
|
543
|
+
if (!packed) continue
|
|
544
|
+
const t = `$__slp${freshIdRef.next++}`
|
|
545
|
+
newLocalDecls.push(['local', t, 'v128']); fnLocals.set(t, 'v128')
|
|
546
|
+
const store = u0.off
|
|
547
|
+
? ['v128.store', `offset=${u0.off}`, u0.addr, ['local.get', t]]
|
|
548
|
+
: ['v128.store', u0.addr, ['local.get', t]]
|
|
549
|
+
node.splice(u0.lo, u1.hi - u0.lo + 1, ['local.set', t, packed], store)
|
|
550
|
+
i = u0.lo
|
|
551
|
+
}
|
|
552
|
+
}
|
|
553
|
+
|
|
270
554
|
// ---- Lane type tables ------------------------------------------------------
|
|
271
555
|
|
|
272
556
|
const LANE_INFO = {
|
|
@@ -352,6 +636,13 @@ const LANE_PURE = {
|
|
|
352
636
|
['f32.neg', { simd: 'f32x4.neg' }],
|
|
353
637
|
['f32.abs', { simd: 'f32x4.abs' }],
|
|
354
638
|
['f32.sqrt', { simd: 'f32x4.sqrt' }],
|
|
639
|
+
// rounding: each f32x4.* rounds lane-for-lane identically to the scalar f32.* (same
|
|
640
|
+
// IEEE rounding mode), so the lift is bit-exact. Math.floor/ceil/trunc and the bare
|
|
641
|
+
// f64.nearest jz emits all reach here in a Float32Array kernel.
|
|
642
|
+
['f32.floor', { simd: 'f32x4.floor' }],
|
|
643
|
+
['f32.ceil', { simd: 'f32x4.ceil' }],
|
|
644
|
+
['f32.trunc', { simd: 'f32x4.trunc' }],
|
|
645
|
+
['f32.nearest', { simd: 'f32x4.nearest' }],
|
|
355
646
|
]),
|
|
356
647
|
f64: new Map([
|
|
357
648
|
['f64.add', { simd: 'f64x2.add' }],
|
|
@@ -363,9 +654,37 @@ const LANE_PURE = {
|
|
|
363
654
|
['f64.neg', { simd: 'f64x2.neg' }],
|
|
364
655
|
['f64.abs', { simd: 'f64x2.abs' }],
|
|
365
656
|
['f64.sqrt', { simd: 'f64x2.sqrt' }],
|
|
657
|
+
// rounding: f64x2.* rounds each lane identically to the scalar f64.* op (same IEEE
|
|
658
|
+
// mode), so bit-exact. Unblocks `out[i] = Math.floor/ceil/trunc(f(in[i]))` f64 maps.
|
|
659
|
+
['f64.floor', { simd: 'f64x2.floor' }],
|
|
660
|
+
['f64.ceil', { simd: 'f64x2.ceil' }],
|
|
661
|
+
['f64.trunc', { simd: 'f64x2.trunc' }],
|
|
662
|
+
['f64.nearest', { simd: 'f64x2.nearest' }],
|
|
366
663
|
]),
|
|
367
664
|
}
|
|
368
665
|
|
|
666
|
+
// Integer-load → f32x4 widening, for `out[i] = intArr[i] (* k)` (Int16Array →
|
|
667
|
+
// Float32Array decode/normalize, the canonical audio/image map). jz emits the
|
|
668
|
+
// scalar as `f64.convert_i32_{s,u}(<intload>(addr))`; lift to: load `lanes` ints,
|
|
669
|
+
// widen to i32x4, then f32x4.convert. `steps` are applied innermost-first.
|
|
670
|
+
// `lossy`: i32→f32 rounds (the scalar converts via exact f64 then demotes — double
|
|
671
|
+
// rounding differs by ≤1 ulp), so it needs relaxedSimd; i8/i16 are exact in f32.
|
|
672
|
+
const INT_WIDEN_F32 = {
|
|
673
|
+
'i32.load': { load: 'v128.load', steps: [], cvt: 's', lossy: true },
|
|
674
|
+
'i32.load16_s': { load: 'v128.load64_zero', steps: ['i32x4.extend_low_i16x8_s'], cvt: 's', lossy: false },
|
|
675
|
+
'i32.load16_u': { load: 'v128.load64_zero', steps: ['i32x4.extend_low_i16x8_u'], cvt: 'u', lossy: false },
|
|
676
|
+
'i32.load8_s': { load: 'v128.load32_zero', steps: ['i16x8.extend_low_i8x16_s', 'i32x4.extend_low_i16x8_s'], cvt: 's', lossy: false },
|
|
677
|
+
'i32.load8_u': { load: 'v128.load32_zero', steps: ['i16x8.extend_low_i8x16_u', 'i32x4.extend_low_i16x8_u'], cvt: 'u', lossy: false },
|
|
678
|
+
}
|
|
679
|
+
|
|
680
|
+
// f64 scalar op → f32x4 SIMD op, for Float32Array arithmetic jz computes in f64
|
|
681
|
+
// (promote→f64 op→demote). Used only in f32-lane context under relaxedSimd, since
|
|
682
|
+
// the f64→f32 intermediate-precision drop is not bit-exact (see _relaxF32).
|
|
683
|
+
const F64_TO_F32X4 = {
|
|
684
|
+
'f64.add': 'f32x4.add', 'f64.sub': 'f32x4.sub', 'f64.mul': 'f32x4.mul', 'f64.div': 'f32x4.div',
|
|
685
|
+
'f64.min': 'f32x4.min', 'f64.max': 'f32x4.max', 'f64.neg': 'f32x4.neg', 'f64.abs': 'f32x4.abs', 'f64.sqrt': 'f32x4.sqrt',
|
|
686
|
+
}
|
|
687
|
+
|
|
369
688
|
// Horizontal reductions: associative+commutative ops applied to one
|
|
370
689
|
// loop-carried accumulator. Each entry maps the SCALAR op (which is also
|
|
371
690
|
// the op used to combine the SIMD result back into the accumulator at the
|
|
@@ -807,19 +1126,45 @@ function tryVectorize(bl, fnLocals, freshIdRef) {
|
|
|
807
1126
|
// pointers (map loops over distinct arrays). Soundness re-checked post-scan.
|
|
808
1127
|
const offsetTees = new Map()
|
|
809
1128
|
|
|
1129
|
+
// The compute/lane type is the WIDEST FLOAT among all loads+stores. A narrower
|
|
1130
|
+
// float/int LOAD is then a widening read (INT_WIDEN_F32 / f32→f64), a narrower
|
|
1131
|
+
// float/int STORE a narrowing write (demote / trunc+wrap) — both sub-width memory
|
|
1132
|
+
// ops around the float lane. Pinning it up front (vs whichever op the recursive
|
|
1133
|
+
// scan hits first) is what lets a narrowing map `i16[i] = f32arr[i]*k` keep the
|
|
1134
|
+
// f32 compute lane instead of locking onto the i16 store. No float → integer lane
|
|
1135
|
+
// (set by the scan below, unchanged).
|
|
1136
|
+
const isNarrowStore = (lane, sty) => (lane === 'f32' && (sty === 'i16' || sty === 'i8'))
|
|
1137
|
+
|| (lane === 'f64' && (sty === 'f32' || sty === 'i32'))
|
|
1138
|
+
let preFloat = null
|
|
1139
|
+
const scanFloatWidth = (n) => {
|
|
1140
|
+
if (!isArr(n)) return
|
|
1141
|
+
const t = LOAD_OPS[n[0]] || STORE_OPS[n[0]]
|
|
1142
|
+
if (t === 'f64') preFloat = 'f64'
|
|
1143
|
+
else if (t === 'f32' && preFloat == null) preFloat = 'f32'
|
|
1144
|
+
for (let i = 1; i < n.length; i++) scanFloatWidth(n[i])
|
|
1145
|
+
}
|
|
1146
|
+
for (const s of body) scanFloatWidth(s)
|
|
1147
|
+
if (preFloat) { laneType = preFloat; stride = LANE_INFO[preFloat].stride }
|
|
1148
|
+
|
|
810
1149
|
function scanForLoadsStores(node, parent, pi) {
|
|
811
1150
|
if (!isArr(node)) return true
|
|
812
1151
|
const op = node[0]
|
|
813
1152
|
if (LOAD_OPS[op]) {
|
|
1153
|
+
const lt = LOAD_OPS[op]
|
|
1154
|
+
// int→f32 widening map (`out[i] = intArr[i] (*k)`): an integer load feeding a
|
|
1155
|
+
// Float32Array store. Accept under the f32 lane and validate at the int element
|
|
1156
|
+
// stride (the loop steps `lanes` f32 = 4 elements; load64_zero/load32_zero read
|
|
1157
|
+
// exactly 4 ints). liftExprV widens via INT_WIDEN_F32.
|
|
1158
|
+
const widenInt = laneType === 'f32' && lt !== 'f32' && INT_WIDEN_F32[op]
|
|
814
1159
|
if (laneType == null) {
|
|
815
|
-
laneType =
|
|
1160
|
+
laneType = lt
|
|
816
1161
|
stride = LANE_INFO[laneType].stride
|
|
817
|
-
} else if (
|
|
1162
|
+
} else if (lt !== laneType && !widenInt) {
|
|
818
1163
|
return false
|
|
819
1164
|
}
|
|
820
1165
|
const m = matchLaneAddr(node[1], incVar, addrLocals, offsetTees)
|
|
821
1166
|
if (!m) return false
|
|
822
|
-
if ((1 << m.strideLog2) !== stride) return false
|
|
1167
|
+
if ((1 << m.strideLog2) !== (widenInt ? LANE_INFO[lt].stride : stride)) return false
|
|
823
1168
|
if (m.teeName) addrLocals.set(m.teeName, { strideLog2: m.strideLog2, base: m.base })
|
|
824
1169
|
if (m.offsetTeeName) offsetTees.set(m.offsetTeeName, m.strideLog2)
|
|
825
1170
|
loadStoreSites.push({ parent, idx: pi, kind: 'load' })
|
|
@@ -827,11 +1172,15 @@ function tryVectorize(bl, fnLocals, freshIdRef) {
|
|
|
827
1172
|
}
|
|
828
1173
|
if (STORE_OPS[op]) {
|
|
829
1174
|
const sty = STORE_OPS[op]
|
|
830
|
-
|
|
1175
|
+
// narrowing store: a narrower element under a wider float lane (`o[i]=narrow(f(x))`,
|
|
1176
|
+
// codec encode / downsample). Validate the store address at the narrow element stride
|
|
1177
|
+
// (the loop steps `lanes` of the float lane; the partial store writes that many).
|
|
1178
|
+
const narrowing = laneType != null && sty !== laneType && isNarrowStore(laneType, sty)
|
|
1179
|
+
if (laneType != null && sty !== laneType && !narrowing) return false
|
|
831
1180
|
if (laneType == null) { laneType = sty; stride = LANE_INFO[laneType].stride }
|
|
832
1181
|
const m = matchLaneAddr(node[1], incVar, addrLocals, offsetTees)
|
|
833
1182
|
if (!m) return false
|
|
834
|
-
if ((1 << m.strideLog2) !== stride) return false
|
|
1183
|
+
if ((1 << m.strideLog2) !== (narrowing ? LANE_INFO[sty].stride : stride)) return false
|
|
835
1184
|
if (m.teeName) addrLocals.set(m.teeName, { strideLog2: m.strideLog2, base: m.base })
|
|
836
1185
|
if (m.offsetTeeName) offsetTees.set(m.offsetTeeName, m.strideLog2)
|
|
837
1186
|
loadStoreSites.push({ parent, idx: pi, kind: 'store' })
|
|
@@ -913,12 +1262,35 @@ function tryVectorize(bl, fnLocals, freshIdRef) {
|
|
|
913
1262
|
}
|
|
914
1263
|
}
|
|
915
1264
|
|
|
1265
|
+
// A ToInt32 (`|0`) narrowing conversion is commonly CSE'd into its own lane-local just before
|
|
1266
|
+
// the store (`set $t (…trunc_sat…); store addr (local.get $t)`), hiding it from the
|
|
1267
|
+
// narrowing-store path (liftStmt would then lift the i32 wrap in the f64 lane and bail). When
|
|
1268
|
+
// such a lane-local is read exactly once by a narrowing store, inline the conversion back into
|
|
1269
|
+
// the store so peelNarrowConv/narrowStore handle it. The original set survives in the scalar
|
|
1270
|
+
// remainder; this only reshapes the SIMD lift (and bails cleanly if liftStmt still declines).
|
|
1271
|
+
let body2 = body
|
|
1272
|
+
{
|
|
1273
|
+
const getCount = new Map()
|
|
1274
|
+
const countGets = (n) => { if (!isArr(n)) return; if (n[0] === 'local.get' && typeof n[1] === 'string') getCount.set(n[1], (getCount.get(n[1]) || 0) + 1); for (let i = 1; i < n.length; i++) countGets(n[i]) }
|
|
1275
|
+
for (const s of body) countGets(s)
|
|
1276
|
+
const dropped = new Set()
|
|
1277
|
+
const inlined = body.map(s => {
|
|
1278
|
+
if (isArr(s) && STORE_OPS[s[0]] && s.length === 3 && STORE_OPS[s[0]] !== laneType &&
|
|
1279
|
+
isLocalGet(s[2]) && localKind.get(s[2][1]) === 'lane' && getCount.get(s[2][1]) === 1) {
|
|
1280
|
+
const def = body.find(x => isArr(x) && x[0] === 'local.set' && x[1] === s[2][1] && x.length === 3)
|
|
1281
|
+
if (def && peelNarrowConv(def[2], STORE_OPS[s[0]])) { dropped.add(def); return [s[0], s[1], def[2]] }
|
|
1282
|
+
}
|
|
1283
|
+
return s
|
|
1284
|
+
})
|
|
1285
|
+
if (dropped.size) body2 = inlined.filter(s => !dropped.has(s))
|
|
1286
|
+
}
|
|
1287
|
+
|
|
916
1288
|
// Build lifted body. If anything fails to lift, bail.
|
|
917
|
-
const newLanedLocals = new Map() // origName →
|
|
1289
|
+
const newLanedLocals = new Map() // origName → laneName (bare string; see getOrAllocLanedLocal)
|
|
918
1290
|
const extraLocals = [] // canon temps allocated during lift
|
|
919
|
-
const ctx = { laneType, incVar, rampVar: null, rampTemp: null, widenLoads: false, localKind, newLanedLocals, extraLocals, freshIdRef, fail: false, failReason: null }
|
|
1291
|
+
const ctx = { laneType, incVar, rampVar: null, rampTemp: null, widenLoads: false, localKind, fnLocals, newLanedLocals, extraLocals, freshIdRef, fail: false, failReason: null }
|
|
920
1292
|
const lifted = []
|
|
921
|
-
for (const s of
|
|
1293
|
+
for (const s of body2) {
|
|
922
1294
|
const r = liftStmt(s, ctx)
|
|
923
1295
|
if (ctx.fail) return null
|
|
924
1296
|
if (r != null) {
|
|
@@ -966,7 +1338,7 @@ function tryVectorize(bl, fnLocals, freshIdRef) {
|
|
|
966
1338
|
// Locals to add to function header.
|
|
967
1339
|
const newLocalDecls = [
|
|
968
1340
|
['local', simdBoundName, 'i32'],
|
|
969
|
-
...[...newLanedLocals.values()].map(
|
|
1341
|
+
...[...newLanedLocals.values()].map(laneName => ['local', laneName, 'v128']),
|
|
970
1342
|
...extraLocals,
|
|
971
1343
|
]
|
|
972
1344
|
|
|
@@ -1204,7 +1576,10 @@ function tryStencil(node, fnLocals, freshIdRef, enabled) {
|
|
|
1204
1576
|
if (name === incVar) continue
|
|
1205
1577
|
const ty = fnLocals.get(name)
|
|
1206
1578
|
if (ty === 'i32') { localKind.set(name, 'addr'); continue }
|
|
1207
|
-
|
|
1579
|
+
// A stencil temp computed in f64 then stored to an f32 array carries `ty === 'f64'`
|
|
1580
|
+
// in an f32 lane (jz computes Float32Array math in f64). Treat it as lane/invariant
|
|
1581
|
+
// data the same as a native-typed local — the lift lanes it as f32x4 (relaxedSimd).
|
|
1582
|
+
if (ty === laneType || (laneType === 'f32' && ty === 'f64')) {
|
|
1208
1583
|
if (writes.has(name)) {
|
|
1209
1584
|
let fk = null; for (const s of body) { const k = firstAccess(s, name); if (k) { fk = k; break } }
|
|
1210
1585
|
if (fk === 'read') return null // loop-carried float local
|
|
@@ -1218,7 +1593,7 @@ function tryStencil(node, fnLocals, freshIdRef, enabled) {
|
|
|
1218
1593
|
|
|
1219
1594
|
// Lift through the shared lifter (addresses kept verbatim; loads → v128.load).
|
|
1220
1595
|
const newLanedLocals = new Map(), extraLocals = []
|
|
1221
|
-
const ctx = { laneType, incVar, rampVar: null, rampTemp: null, widenLoads: false, localKind, newLanedLocals, extraLocals, freshIdRef, fail: false, failReason: null }
|
|
1596
|
+
const ctx = { laneType, incVar, rampVar: null, rampTemp: null, widenLoads: false, localKind, fnLocals, newLanedLocals, extraLocals, freshIdRef, fail: false, failReason: null }
|
|
1222
1597
|
const lifted = []
|
|
1223
1598
|
for (const s of body) {
|
|
1224
1599
|
const r = liftStmt(s, ctx)
|
|
@@ -1255,7 +1630,7 @@ function tryStencil(node, fnLocals, freshIdRef, enabled) {
|
|
|
1255
1630
|
// LICM-hoisted $__li invariants run ahead of the SIMD block (the scalar tail's
|
|
1256
1631
|
// copy inside bl.blockNode re-runs them harmlessly — pure & loop-invariant).
|
|
1257
1632
|
const wrapper = ['block', ...preamble.map(cloneNode), ...peelStmts, boundSetup, simdBlock, bl.blockNode]
|
|
1258
|
-
const newLocalDecls = [['local', simdBoundName, 'i32'], ...[...newLanedLocals.values()].map(
|
|
1633
|
+
const newLocalDecls = [['local', simdBoundName, 'i32'], ...[...newLanedLocals.values()].map(laneName => ['local', laneName, 'v128']), ...extraLocals]
|
|
1259
1634
|
return { wrapper, newLocalDecls }
|
|
1260
1635
|
}
|
|
1261
1636
|
|
|
@@ -1287,17 +1662,67 @@ function tryReduceVectorize(bl, fnLocals, freshIdRef, multiAcc = false) {
|
|
|
1287
1662
|
// — or a NaN-canonicalized two-statement min/max reduction —
|
|
1288
1663
|
// (local.set $cn (OP (local.get $acc) EXPR))
|
|
1289
1664
|
// (local.set $acc (select C (local.get $cn) (T.ne $cn $cn)))
|
|
1290
|
-
|
|
1665
|
+
// A conditional-store min/max (`if (a[i] > m) m = a[i]`) is the SAME reduction as the ternary
|
|
1666
|
+
// `m = a[i] > m ? a[i] : m`; rewrite it to the select-assign form so one recognizer covers both.
|
|
1667
|
+
// Sound for recognition: the lane EXPR (an array load) is pure, and the SIMD lift reads it
|
|
1668
|
+
// unconditionally anyway (pmax), while the scalar remainder keeps the original conditional store.
|
|
1669
|
+
const asSelectAssign = (stmt) => {
|
|
1670
|
+
if (isArr(stmt) && stmt[0] === 'if' && stmt.length === 3 && isArr(stmt[2]) && stmt[2][0] === 'then' && stmt[2].length === 2) {
|
|
1671
|
+
const set = stmt[2][1]
|
|
1672
|
+
if (isArr(set) && set[0] === 'local.set' && set.length === 3 && !hasSideEffect(set[2]))
|
|
1673
|
+
return ['local.set', set[1], ['select', set[2], ['local.get', set[1]], stmt[1]]]
|
|
1674
|
+
}
|
|
1675
|
+
return stmt
|
|
1676
|
+
}
|
|
1677
|
+
const bodyStmts = []
|
|
1678
|
+
for (let i = 3; i < incIdx; i++) bodyStmts.push(asSelectAssign(loopNode[i]))
|
|
1679
|
+
// CSE collapse: `m = a[i] > m ? a[i] : m` hoists the load into its own `(local.set $t LOAD)`
|
|
1680
|
+
// ahead of the reduction, making a 2-statement body the single-statement min/max recognizer
|
|
1681
|
+
// misses. When $t is pure (no side effect, no accumulator reference) inline it back into the
|
|
1682
|
+
// reduction so the canonical one-statement shape is recognized. Sound: the lift only consumes
|
|
1683
|
+
// the inlined lane expr for the SIMD prefix; the original $t set survives in the scalar
|
|
1684
|
+
// remainder (the unchanged blockNode), so $t stays defined wherever else it is read.
|
|
1685
|
+
let body0 = bodyStmts
|
|
1686
|
+
if (bodyStmts.length === 2) {
|
|
1687
|
+
const [s1, s2] = bodyStmts
|
|
1688
|
+
if (isArr(s1) && s1[0] === 'local.set' && typeof s1[1] === 'string' && s1.length === 3 &&
|
|
1689
|
+
isArr(s2) && s2[0] === 'local.set' && typeof s2[1] === 'string' && s2.length === 3 && s1[1] !== s2[1]) {
|
|
1690
|
+
const t = s1[1], expr = s1[2]
|
|
1691
|
+
const usesName = (n, name) => isArr(n) && ((n[0] === 'local.get' && n[1] === name) || n.some(c => usesName(c, name)))
|
|
1692
|
+
if (!hasSideEffect(expr) && !usesName(expr, s2[1]) && !usesName(expr, t)) {
|
|
1693
|
+
const subst = (n) => isArr(n) ? (n[0] === 'local.get' && n[1] === t ? expr : n.map(subst)) : n
|
|
1694
|
+
body0 = [['local.set', s2[1], subst(s2[2])]]
|
|
1695
|
+
}
|
|
1696
|
+
}
|
|
1697
|
+
}
|
|
1698
|
+
const bodyLen = body0.length
|
|
1291
1699
|
let accName, opName, reduceEntry, exprNode, canonC = null
|
|
1292
1700
|
if (bodyLen === 1) {
|
|
1293
|
-
const stmt =
|
|
1701
|
+
const stmt = body0[0]
|
|
1294
1702
|
if (!isArr(stmt) || stmt[0] !== 'local.set' || stmt.length !== 3) return null
|
|
1295
1703
|
accName = stmt[1]
|
|
1296
1704
|
if (typeof accName !== 'string') return null
|
|
1297
1705
|
const rhs = stmt[2]
|
|
1298
1706
|
if (!isArr(rhs)) return null
|
|
1299
1707
|
const minmax = matchIntMinMaxReduce(rhs, accName)
|
|
1300
|
-
if (minmax) {
|
|
1708
|
+
if (minmax && minmax.laneType === 'f64') {
|
|
1709
|
+
// Comparison min/max over an f64 array (`m = a[i] > m ? a[i] : m`). f64x2.pmax/pmin
|
|
1710
|
+
// replicate the scalar `(a>m)?a:m` EXACTLY per element — pmax(m,a) = (m<a)?a:m keeps the
|
|
1711
|
+
// accumulator on NaN (m<NaN is false) and on a ±0 tie, never NaN-poisoning the way
|
|
1712
|
+
// f64x2.max would. They preserve the data's exact NaN bits (a selection, not a compute),
|
|
1713
|
+
// so no canon is needed. The ONLY divergence from the sequential scalar is the SIGN of a
|
|
1714
|
+
// zero RESULT when the extremum is hit by both +0 and −0 in different lanes (a cross-lane
|
|
1715
|
+
// reorder) — strictly less than the ULP reassociation the sum reductions already accept,
|
|
1716
|
+
// so it rides the relaxedSimd tier (on at 'speed'); strict callers opt out (scalar).
|
|
1717
|
+
if (!_relaxF32) return null
|
|
1718
|
+
reduceEntry = {
|
|
1719
|
+
simd: minmax.isMax ? 'f64x2.pmax' : 'f64x2.pmin',
|
|
1720
|
+
extract: 'f64x2.extract_lane', laneType: 'f64',
|
|
1721
|
+
identity: ['f64.const', minmax.isMax ? '-inf' : 'inf'],
|
|
1722
|
+
minmaxSelect: true, isMax: minmax.isMax, pmaxF64: true,
|
|
1723
|
+
}
|
|
1724
|
+
exprNode = minmax.exprNode
|
|
1725
|
+
} else if (minmax) {
|
|
1301
1726
|
// Synthetic entry: WASM has the SIMD i32x4.max_s/min_s but no scalar i32.max, so the
|
|
1302
1727
|
// horizontal fold + merge below use select (flagged by minmaxSelect). Identity is the
|
|
1303
1728
|
// op's neutral — INT_MIN for max, INT_MAX for min. A bare narrow load instead folds
|
|
@@ -1343,7 +1768,7 @@ function tryReduceVectorize(bl, fnLocals, freshIdRef, multiAcc = false) {
|
|
|
1343
1768
|
exprNode = rhs[2]
|
|
1344
1769
|
}
|
|
1345
1770
|
} else if (bodyLen === 2) {
|
|
1346
|
-
const s1 =
|
|
1771
|
+
const s1 = body0[0], s2 = body0[1]
|
|
1347
1772
|
if (!isArr(s1) || s1[0] !== 'local.set' || s1.length !== 3) return null
|
|
1348
1773
|
if (!isArr(s2) || s2[0] !== 'local.set' || s2.length !== 3) return null
|
|
1349
1774
|
const cnName = s1[1], rhs = s1[2]
|
|
@@ -1432,15 +1857,22 @@ function tryReduceVectorize(bl, fnLocals, freshIdRef, multiAcc = false) {
|
|
|
1432
1857
|
const stride = LANE_INFO[laneType].stride
|
|
1433
1858
|
const addrLocals = new Map()
|
|
1434
1859
|
const offsetTees = new Map()
|
|
1435
|
-
let loadCount = 0
|
|
1860
|
+
let loadCount = 0, sawWidenF32 = false
|
|
1436
1861
|
function scanExpr(node) {
|
|
1437
1862
|
if (!isArr(node)) return true
|
|
1438
1863
|
const op = node[0]
|
|
1439
1864
|
if (LOAD_OPS[op]) {
|
|
1440
|
-
|
|
1865
|
+
// f32→f64 widening reduction (`s += f32arr[i]`, acc f64): liftExprV promotes
|
|
1866
|
+
// the f32.load to f64x2.promote_low_f32x4, so accept it under an f64 lane and
|
|
1867
|
+
// validate at the f32 element stride (4) — the loop still steps `lanes` (2)
|
|
1868
|
+
// elements, advancing the f32 address by 8 bytes (the load64_zero the lift reads).
|
|
1869
|
+
const ltw = LOAD_OPS[op]
|
|
1870
|
+
const widenF32 = ltw === 'f32' && laneType === 'f64'
|
|
1871
|
+
if (ltw !== laneType && !widenF32) return false
|
|
1872
|
+
if (widenF32) sawWidenF32 = true
|
|
1441
1873
|
const m = matchLaneAddr(node[1], incVar, addrLocals, offsetTees)
|
|
1442
1874
|
if (!m) return false
|
|
1443
|
-
if ((1 << m.strideLog2) !== stride) return false
|
|
1875
|
+
if ((1 << m.strideLog2) !== (widenF32 ? 4 : stride)) return false
|
|
1444
1876
|
if (m.teeName) addrLocals.set(m.teeName, { strideLog2: m.strideLog2, base: m.base })
|
|
1445
1877
|
if (m.offsetTeeName) offsetTees.set(m.offsetTeeName, m.strideLog2)
|
|
1446
1878
|
loadCount++
|
|
@@ -1477,9 +1909,14 @@ function tryReduceVectorize(bl, fnLocals, freshIdRef, multiAcc = false) {
|
|
|
1477
1909
|
for (const name of addrLocals.keys()) localKind.set(name, 'addr')
|
|
1478
1910
|
for (const name of offsetTees.keys()) localKind.set(name, 'addr')
|
|
1479
1911
|
|
|
1480
|
-
const ctx = { laneType, incVar, rampVar: null, rampTemp: null, widenLoads: false, localKind, newLanedLocals: new Map(), extraLocals: [], freshIdRef, fail: false, failReason: null }
|
|
1912
|
+
const ctx = { laneType, incVar, rampVar: null, rampTemp: null, widenLoads: false, localKind, fnLocals, newLanedLocals: new Map(), extraLocals: [], freshIdRef, fail: false, failReason: null }
|
|
1913
|
+
|
|
1481
1914
|
const liftedExpr = liftExprV(exprNode, ctx)
|
|
1482
|
-
|
|
1915
|
+
// liftExprV's contract is "null ⟺ ctx.fail"; under self-host (jz.wasm) it can diverge and
|
|
1916
|
+
// return null WITHOUT the flag, which would otherwise splice a literal `null` operand into the
|
|
1917
|
+
// emitted `(<reduce>.add acc null)` — invalid wasm ("not enough arguments on the stack"). Treat
|
|
1918
|
+
// a null lift as a bail (the loop stays scalar — correct, just unvectorized on that leg).
|
|
1919
|
+
if (ctx.fail || liftedExpr == null) return null
|
|
1483
1920
|
if (ctx.newLanedLocals.size > 0 || ctx.extraLocals.length > 0) return null
|
|
1484
1921
|
|
|
1485
1922
|
// Synthesize SIMD prefix block + horizontal reduce + (preserved scalar tail).
|
|
@@ -1501,7 +1938,9 @@ function tryReduceVectorize(bl, fnLocals, freshIdRef, multiAcc = false) {
|
|
|
1501
1938
|
// the same kind the existing 2-lane fold already does, identical on every engine.
|
|
1502
1939
|
// Restricted to the plain horizontal-fold FP path (not min/max-select, the
|
|
1503
1940
|
// narrow-widening sums, or NaN-canon — those have their own fold shapes).
|
|
1504
|
-
|
|
1941
|
+
// The f32→f64 widening sum uses half-width load64_zero loads; the multi-accumulator
|
|
1942
|
+
// offsetLoads/laneBytes logic assumes full v128 loads, so keep it single-accumulator.
|
|
1943
|
+
const plainReduce = !reduceEntry.minmaxSelect && !widen && !sawWidenF32 && canonC == null
|
|
1505
1944
|
const NACC = (multiAcc && plainReduce && (laneType === 'f64' || laneType === 'f32')) ? 4 : 1
|
|
1506
1945
|
const accK = (k) => k === 0 ? simdAccName : `$__simd_acc${id}_${k}`
|
|
1507
1946
|
const laneBytes = lanes * stride
|
|
@@ -1539,14 +1978,17 @@ function tryReduceVectorize(bl, fnLocals, freshIdRef, multiAcc = false) {
|
|
|
1539
1978
|
const extraDecls = []
|
|
1540
1979
|
let mergeStmts
|
|
1541
1980
|
if (reduceEntry.minmaxSelect) {
|
|
1542
|
-
// No scalar
|
|
1543
|
-
//
|
|
1544
|
-
// `select(a,b,(gt|lt)_s a b)` = a
|
|
1545
|
-
|
|
1981
|
+
// No scalar max/min op — fold via select through a temp (no exponential operand
|
|
1982
|
+
// duplication): ht = lane0; ht = minmax(ht, lane_k); acc = minmax(acc, ht). For int,
|
|
1983
|
+
// `select(a,b,(gt|lt)_s a b)` = max/min(a,b). For the f64 pmax/pmin reduction the scalar
|
|
1984
|
+
// equivalent is the pmax/pmin select — `pmax(a,b) = (a<b)?b:a` — so the merge keeps the
|
|
1985
|
+
// same NaN/±0 tie semantics as the f64x2.pmax lanes.
|
|
1546
1986
|
const ht = `$__simd_h${id}`
|
|
1547
|
-
extraDecls.push(['local', ht, 'i32'])
|
|
1987
|
+
extraDecls.push(['local', ht, reduceEntry.pmaxF64 ? 'f64' : 'i32'])
|
|
1548
1988
|
const lane = (k) => [reduceEntry.extract, k, ['local.get', simdAccName]]
|
|
1549
|
-
const minmaxSel =
|
|
1989
|
+
const minmaxSel = reduceEntry.pmaxF64
|
|
1990
|
+
? (a, b) => reduceEntry.isMax ? ['select', b, a, ['f64.lt', a, b]] : ['select', b, a, ['f64.lt', b, a]]
|
|
1991
|
+
: (a, b) => ['select', a, b, [reduceEntry.isMax ? 'i32.gt_s' : 'i32.lt_s', a, b]]
|
|
1550
1992
|
mergeStmts = [['local.set', ht, lane(0)]]
|
|
1551
1993
|
for (let k = 1; k < lanes; k++) mergeStmts.push(['local.set', ht, minmaxSel(lane(k), ['local.get', ht])])
|
|
1552
1994
|
if (reduceEntry.accF64) {
|
|
@@ -1735,13 +2177,17 @@ function _isAddressLocal(body, name, ind) {
|
|
|
1735
2177
|
|
|
1736
2178
|
// ---- Lifter ----------------------------------------------------------------
|
|
1737
2179
|
|
|
1738
|
-
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
2180
|
+
// Returns the v128 lane-local NAME (a string) for `name`, allocating once. We store the bare
|
|
2181
|
+
// string — NOT a `{laneName}` object — because a schema-object read back through the Map in a
|
|
2182
|
+
// DIFFERENT function returns undefined under self-host. Takes `newLanedLocals` directly
|
|
2183
|
+
// (not ctx) so callers don't need to pass the full ctx object to a helper at call-depth 2.
|
|
2184
|
+
function getOrAllocLanedLocal(name, newLanedLocals) {
|
|
2185
|
+
let laneName = newLanedLocals.get(name)
|
|
2186
|
+
if (!laneName) {
|
|
2187
|
+
laneName = `${name}__v`
|
|
2188
|
+
newLanedLocals.set(name, laneName)
|
|
2189
|
+
}
|
|
2190
|
+
return laneName
|
|
1745
2191
|
}
|
|
1746
2192
|
|
|
1747
2193
|
// Wrap an already-lifted v128 value `coreV` in per-lane NaN canonicalization:
|
|
@@ -1752,7 +2198,10 @@ function getOrAllocLanedLocal(name, ctx) {
|
|
|
1752
2198
|
// Otherwise we materialize a fresh v128 temp so the core evaluates once.
|
|
1753
2199
|
function liftCanon(coreV, C, ctx, info) {
|
|
1754
2200
|
const laneNe = ctx.laneType === 'f32' ? 'f32x4.ne' : 'f64x2.ne'
|
|
1755
|
-
const
|
|
2201
|
+
// The f32-via-f64 canon carries an f64 NaN const — splat it as f32 (demote is
|
|
2202
|
+
// exact for the canonical NaN, and the lane value coreV is already f32x4).
|
|
2203
|
+
const cF = ctx.laneType === 'f32' && isArr(C) && C[0] === 'f64.const' ? ['f32.const', C[1]] : C
|
|
2204
|
+
const splatC = [info.splat, cF]
|
|
1756
2205
|
if (isArr(coreV) && coreV[0] === 'local.get') {
|
|
1757
2206
|
return ['v128.bitselect', splatC, coreV, [laneNe, coreV, coreV]]
|
|
1758
2207
|
}
|
|
@@ -1770,6 +2219,15 @@ function liftCanon(coreV, C, ctx, info) {
|
|
|
1770
2219
|
// bail for the block currently under the recognizer chain; the walk reads it after.
|
|
1771
2220
|
let _whyNotActive = false
|
|
1772
2221
|
let _whyNotReason = null
|
|
2222
|
+
// Precision-relaxed f32 SIMD. jz computes Float32Array arithmetic in f64
|
|
2223
|
+
// (`f32.demote_f64 (f64.mul (f64.promote_f32 …) …)`); lifting that chain to
|
|
2224
|
+
// `f32x4.mul` over `v128.load` changes the intermediate from f64 to f32 — a
|
|
2225
|
+
// sub-ulp difference at f32 precision (inaudible for audio/DSP, the canonical
|
|
2226
|
+
// f32-SIMD trade every audio engine makes), but NOT bit-exact, so it is gated
|
|
2227
|
+
// on the same `relaxedSimd` opt-in that enables relaxed-FMA. The promote/demote
|
|
2228
|
+
// *strip* for a pure f32 copy (no arithmetic) round-trips losslessly and stays
|
|
2229
|
+
// on unconditionally. Armed for the duration of a vectorizeLaneLocal call.
|
|
2230
|
+
let _relaxF32 = false
|
|
1773
2231
|
|
|
1774
2232
|
// Mark a lift bail and record its reason. First-write-wins: the innermost failing op
|
|
1775
2233
|
// sets ctx.failReason; outer frames see ctx.fail already set and return without
|
|
@@ -1782,6 +2240,70 @@ const liftFail = (ctx, reason) => {
|
|
|
1782
2240
|
}
|
|
1783
2241
|
|
|
1784
2242
|
/** Lift a statement. Returns lifted stmt, or null to skip, or ['__seq__', ...] for multiple. */
|
|
2243
|
+
// Conditional lane-local assignment in a stencil/map body — the sibling of the
|
|
2244
|
+
// if-STORE path below, but the destination is a 'lane' LOCAL, not memory:
|
|
2245
|
+
// if (C) L = A [else L = B | else <nested if assigning L>]
|
|
2246
|
+
// the saturation / clamp shape (waves' amplitude clamp `if(nb>CAP)nb=CAP; else
|
|
2247
|
+
// if(nb<-CAP)nb=-CAP`). Built as ONE nested `v128.bitselect` EXPRESSION so every
|
|
2248
|
+
// mask reads the PRE-assignment lane value (no intermediate stores ⇒ order-free,
|
|
2249
|
+
// and bit-exact with the scalar select chain — a comparison mask is all-ones /
|
|
2250
|
+
// all-zeros per lane, so bitselect is an exact lane select), then a single
|
|
2251
|
+
// `local.set` of the laned local. Returns the lifted node, or null when `stmt` is
|
|
2252
|
+
// not a lane-assignment if (so the if-STORE path can try). Sets ctx.fail only once
|
|
2253
|
+
// committed (a lane-if shape that cannot be lifted).
|
|
2254
|
+
function tryLiftLaneIf(stmt, ctx) {
|
|
2255
|
+
const armBody = (arm) => {
|
|
2256
|
+
let body = arm.slice(1)
|
|
2257
|
+
if (body.length === 1 && isArr(body[0]) && body[0][0] === 'block') {
|
|
2258
|
+
const b = body[0]; let i = 1
|
|
2259
|
+
if (typeof b[i] === 'string' && b[i].startsWith('$')) i++
|
|
2260
|
+
if (isArr(b[i]) && b[i][0] === 'result') i++
|
|
2261
|
+
body = b.slice(i)
|
|
2262
|
+
}
|
|
2263
|
+
return body
|
|
2264
|
+
}
|
|
2265
|
+
// The lane being assigned: the innermost then-arm's single local.set target.
|
|
2266
|
+
const laneOf = (node) => {
|
|
2267
|
+
if (!isArr(node)) return null
|
|
2268
|
+
if (node[0] === 'local.set' && typeof node[1] === 'string' && ctx.localKind.get(node[1]) === 'lane') return node[1]
|
|
2269
|
+
if (node[0] === 'if' && isArr(node[2]) && node[2][0] === 'then') {
|
|
2270
|
+
const body = armBody(node[2])
|
|
2271
|
+
if (body.length === 1) return laneOf(body[0])
|
|
2272
|
+
}
|
|
2273
|
+
return null
|
|
2274
|
+
}
|
|
2275
|
+
const lane = laneOf(stmt)
|
|
2276
|
+
if (!lane) return null
|
|
2277
|
+
// Recurse the if-chain into nested bitselects; each `local.set L = V` is a leaf value.
|
|
2278
|
+
const buildVal = (node) => {
|
|
2279
|
+
if (isArr(node) && node[0] === 'local.set' && node[1] === lane) return liftExprV(node[2], ctx)
|
|
2280
|
+
if (isArr(node) && node[0] === 'if' && isArr(node[2]) && node[2][0] === 'then') {
|
|
2281
|
+
const thenBody = armBody(node[2])
|
|
2282
|
+
if (thenBody.length !== 1) return liftFail(ctx, 'lane-if: non-single then arm')
|
|
2283
|
+
let cond = node[1]
|
|
2284
|
+
if (isArr(cond) && cond[0] === 'i32.ne' && isI32Const(cond[2]) && cond[2][1] === 0) cond = cond[1]
|
|
2285
|
+
const cmp = isArr(cond) && cond.length === 3 ? LANE_COMPARE[ctx.laneType]?.[cond[0]] : null
|
|
2286
|
+
if (!cmp) return liftFail(ctx, 'lane-if: condition is not a lane comparison')
|
|
2287
|
+
const ca = liftExprV(cond[1], ctx); if (ctx.fail) return null
|
|
2288
|
+
const cb = liftExprV(cond[2], ctx); if (ctx.fail) return null
|
|
2289
|
+
const thenVal = buildVal(thenBody[0]); if (ctx.fail) return null
|
|
2290
|
+
const elseArm = (isArr(node[3]) && node[3][0] === 'else') ? armBody(node[3]) : null
|
|
2291
|
+
let elseVal
|
|
2292
|
+
if (elseArm) {
|
|
2293
|
+
if (elseArm.length !== 1) return liftFail(ctx, 'lane-if: non-single else arm')
|
|
2294
|
+
elseVal = buildVal(elseArm[0]); if (ctx.fail) return null
|
|
2295
|
+
} else {
|
|
2296
|
+
elseVal = ['local.get', getOrAllocLanedLocal(lane, ctx.newLanedLocals)] // no else ⇒ keep current
|
|
2297
|
+
}
|
|
2298
|
+
return ['v128.bitselect', thenVal, elseVal, [cmp, ca, cb]]
|
|
2299
|
+
}
|
|
2300
|
+
return liftFail(ctx, 'lane-if: unrecognized arm shape')
|
|
2301
|
+
}
|
|
2302
|
+
const val = buildVal(stmt)
|
|
2303
|
+
if (ctx.fail || val == null) return null
|
|
2304
|
+
return ['local.set', getOrAllocLanedLocal(lane, ctx.newLanedLocals), val]
|
|
2305
|
+
}
|
|
2306
|
+
|
|
1785
2307
|
function liftStmt(stmt, ctx) {
|
|
1786
2308
|
if (!isArr(stmt)) {
|
|
1787
2309
|
// Bare strings like "drop" — produced by stack-form WAT. We unwrap value-blocks
|
|
@@ -1799,7 +2321,7 @@ function liftStmt(stmt, ctx) {
|
|
|
1799
2321
|
return ['local.set', name, stmt[2]]
|
|
1800
2322
|
}
|
|
1801
2323
|
if (kind === 'lane') {
|
|
1802
|
-
const
|
|
2324
|
+
const laneName = getOrAllocLanedLocal(name, ctx.newLanedLocals)
|
|
1803
2325
|
const v = liftExprV(stmt[2], ctx)
|
|
1804
2326
|
if (ctx.fail) return null
|
|
1805
2327
|
return ['local.set', laneName, v]
|
|
@@ -1808,14 +2330,32 @@ function liftStmt(stmt, ctx) {
|
|
|
1808
2330
|
}
|
|
1809
2331
|
|
|
1810
2332
|
if (STORE_OPS[op]) {
|
|
1811
|
-
const simdStore = 'v128.store'
|
|
1812
2333
|
const addr = stmt[1] // we leave addresses as-is (scalar i32 expressions)
|
|
1813
|
-
const val = liftExprV(stmt[2], ctx)
|
|
1814
|
-
if (ctx.fail) return null
|
|
1815
2334
|
// Handle memarg if present (last positional after addr/val): unlikely in
|
|
1816
2335
|
// pre-watr IR for this shape; bail if more than 3 children.
|
|
1817
2336
|
if (stmt.length !== 3) return liftFail(ctx, `${op} with memarg`)
|
|
1818
|
-
|
|
2337
|
+
const sty = STORE_OPS[op]
|
|
2338
|
+
// Narrowing store: a narrower element written from a wider float lane (`o[i] =
|
|
2339
|
+
// narrow(f(x))` — codec encode / downsample). The scalar store value carries a
|
|
2340
|
+
// conversion (f32.demote_f64, or the float→int ToInt32 idiom); peel it, lift the
|
|
2341
|
+
// inner float expr, and let narrowStore apply the SIMD narrow + low-byte store.
|
|
2342
|
+
if (sty !== ctx.laneType) {
|
|
2343
|
+
// Integer narrowing (`o[i] = (f(x)) | 0` into Int32Array/…) lowers via the saturating
|
|
2344
|
+
// i32x4.trunc_sat_f64x2_s_zero, which clamps +Inf / |x|≥2³¹ to INT_MAX where scalar
|
|
2345
|
+
// ToInt32 wraps mod 2³² — bit-exact for in-range finite values (every pixel/coordinate/
|
|
2346
|
+
// typical-DSP value), divergent only at that edge, so it rides relaxedSimd. Float demote
|
|
2347
|
+
// (f64→f32) is bit-exact (round-to-nearest both ways) and stays ungated.
|
|
2348
|
+
if (sty !== 'f32' && !_relaxF32) return liftFail(ctx, `narrowing ${ctx.laneType}->${sty} store saturates out-of-range (needs relaxedSimd)`)
|
|
2349
|
+
const inner = peelNarrowConv(stmt[2], sty)
|
|
2350
|
+
if (!inner) return liftFail(ctx, `narrowing store ${ctx.laneType}->${sty}: unrecognized conversion`)
|
|
2351
|
+
const innerV = liftExprV(inner, ctx)
|
|
2352
|
+
if (ctx.fail) return null
|
|
2353
|
+
const ns = narrowStore(addr, innerV, ctx.laneType, sty, ctx)
|
|
2354
|
+
return ns || liftFail(ctx, `no narrowing ${ctx.laneType}->${sty}`)
|
|
2355
|
+
}
|
|
2356
|
+
const val = liftExprV(stmt[2], ctx)
|
|
2357
|
+
if (ctx.fail) return null
|
|
2358
|
+
return ['v128.store', addr, val]
|
|
1819
2359
|
}
|
|
1820
2360
|
|
|
1821
2361
|
// (block (result T) STMTS... TAIL_EXPR) followed by sibling "drop" — we get
|
|
@@ -1847,6 +2387,10 @@ function liftStmt(stmt, ctx) {
|
|
|
1847
2387
|
// are trap-free) and emit ONE store of `bitselect(A, B, mask(COND))`. Unlocks per-pixel conditional
|
|
1848
2388
|
// maps like lorenz's i32x4 trail fade (`if (p & 0xffffff) px[i] = fade(p)`).
|
|
1849
2389
|
if (op === 'if' && isArr(stmt[2]) && stmt[2][0] === 'then') {
|
|
2390
|
+
// First: conditional lane-LOCAL assignment (clamp/saturation) → bitselect into the laned local.
|
|
2391
|
+
const laneLifted = tryLiftLaneIf(stmt, ctx)
|
|
2392
|
+
if (ctx.fail) return null
|
|
2393
|
+
if (laneLifted) return laneLifted
|
|
1850
2394
|
const armOf = (arm) => {
|
|
1851
2395
|
let body = arm.slice(1)
|
|
1852
2396
|
if (body.length === 1 && isArr(body[0]) && body[0][0] === 'block') { // unwrap a single block arm
|
|
@@ -1914,6 +2458,39 @@ function liftExprV(expr, ctx) {
|
|
|
1914
2458
|
return ['f64x2.promote_low_f32x4', addr]
|
|
1915
2459
|
}
|
|
1916
2460
|
|
|
2461
|
+
// f32-lane: jz computes Float32Array arithmetic in f64, wrapping the f32 load in
|
|
2462
|
+
// `f64.promote_f32` and the result in `f32.demote_f64`. The promote/demote are
|
|
2463
|
+
// lane-space identities — strip them (a promote-of-load + demote round-trips
|
|
2464
|
+
// losslessly: a pure `b[i]=a[i]` copy vectorizes bit-exactly, always). The f64
|
|
2465
|
+
// arithmetic op and any f64 constant map to their f32x4 forms only under
|
|
2466
|
+
// relaxedSimd, since computing in f32 (vs f64-then-demote) drops sub-ulp precision.
|
|
2467
|
+
if (ctx.laneType === 'f32') {
|
|
2468
|
+
if (op === 'f64.promote_f32') return liftExprV(expr[1], ctx)
|
|
2469
|
+
if (op === 'f32.demote_f64') return liftExprV(expr[1], ctx)
|
|
2470
|
+
// int→f32 widening load: `f64.convert_i32_{s,u}(<intload>(addr))` → load 4 ints,
|
|
2471
|
+
// widen to i32x4, f32x4.convert. i8/i16 are exact in f32; i32 rounds (gated).
|
|
2472
|
+
if ((op === 'f64.convert_i32_s' || op === 'f64.convert_i32_u') && isArr(expr[1]) && INT_WIDEN_F32[expr[1][0]]) {
|
|
2473
|
+
const ld = expr[1], w = INT_WIDEN_F32[ld[0]]
|
|
2474
|
+
if (w.lossy && !_relaxF32) return liftFail(ctx, `${ld[0]}→f32 SIMD rounds (i32 exceeds f32 mantissa) — needs relaxedSimd`)
|
|
2475
|
+
const addr = typeof ld[1] === 'string' && ld[1].startsWith('offset=') ? [w.load, ld[1], ld[2]] : [w.load, ld[1]]
|
|
2476
|
+
let v = addr
|
|
2477
|
+
for (const step of w.steps) v = [step, v]
|
|
2478
|
+
return [w.cvt === 'u' ? 'f32x4.convert_i32x4_u' : 'f32x4.convert_i32x4_s', v]
|
|
2479
|
+
}
|
|
2480
|
+
if (op === 'f64.const') {
|
|
2481
|
+
if (!_relaxF32) return liftFail(ctx, 'f64 constant in f32 lane needs relaxedSimd (f32 round of the constant)')
|
|
2482
|
+
return ['f32x4.splat', ['f32.const', expr[1]]]
|
|
2483
|
+
}
|
|
2484
|
+
const f32op = F64_TO_F32X4[op]
|
|
2485
|
+
if (f32op) {
|
|
2486
|
+
if (!_relaxF32) return liftFail(ctx, `${op}: f32 SIMD computes in f32 not f64 (sub-ulp) — needs relaxedSimd`)
|
|
2487
|
+
const a = liftExprV(expr[1], ctx); if (ctx.fail) return null
|
|
2488
|
+
if (expr.length === 2) return [f32op, a] // unary: neg / abs / sqrt
|
|
2489
|
+
const b = liftExprV(expr[2], ctx); if (ctx.fail) return null
|
|
2490
|
+
return [f32op, a, b]
|
|
2491
|
+
}
|
|
2492
|
+
}
|
|
2493
|
+
|
|
1917
2494
|
// Loads → v128.load (preserving address, including any local.tee).
|
|
1918
2495
|
if (LOAD_OPS[op]) {
|
|
1919
2496
|
if (LOAD_OPS[op] !== ctx.laneType) return liftFail(ctx, `${op}: load type ≠ lane type ${ctx.laneType}`)
|
|
@@ -1942,10 +2519,18 @@ function liftExprV(expr, ctx) {
|
|
|
1942
2519
|
}
|
|
1943
2520
|
const kind = ctx.localKind.get(name)
|
|
1944
2521
|
if (kind === 'lane') {
|
|
1945
|
-
const
|
|
2522
|
+
const laneName = getOrAllocLanedLocal(name, ctx.newLanedLocals)
|
|
1946
2523
|
return ['local.get', laneName]
|
|
1947
2524
|
}
|
|
1948
2525
|
if (kind === 'invariant') {
|
|
2526
|
+
// An invariant whose wasm type ≠ the lane element type needs a scalar
|
|
2527
|
+
// convert before the splat. The common case: an f64 multiplier/gain splat
|
|
2528
|
+
// into an f32 lane (`out[i] = in[i] * k`). Demoting k to f32 is the same
|
|
2529
|
+
// precision relaxation as the f32 arithmetic itself, so gate it on relaxedSimd.
|
|
2530
|
+
if (ctx.laneType === 'f32' && ctx.fnLocals?.get(name) === 'f64') {
|
|
2531
|
+
if (!_relaxF32) return liftFail(ctx, `${name}: f64 invariant in f32 lane needs relaxedSimd`)
|
|
2532
|
+
return [info.splat, ['f32.demote_f64', ['local.get', name]]]
|
|
2533
|
+
}
|
|
1949
2534
|
return [info.splat, ['local.get', name]]
|
|
1950
2535
|
}
|
|
1951
2536
|
if (kind === 'addr' || name === ctx.incVar) {
|
|
@@ -1974,19 +2559,44 @@ function liftExprV(expr, ctx) {
|
|
|
1974
2559
|
// it). Both the flattened `select` form and the un-flattened `block` form
|
|
1975
2560
|
// lift to a per-lane v128.bitselect — canonical value in NaN lanes, X
|
|
1976
2561
|
// elsewhere — exactly reproducing the scalar canonicalization lane-by-lane.
|
|
1977
|
-
if (
|
|
1978
|
-
|
|
2562
|
+
if (op === 'select') {
|
|
2563
|
+
// NaN-canonicalization idiom `(select C X (T.ne X X))` — FLOAT lanes only (an
|
|
2564
|
+
// integer lane never carries it, since no i32 value is NaN).
|
|
2565
|
+
if (ctx.laneType === 'f64' || ctx.laneType === 'f32') {
|
|
1979
2566
|
const m = matchCanonSelect(expr, ctx.laneType)
|
|
1980
|
-
if (
|
|
1981
|
-
|
|
1982
|
-
|
|
2567
|
+
if (m) {
|
|
2568
|
+
const coreV = liftExprV(m.val, ctx)
|
|
2569
|
+
return ctx.fail ? null : liftCanon(coreV, m.C, ctx, info)
|
|
2570
|
+
}
|
|
1983
2571
|
}
|
|
1984
|
-
|
|
1985
|
-
|
|
1986
|
-
|
|
1987
|
-
|
|
1988
|
-
|
|
2572
|
+
// General `select(X, Y, COND)` (wasm: X if COND else Y) — jz lowers a value
|
|
2573
|
+
// ternary `COND ? X : Y` to this when both arms are cheap/pure. Lift to
|
|
2574
|
+
// v128.bitselect(X, Y, mask) like the `if` form below — valid for EVERY lane type
|
|
2575
|
+
// (i32 included: COND maps via LANE_COMPARE[laneType], NaN is irrelevant). Both
|
|
2576
|
+
// arms are lane-pure (recursion forbids stores/sets) and trap-free, so evaluating
|
|
2577
|
+
// both then selecting is sound. f32 lane promotes operands → f64.* compare → f32x4.
|
|
2578
|
+
if (expr.length === 4) {
|
|
2579
|
+
const cond = expr[3]
|
|
2580
|
+
const cmpOp = isArr(cond) && ctx.laneType === 'f32' && typeof cond[0] === 'string' && cond[0].startsWith('f64.') ? 'f32.' + cond[0].slice(4) : (isArr(cond) ? cond[0] : null)
|
|
2581
|
+
const cmpSimd = cmpOp && cond.length === 3 ? LANE_COMPARE[ctx.laneType]?.[cmpOp] : null
|
|
2582
|
+
if (!cmpSimd) return liftFail(ctx, `select condition ${isArr(cond) ? cond[0] : '?'} not a lane comparison`)
|
|
2583
|
+
const x = liftExprV(expr[1], ctx); if (ctx.fail) return null
|
|
2584
|
+
const y = liftExprV(expr[2], ctx); if (ctx.fail) return null
|
|
2585
|
+
const ca = liftExprV(cond[1], ctx); if (ctx.fail) return null
|
|
2586
|
+
const cb = liftExprV(cond[2], ctx); if (ctx.fail) return null
|
|
2587
|
+
const mtmp = `$__mask${ctx.freshIdRef.next++}`
|
|
2588
|
+
ctx.extraLocals.push(['local', mtmp, 'v128'])
|
|
2589
|
+
return ['block', ['result', 'v128'],
|
|
2590
|
+
['local.set', mtmp, [cmpSimd, ca, cb]],
|
|
2591
|
+
['v128.bitselect', x, y, ['local.get', mtmp]]]
|
|
1989
2592
|
}
|
|
2593
|
+
return liftFail(ctx, 'non-canonical select (not a NaN-canon idiom)')
|
|
2594
|
+
}
|
|
2595
|
+
if ((ctx.laneType === 'f64' || ctx.laneType === 'f32') && op === 'block') {
|
|
2596
|
+
const m = matchCanonBlock(expr, ctx.laneType)
|
|
2597
|
+
if (!m) return liftFail(ctx, 'non-canonical value-block')
|
|
2598
|
+
const coreV = liftExprV(m.core, ctx)
|
|
2599
|
+
return ctx.fail ? null : liftCanon(coreV, m.C, ctx, info)
|
|
1990
2600
|
}
|
|
1991
2601
|
|
|
1992
2602
|
// Conditional select — jz lowers `cond ? X : Y` to (if (result LT) COND (then X)
|
|
@@ -1998,13 +2608,21 @@ function liftExprV(expr, ctx) {
|
|
|
1998
2608
|
// evaluates X,Y before its 3rd operand, but any address `local.tee` lives in
|
|
1999
2609
|
// COND and must run before the branches read it (matching scalar order).
|
|
2000
2610
|
if (op === 'if') {
|
|
2001
|
-
|
|
2611
|
+
// jz lowers `cond ? X : Y` to (if (result T) COND (then X)(else Y)). In an f32
|
|
2612
|
+
// lane it computes in f64 (promote/demote around the store), so the `if` carries
|
|
2613
|
+
// `(result f64)` and COND is an `f64.*` compare — accept both, mapping to f32x4.
|
|
2614
|
+
// The branch values are f32-mapped by recursion; gated by relaxedSimd via those.
|
|
2615
|
+
const resTy = isArr(expr[1]) && expr[1][0] === 'result' ? expr[1][1] : null
|
|
2616
|
+
if (resTy !== ctx.laneType && !(ctx.laneType === 'f32' && resTy === 'f64')) return liftFail(ctx, 'conditional without lane-typed result')
|
|
2002
2617
|
const thenN = expr[3], elseN = expr[4]
|
|
2003
2618
|
if (!isArr(thenN) || thenN[0] !== 'then' || thenN.length !== 2) return liftFail(ctx, 'malformed conditional then-branch')
|
|
2004
2619
|
if (!isArr(elseN) || elseN[0] !== 'else' || elseN.length !== 2) return liftFail(ctx, 'malformed conditional else-branch')
|
|
2005
2620
|
let cond = expr[2]
|
|
2006
2621
|
if (isArr(cond) && cond[0] === 'i32.ne' && isI32Const(cond[2]) && cond[2][1] === 0) cond = cond[1] // strip `!= 0`
|
|
2007
|
-
|
|
2622
|
+
// f32 lane: operands were promoted, so the compare is `f64.*` — use its f32x4 form
|
|
2623
|
+
// (operands are exact f32→f64 promotions, so the lane comparison is unchanged).
|
|
2624
|
+
const cmpOp = isArr(cond) && ctx.laneType === 'f32' && typeof cond[0] === 'string' && cond[0].startsWith('f64.') ? 'f32.' + cond[0].slice(4) : (isArr(cond) ? cond[0] : null)
|
|
2625
|
+
const cmpSimd = cmpOp && cond.length === 3 ? LANE_COMPARE[ctx.laneType]?.[cmpOp] : null
|
|
2008
2626
|
if (!cmpSimd) return liftFail(ctx, `${isArr(cond) ? cond[0] : 'condition'}: not a lane-vectorizable comparison`)
|
|
2009
2627
|
const ca = liftExprV(cond[1], ctx); if (ctx.fail) return null
|
|
2010
2628
|
const cb = liftExprV(cond[2], ctx); if (ctx.fail) return null
|
|
@@ -2510,15 +3128,17 @@ function tryRampMap(blockNode, fnLocals, freshIdRef) {
|
|
|
2510
3128
|
// pointer is still expressed in terms of the IV. We keep the address verbatim
|
|
2511
3129
|
// (scalar i32) and advance the IV by LANES, so `base + (i<<K)` lands on the
|
|
2512
3130
|
// next group's first element each SIMD step — for any element width.
|
|
2513
|
-
|
|
3131
|
+
// Collect every store. One store → the original single-map paths (ramp pack / widening /
|
|
3132
|
+
// 4-wide). Two or more independent store8s → a multi-channel in-place fade (boids' 4-channel
|
|
3133
|
+
// u8 trail), handled by the multi-store WIDEN16 branch below; one pass over memory, N widening
|
|
3134
|
+
// stores. Stores beyond the first don't reach the single-store paths.
|
|
3135
|
+
const storeStmts = []
|
|
2514
3136
|
for (let i = 0; i < body.length; i++) {
|
|
2515
3137
|
const s = body[i]
|
|
2516
|
-
if (isArr(s) && STORE_OPS[s[0]]) {
|
|
2517
|
-
if (storeStmt) return null
|
|
2518
|
-
storeStmt = s; storeIdx = i
|
|
2519
|
-
}
|
|
3138
|
+
if (isArr(s) && STORE_OPS[s[0]]) storeStmts.push({ stmt: s, idx: i })
|
|
2520
3139
|
}
|
|
2521
|
-
if (!
|
|
3140
|
+
if (!storeStmts.length) return null
|
|
3141
|
+
const storeStmt = storeStmts[0].stmt, storeIdx = storeStmts[0].idx
|
|
2522
3142
|
const storeOp = storeStmt[0]
|
|
2523
3143
|
if (storeStmt.length !== 3) return null
|
|
2524
3144
|
const elemLog2 = { 'i32.store8': 0, 'i32.store': 2 }[storeOp]
|
|
@@ -2535,8 +3155,24 @@ function tryRampMap(blockNode, fnLocals, freshIdRef) {
|
|
|
2535
3155
|
for (const s of body) gatherNames(s)
|
|
2536
3156
|
for (const name of allNames) { const k = _offsetLocalStride(body, name, ivName); if (k != null) offsetTees.set(name, k) }
|
|
2537
3157
|
|
|
3158
|
+
// CSE'd FULL lane address: an in-place map `a[i] = f(a[i])` shares one `(local.tee $A
|
|
3159
|
+
// (i32.add base i))` between the load and the store, reused as `(local.get $A)`. Without
|
|
3160
|
+
// resolving it the store/load address matchers reject the bare get (the empty-addrLocals
|
|
3161
|
+
// bug that kept every in-place trail-fade scalar). Record each such tee — its lifted
|
|
3162
|
+
// address (the tee) runs in the hoisted v128.load, so the store's get reads it back.
|
|
3163
|
+
const addrLocals = new Map()
|
|
3164
|
+
const recordAddrTees = (n) => {
|
|
3165
|
+
if (!isArr(n)) return
|
|
3166
|
+
if (n[0] === 'local.tee' && typeof n[1] === 'string' && isArr(n[2]) && n[2][0] === 'i32.add') {
|
|
3167
|
+
const m = matchLaneAddr(n[2], ivName, addrLocals, offsetTees)
|
|
3168
|
+
if (m && m.teeName == null) addrLocals.set(n[1], { strideLog2: m.strideLog2, base: m.base })
|
|
3169
|
+
}
|
|
3170
|
+
for (let i = 1; i < n.length; i++) recordAddrTees(n[i])
|
|
3171
|
+
}
|
|
3172
|
+
for (const s of body) recordAddrTees(s)
|
|
3173
|
+
|
|
2538
3174
|
const storeAddr = storeStmt[1]
|
|
2539
|
-
const addrM = matchLaneAddr(storeAddr, ivName,
|
|
3175
|
+
const addrM = matchLaneAddr(storeAddr, ivName, addrLocals, offsetTees)
|
|
2540
3176
|
if (!addrM || addrM.strideLog2 !== elemLog2) return null
|
|
2541
3177
|
|
|
2542
3178
|
// Memory loads turn this into a widening byte-map: out[i] = narrow(f(widen(a[i])…)).
|
|
@@ -2549,7 +3185,7 @@ function tryRampMap(blockNode, fnLocals, freshIdRef) {
|
|
|
2549
3185
|
if (LOAD_OPS[n[0]]) {
|
|
2550
3186
|
hasLoads = true
|
|
2551
3187
|
if (storeOp !== 'i32.store8' || n[0] !== 'i32.load8_u') { loadsOk = false; return }
|
|
2552
|
-
const m = matchLaneAddr(n[1], ivName,
|
|
3188
|
+
const m = matchLaneAddr(n[1], ivName, addrLocals, offsetTees)
|
|
2553
3189
|
if (!m || m.strideLog2 !== 0) loadsOk = false
|
|
2554
3190
|
return // address validated; the IV-strided subtree is not data
|
|
2555
3191
|
}
|
|
@@ -2588,7 +3224,7 @@ function tryRampMap(blockNode, fnLocals, freshIdRef) {
|
|
|
2588
3224
|
const newLanedLocals = new Map()
|
|
2589
3225
|
const extraLocals = []
|
|
2590
3226
|
const freshV128 = (tag) => { const n = `$__${tag}${freshIdRef.next++}`; extraLocals.push(['local', n, 'v128']); return n }
|
|
2591
|
-
const ctx = { laneType: 'i32', incVar: ivName, rampVar: ivName, rampTemp: null, widenLoads: true, localKind, newLanedLocals, extraLocals, freshIdRef, fail: false, failReason: null }
|
|
3227
|
+
const ctx = { laneType: 'i32', incVar: ivName, rampVar: ivName, rampTemp: null, widenLoads: true, localKind, fnLocals: null, newLanedLocals, extraLocals, freshIdRef, fail: false, failReason: null }
|
|
2592
3228
|
|
|
2593
3229
|
// A byte store fed by one value expression (inline, or via a single lane-local
|
|
2594
3230
|
// temp `tw = EXPR; store(addr, tw)`) carries no loop-carried state, so we can
|
|
@@ -2631,21 +3267,12 @@ function tryRampMap(blockNode, fnLocals, freshIdRef) {
|
|
|
2631
3267
|
else return null
|
|
2632
3268
|
return (r[0] < 0 || r[1] > 65535) ? null : r
|
|
2633
3269
|
}
|
|
2634
|
-
|
|
2635
|
-
|
|
2636
|
-
|
|
2637
|
-
|
|
2638
|
-
|
|
2639
|
-
const
|
|
2640
|
-
['v128.const', 'i32x4', String(off), String(off + 1), String(off + 2), String(off + 3)]]
|
|
2641
|
-
|
|
2642
|
-
let lifted
|
|
2643
|
-
if (WIDEN16) {
|
|
2644
|
-
// 16-wide widening byte-map: load each u8 input once (v128.load of 16 bytes),
|
|
2645
|
-
// extend_low/high to two i16x8 halves, run the affine map in i16x8 on each half,
|
|
2646
|
-
// narrow_u the two result halves to one i8x16, store 16. Each load is hoisted to a
|
|
2647
|
-
// temp (so extend_low + extend_high share one load); the loads run in source order,
|
|
2648
|
-
// so the offset `local.tee` in the first one is set before later loads read it.
|
|
3270
|
+
// The i16x8 widening byte-map emit for ONE store, factored so a multi-channel fade reuses it
|
|
3271
|
+
// per channel: load each u8 input once (v128.load 16), extend_low/high to two i16x8 halves,
|
|
3272
|
+
// run the affine map in i16x8 on each half, narrow_u to i8x16, store 16. Each load is hoisted
|
|
3273
|
+
// (extend_low + extend_high share one load); loads run in source order, so an offset/address
|
|
3274
|
+
// `local.tee` in a load is set before the store's `local.get` reads it.
|
|
3275
|
+
const widen16Emit = (sAddr, valueExpr) => {
|
|
2649
3276
|
const loadTemps = new Map()
|
|
2650
3277
|
const loadSets = []
|
|
2651
3278
|
const collectLoads = (e) => {
|
|
@@ -2655,7 +3282,7 @@ function tryRampMap(blockNode, fnLocals, freshIdRef) {
|
|
|
2655
3282
|
if (!loadTemps.has(k)) { const t = freshV128('win'); loadTemps.set(k, t); loadSets.push(['local.set', t, ['v128.load', e[1]]]) }
|
|
2656
3283
|
} else for (let i = 1; i < e.length; i++) collectLoads(e[i])
|
|
2657
3284
|
}
|
|
2658
|
-
collectLoads(
|
|
3285
|
+
collectLoads(valueExpr)
|
|
2659
3286
|
const liftW = (e, half) => {
|
|
2660
3287
|
const op = e[0]
|
|
2661
3288
|
if (op === 'i32.const') return ['i16x8.splat', e]
|
|
@@ -2667,45 +3294,87 @@ function tryRampMap(blockNode, fnLocals, freshIdRef) {
|
|
|
2667
3294
|
if (op === 'i32.and') return ['v128.and', liftW(e[1], half), ['i16x8.splat', e[2]]]
|
|
2668
3295
|
return null // byteValueRange already proved every op is one of the above
|
|
2669
3296
|
}
|
|
2670
|
-
|
|
2671
|
-
['v128.store',
|
|
2672
|
-
}
|
|
3297
|
+
return [...loadSets,
|
|
3298
|
+
['v128.store', sAddr, ['i8x16.narrow_i16x8_u', liftW(valueExpr, 'low'), liftW(valueExpr, 'high')]]]
|
|
3299
|
+
}
|
|
3300
|
+
|
|
3301
|
+
let lifted, LANES
|
|
3302
|
+
if (storeStmts.length > 1) {
|
|
3303
|
+
// MULTI-CHANNEL in-place byte fade: N independent store8s in one loop (boids' 4-channel u8
|
|
3304
|
+
// trail). Each store must be a u8 store of a WIDEN16-eligible value (range ≤ 255); it emits
|
|
3305
|
+
// its own load→i16x8→narrow→store sequence, all concatenated into ONE 16-wide pass — so the
|
|
3306
|
+
// memory traffic stays single-pass (vs N separate vectorized loops). Every body statement
|
|
3307
|
+
// must be a store or the lane-local set feeding one; any other (shared/invariant) compute
|
|
3308
|
+
// bails to scalar, since it would be dropped.
|
|
3309
|
+
LANES = 16
|
|
2673
3310
|
lifted = []
|
|
2674
|
-
const
|
|
2675
|
-
for (
|
|
2676
|
-
|
|
2677
|
-
|
|
2678
|
-
|
|
2679
|
-
|
|
2680
|
-
|
|
2681
|
-
|
|
2682
|
-
|
|
2683
|
-
|
|
2684
|
-
|
|
2685
|
-
|
|
2686
|
-
|
|
2687
|
-
|
|
2688
|
-
|
|
2689
|
-
|
|
2690
|
-
|
|
2691
|
-
|
|
3311
|
+
const consumed = new Set()
|
|
3312
|
+
for (const { stmt, idx } of storeStmts) {
|
|
3313
|
+
if (stmt[0] !== 'i32.store8' || stmt.length !== 3) return null
|
|
3314
|
+
const a = matchLaneAddr(stmt[1], ivName, addrLocals, offsetTees)
|
|
3315
|
+
if (!a || a.strideLog2 !== 0) return null
|
|
3316
|
+
consumed.add(idx)
|
|
3317
|
+
let val = stmt[2]
|
|
3318
|
+
if (isArr(val) && val[0] === 'local.get' && typeof val[1] === 'string' && localKind.get(val[1]) === 'lane') {
|
|
3319
|
+
let setIdx = -1
|
|
3320
|
+
for (let j = 0; j < idx; j++) { const s = body[j]; if (isArr(s) && s[0] === 'local.set' && s[1] === val[1] && s.length === 3) { setIdx = j; val = s[2] } }
|
|
3321
|
+
if (setIdx < 0) return null
|
|
3322
|
+
consumed.add(setIdx)
|
|
3323
|
+
}
|
|
3324
|
+
const rng = byteValueRange(val)
|
|
3325
|
+
if (!rng || rng[1] > 255) return null // not WIDEN16-eligible (overflows u16 or u8) → scalar
|
|
3326
|
+
lifted.push(...widen16Emit(stmt[1], val))
|
|
3327
|
+
}
|
|
3328
|
+
if (consumed.size !== body.length) return null
|
|
2692
3329
|
} else {
|
|
2693
|
-
|
|
2694
|
-
|
|
2695
|
-
|
|
2696
|
-
|
|
2697
|
-
|
|
2698
|
-
|
|
3330
|
+
const wideValueExpr = (!hasLoads && byteValueExpr) ? byteValueExpr : null // pure ramp → 16-wide pack
|
|
3331
|
+
const widenRange = (hasLoads && byteValueExpr) ? byteValueRange(byteValueExpr) : null
|
|
3332
|
+
const WIDEN16 = widenRange != null && widenRange[1] <= 255 // result fits a byte ⇒ narrow_u exact
|
|
3333
|
+
const WIDE16 = wideValueExpr != null
|
|
3334
|
+
LANES = (WIDE16 || WIDEN16) ? 16 : 4
|
|
3335
|
+
const ramp = (off) => ['i32x4.add', ['i32x4.splat', ['local.get', ivName]],
|
|
3336
|
+
['v128.const', 'i32x4', String(off), String(off + 1), String(off + 2), String(off + 3)]]
|
|
3337
|
+
|
|
3338
|
+
if (WIDEN16) {
|
|
3339
|
+
lifted = widen16Emit(storeAddr, byteValueExpr)
|
|
3340
|
+
} else if (WIDE16) {
|
|
3341
|
+
lifted = []
|
|
3342
|
+
const vv = []
|
|
3343
|
+
for (let j = 0; j < 4; j++) {
|
|
3344
|
+
const rt = freshV128('ramp')
|
|
3345
|
+
lifted.push(['local.set', rt, ramp(j * 4)])
|
|
3346
|
+
ctx.rampTemp = rt
|
|
3347
|
+
const v = liftExprV(wideValueExpr, ctx)
|
|
2699
3348
|
if (ctx.fail) return null
|
|
2700
|
-
|
|
2701
|
-
|
|
2702
|
-
|
|
2703
|
-
|
|
2704
|
-
|
|
3349
|
+
const vn = freshV128('rampv')
|
|
3350
|
+
lifted.push(['local.set', vn, v])
|
|
3351
|
+
vv.push(vn)
|
|
3352
|
+
}
|
|
3353
|
+
// Pack the low byte of all 16 i32 lanes (4 vectors) into one i8x16, in order.
|
|
3354
|
+
const g = (n) => ['local.get', n]
|
|
3355
|
+
const sh = (a, b, idx) => ['i8x16.shuffle', ...idx.map(String), a, b]
|
|
3356
|
+
const lo = freshV128('ramplo'), hi = freshV128('ramphi')
|
|
3357
|
+
lifted.push(['local.set', lo, sh(g(vv[0]), g(vv[1]), [0, 4, 8, 12, 16, 20, 24, 28, 0, 0, 0, 0, 0, 0, 0, 0])])
|
|
3358
|
+
lifted.push(['local.set', hi, sh(g(vv[2]), g(vv[3]), [0, 4, 8, 12, 16, 20, 24, 28, 0, 0, 0, 0, 0, 0, 0, 0])])
|
|
3359
|
+
lifted.push(['v128.store', storeAddr, sh(g(lo), g(hi), [0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23])])
|
|
3360
|
+
} else {
|
|
3361
|
+
ctx.rampTemp = freshV128('ramp')
|
|
3362
|
+
// ramp = [i, i+1, i+2, i+3], computed once per SIMD iteration.
|
|
3363
|
+
lifted = [['local.set', ctx.rampTemp, ramp(0)]]
|
|
3364
|
+
for (let i = 0; i < body.length; i++) {
|
|
3365
|
+
if (i === storeIdx) {
|
|
3366
|
+
const vval = liftExprV(storeStmt[2], ctx)
|
|
3367
|
+
if (ctx.fail) return null
|
|
3368
|
+
lifted.push(buildRampStore(storeOp, storeAddr, vval, ctx))
|
|
3369
|
+
} else {
|
|
3370
|
+
const r = liftStmt(body[i], ctx)
|
|
3371
|
+
if (ctx.fail) return null
|
|
3372
|
+
if (r != null) { if (Array.isArray(r) && r[0] === '__seq__') lifted.push(...r.slice(1)); else lifted.push(r) }
|
|
3373
|
+
}
|
|
2705
3374
|
}
|
|
2706
3375
|
}
|
|
2707
3376
|
}
|
|
2708
|
-
if (!lifted.length) return null
|
|
3377
|
+
if (!lifted || !lifted.length) return null
|
|
2709
3378
|
|
|
2710
3379
|
const id = freshIdRef.next++
|
|
2711
3380
|
const simdBoundName = `$__simd_bound${id}`
|
|
@@ -2726,7 +3395,7 @@ function tryRampMap(blockNode, fnLocals, freshIdRef) {
|
|
|
2726
3395
|
const wrapper = ['block', boundSetup, simdBlock, blockNode]
|
|
2727
3396
|
const newLocalDecls = [
|
|
2728
3397
|
['local', simdBoundName, 'i32'],
|
|
2729
|
-
...[...newLanedLocals.values()].map(
|
|
3398
|
+
...[...newLanedLocals.values()].map(laneName => ['local', laneName, 'v128']),
|
|
2730
3399
|
...extraLocals,
|
|
2731
3400
|
]
|
|
2732
3401
|
return { wrapper, newLocalDecls }
|
|
@@ -2736,6 +3405,62 @@ function tryRampMap(blockNode, fnLocals, freshIdRef) {
|
|
|
2736
3405
|
// `storeOp` at scalar address `addr`. i32.store is the full vector; i32.store8
|
|
2737
3406
|
// truncates (low byte of each lane) via i8x16.shuffle — exactly matching scalar
|
|
2738
3407
|
// store8, with no value-range assumption (shuffle selects, never saturates).
|
|
3408
|
+
// Narrowing-map store: pack a wider float lane vector `val` down to a narrower
|
|
3409
|
+
// store element and write the low bytes (extract_lane 0 + scalar store, like
|
|
3410
|
+
// buildRampStore). f64→f32 demotes (bit-exact vs scalar); f64→i32 truncates;
|
|
3411
|
+
// f32→i16/i8 truncate to i32x4 then WRAP via i8x16.shuffle (low bytes = scalar
|
|
3412
|
+
// store{8,16}, never saturates). Returns the store stmt or null (unsupported).
|
|
3413
|
+
// Peel the scalar narrowing conversion off a store value, returning the inner float
|
|
3414
|
+
// expr to lift (narrowStore then applies the SIMD narrow). f32 store: f32.demote_f64(X).
|
|
3415
|
+
// int store: the ToInt32 idiom `i32.wrap_i64(X<0 ? trunc_sat_f64_s X : trunc_sat_f64_u X)`
|
|
3416
|
+
// (or a bare trunc_sat). The inner X is the f64/f32 lane value computed before the cast.
|
|
3417
|
+
function peelNarrowConv(val, sty) {
|
|
3418
|
+
if (!isArr(val)) return null
|
|
3419
|
+
if (sty === 'f32') return val[0] === 'f32.demote_f64' ? val[1] : null
|
|
3420
|
+
// int element (i8/i16/i32): peel ToInt32 (`x | 0`). jz's general lowering is an
|
|
3421
|
+
// Infinity-guarded saturating trunc:
|
|
3422
|
+
// (select (i32.wrap_i64 (i64.trunc_sat_f64_s X)) (i32.const 0) (f64.ne X' Inf))
|
|
3423
|
+
// where X is `(local.tee $inf <f64 expr>)` and X' the matching get. Peel to the inner f64.
|
|
3424
|
+
// (The SIMD narrow i32x4.trunc_sat_f64x2_s_zero saturates +Inf / |x|≥2³¹ to INT_MAX where
|
|
3425
|
+
// ToInt32 wraps mod 2³² — caller gates the int narrowing on relaxedSimd for that edge.)
|
|
3426
|
+
if (val[0] === 'select' && val.length === 4 && isI32Const(val[2]) && val[2][1] === 0 &&
|
|
3427
|
+
isArr(val[1]) && val[1][0] === 'i32.wrap_i64' && isArr(val[1][1]) && val[1][1][0] === 'i64.trunc_sat_f64_s') {
|
|
3428
|
+
let inner = val[1][1][1] // the f64 operand of the trunc, captured in a `(local.tee $inf …)`
|
|
3429
|
+
if (isArr(inner) && inner[0] === 'local.tee' && inner.length === 3) inner = inner[2] // peel to the tee's VALUE
|
|
3430
|
+
return inner
|
|
3431
|
+
}
|
|
3432
|
+
if (val[0] === 'i32.wrap_i64' && isArr(val[1]) && val[1][0] === 'if') {
|
|
3433
|
+
const iff = val[1], thenA = iff[3], elseA = iff[4]
|
|
3434
|
+
const s = isArr(thenA) && isArr(thenA[1]) && thenA[1][0] === 'i64.trunc_sat_f64_s' ? thenA[1][1] : null
|
|
3435
|
+
const u = isArr(elseA) && isArr(elseA[1]) && elseA[1][0] === 'i64.trunc_sat_f64_u' ? elseA[1][1] : null
|
|
3436
|
+
if (s && u && exprEq(s, u)) return s
|
|
3437
|
+
}
|
|
3438
|
+
if (val[0] === 'i32.trunc_sat_f64_s' || val[0] === 'i32.trunc_sat_f64_u') return val[1]
|
|
3439
|
+
return null
|
|
3440
|
+
}
|
|
3441
|
+
|
|
3442
|
+
// i8x16.shuffle masks that pack a 4×i32 vector down to the low bytes of each lane
|
|
3443
|
+
// (truncating-wrap = scalar store{8,16}), tail zero-filled: _I16 keeps the low 2
|
|
3444
|
+
// bytes of each lane (→ 8 bytes / i64.store), _I8 the low byte (→ 4 bytes / i32.store).
|
|
3445
|
+
const PACK_I32_TO_I16 = [0, 1, 4, 5, 8, 9, 12, 13, 0, 0, 0, 0, 0, 0, 0, 0]
|
|
3446
|
+
const PACK_I32_TO_I8 = [0, 4, 8, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
|
|
3447
|
+
|
|
3448
|
+
function narrowStore(addr, val, laneType, sty, ctx) {
|
|
3449
|
+
const tmp = `$__nv${ctx.freshIdRef.next++}`
|
|
3450
|
+
ctx.extraLocals.push(['local', tmp, 'v128'])
|
|
3451
|
+
const g = ['local.get', tmp]
|
|
3452
|
+
const sh = (idx) => ['i8x16.shuffle', ...idx.map(String), g, g]
|
|
3453
|
+
let pre, lane8, store
|
|
3454
|
+
if (laneType === 'f64' && sty === 'f32') { pre = ['f32x4.demote_f64x2_zero', val]; lane8 = g; store = 'i64.store' }
|
|
3455
|
+
else if (laneType === 'f64' && sty === 'i32') { pre = ['i32x4.trunc_sat_f64x2_s_zero', val]; lane8 = g; store = 'i64.store' }
|
|
3456
|
+
else if (laneType === 'f32' && sty === 'i16') { pre = ['i32x4.trunc_sat_f32x4_s', val]; lane8 = sh(PACK_I32_TO_I16); store = 'i64.store' }
|
|
3457
|
+
else if (laneType === 'f32' && sty === 'i8') { pre = ['i32x4.trunc_sat_f32x4_s', val]; lane8 = sh(PACK_I32_TO_I8); store = 'i32.store' }
|
|
3458
|
+
else return null
|
|
3459
|
+
// 8-byte stores extract an i64 lane; the 4-byte i8 pack extracts an i32 lane.
|
|
3460
|
+
const packed = store === 'i64.store' ? ['i64x2.extract_lane', 0, lane8] : ['i32x4.extract_lane', 0, lane8]
|
|
3461
|
+
return ['block', ['local.set', tmp, pre], [store, addr, packed]]
|
|
3462
|
+
}
|
|
3463
|
+
|
|
2739
3464
|
function buildRampStore(storeOp, addr, vval, ctx) {
|
|
2740
3465
|
if (storeOp === 'i32.store') return ['v128.store', addr, vval] // 4 i32 lanes → 16 bytes
|
|
2741
3466
|
// i32.store8: hoist vval to a temp so the shuffle reads it once; low byte of
|
|
@@ -3336,6 +4061,16 @@ function tryDivergentEscapeVectorize(blockNode, fnLocals, freshIdRef) {
|
|
|
3336
4061
|
const kTop = kindOf(keepTopC)
|
|
3337
4062
|
const limitKeep = keepTopC // initialised here; may be reassigned below for single-break
|
|
3338
4063
|
let kMid = null, boundExpr, boundI32, itLeft
|
|
4064
|
+
// Compound-top guard `while (A && B)`: classify the SECOND keep too. Left unclassified (kMid=null),
|
|
4065
|
+
// the downstream lift/keepMask treats keepMidC as a per-lane f64 escape — right for the Julia order
|
|
4066
|
+
// (limit, escape) but it then lifts the i32 LIMIT of the mandelbrot order (escape, `it<MAX`) into an
|
|
4067
|
+
// f64 lane and crashes. Setting kMid routes the limit through the scalar guard instead; for the
|
|
4068
|
+
// Julia order kMid='escape' is identical to the old null (both lift). Bail if neither keep is an
|
|
4069
|
+
// escape — a two-limit guard has no per-lane divergence for this vectorizer to exploit.
|
|
4070
|
+
if (compoundTop) {
|
|
4071
|
+
kMid = kindOf(keepMidC)
|
|
4072
|
+
if (kTop === 'limit' && kMid === 'limit') return null
|
|
4073
|
+
}
|
|
3339
4074
|
if (midBreaks.length === 1) {
|
|
3340
4075
|
kMid = kindOf(keepMidC)
|
|
3341
4076
|
if (kTop === kMid) return null
|
|
@@ -3726,8 +4461,8 @@ function tryDivergentEscapeVectorize(blockNode, fnLocals, freshIdRef) {
|
|
|
3726
4461
|
// public `$math.{sin,cos}` wrap the same core. Their f64x2 mirrors $math.sin2/$math.cos2 (the
|
|
3727
4462
|
// vectorized reduce+horner, module/math.js:543) are BIT-EXACT per lane to the scalar core — so we
|
|
3728
4463
|
// can lift the call straight to the *2 helper. Phase-2 adds pow/log/atan2 here (see PPC_CALL2).
|
|
3729
|
-
// NOTE: scalar targets here must be kept out of
|
|
3730
|
-
//
|
|
4464
|
+
// NOTE: scalar targets here must be kept out of watr's single-caller inlining — jz passes these
|
|
4465
|
+
// keys (SIMD_PINNED, below) as watOptimize's `pin` list, else the call node is gone before this lift runs.
|
|
3731
4466
|
const PPC_CALL2 = {
|
|
3732
4467
|
'$math.sin_core': '$math.sin2', '$math.cos_core': '$math.cos2',
|
|
3733
4468
|
'$math.sin': '$math.sin2', '$math.cos': '$math.cos2',
|
|
@@ -3738,6 +4473,11 @@ const PPC_CALL2 = {
|
|
|
3738
4473
|
'$math.log': '$math.log_v', '$math.exp': '$math.exp_v', '$math.exp2': '$math.exp2_v',
|
|
3739
4474
|
}
|
|
3740
4475
|
|
|
4476
|
+
// Scalar transcendentals the auto-vectorizer rewrites to f64x2 mirrors above. watr's inline
|
|
4477
|
+
// passes must NOT dissolve their (single-caller) call nodes before this lift runs — jz passes
|
|
4478
|
+
// these to watOptimize's `pin` option (the protection policy lives here, not hardcoded in watr).
|
|
4479
|
+
export const SIMD_PINNED = Object.keys(PPC_CALL2)
|
|
4480
|
+
|
|
3741
4481
|
// Per-pixel-color vectorizer. The dual of tryDivergentEscapeVectorize for kernels with NO inner
|
|
3742
4482
|
// escape loop: an outer pixel loop whose body computes an f64 value from the pixel index (via
|
|
3743
4483
|
// cos/sin/sqrt/…), packs it to a u32 colour, and stores it — every pixel independent. We lift the
|
|
@@ -4134,13 +4874,19 @@ function tryOuterStrip(blockNode, fnLocals, freshIdRef, enabled) {
|
|
|
4134
4874
|
// accumulator seeds (→ splat), scalar inner-IV init. MUST run before the inner-body lift so the
|
|
4135
4875
|
// per-pixel coord lanes are registered when the inner loop references them. ----
|
|
4136
4876
|
const laneInit = []
|
|
4877
|
+
const seededAccs = new Set()
|
|
4137
4878
|
for (let i = 0; i < innerIdx; i++) {
|
|
4138
4879
|
const s = obody[i]
|
|
4139
4880
|
if (!(isArr(s) && s[0] === 'local.set' && typeof s[1] === 'string' && s.length === 3)) { laneInit.push(s); continue }
|
|
4140
4881
|
const name = s[1]
|
|
4141
4882
|
if (accNames.has(name)) { // accumulator seed → splat
|
|
4883
|
+
// The seed must be a FRESH per-pixel value, independent of the accumulator's own carry.
|
|
4884
|
+
// A seed that reads `name` (e.g. `acc = acc * decay`) propagates the previous pixel's
|
|
4885
|
+
// running value across pixels — that's a loop-carried recurrence, not a per-pixel reset.
|
|
4886
|
+
if (readsName(s[2], name)) return null
|
|
4142
4887
|
const seed = liftOS(s[2])
|
|
4143
4888
|
if (!seed) return null
|
|
4889
|
+
seededAccs.add(name)
|
|
4144
4890
|
laneInit.push(['local.set', laneMap.get(name), seed]); continue
|
|
4145
4891
|
}
|
|
4146
4892
|
if (fnLocals.get(name) === 'f64' && readsName(s[2], pxVar)) { // per-pixel coord (cx = xi/W) → ramp lane
|
|
@@ -4153,6 +4899,14 @@ function tryOuterStrip(blockNode, fnLocals, freshIdRef, enabled) {
|
|
|
4153
4899
|
laneInit.push(s) // scalar (inner IV init `b=0`, invariant setup)
|
|
4154
4900
|
}
|
|
4155
4901
|
|
|
4902
|
+
// LEGALITY: every accumulator must be FRESHLY SEEDED inside the outer-loop body. An accumulator
|
|
4903
|
+
// with no per-pixel seed is live-in — carried across outer iterations (a recurrence like the
|
|
4904
|
+
// lorenz `x = x + S·(…)` evolving over the sample loop). The two pixel lanes are then DEPENDENT:
|
|
4905
|
+
// lane xi+1 must continue from lane xi's final value, not restart from a splat of the shared
|
|
4906
|
+
// carry. Strip-mining it runs both lanes from the same seed in lockstep — halving the real work
|
|
4907
|
+
// and producing a wrong result (a bogus speedup on a serial recurrence). Reject.
|
|
4908
|
+
for (const a of accNames) if (!seededAccs.has(a)) return null
|
|
4909
|
+
|
|
4156
4910
|
// ---- lift the inner-loop body: temp f64 lane locals + accumulate into the acc shadows; the
|
|
4157
4911
|
// inner IV bump stays scalar. Per-pixel coords now resolve via laneMap. ----
|
|
4158
4912
|
const liftedInner = []
|
|
@@ -4218,6 +4972,339 @@ function tryOuterStrip(blockNode, fnLocals, freshIdRef, enabled) {
|
|
|
4218
4972
|
return { wrapper, newLocalDecls }
|
|
4219
4973
|
}
|
|
4220
4974
|
|
|
4975
|
+
// ---- Per-pixel iterated-map reduction (tryIteratedReduce, experimental) ----------------------
|
|
4976
|
+
//
|
|
4977
|
+
// Generalizes the outer-strip to the ITERATED-MAP fractal shape — lyapunov, bifurcation, smooth-
|
|
4978
|
+
// escape attractors — whose per-pixel value runs a recurrence many times and accumulates a
|
|
4979
|
+
// transcendental. Beyond tryOuterStrip (one inner loop, a plain additive accumulator) it handles:
|
|
4980
|
+
// • MULTIPLE inner loops carrying per-pixel f64 state between them (lyapunov warmup → accumulate),
|
|
4981
|
+
// • loop-carried f64 RECURRENCES x = r·x·(1−x) (not just acc = acc + …),
|
|
4982
|
+
// • lane-invariant scalar bookkeeping kept SCALAR — integer counters with wraparound and the
|
|
4983
|
+
// forcing-sequence gather seq[si] (same index for both lanes → one scalar load),
|
|
4984
|
+
// • a scalar-condition select seq[si]<1 ? a : b (a ramps per lane, b splats) → a scalar
|
|
4985
|
+
// `if (result v128)`, and a per-lane conditional accumulate if(d>0) L += log(d) → bitselect.
|
|
4986
|
+
// Two adjacent pixels (xi, xi+1) run as f64x2 lanes; the colour pack+store runs scalar per lane,
|
|
4987
|
+
// and the original scalar loop, kept as the tail, finishes the odd last pixel.
|
|
4988
|
+
//
|
|
4989
|
+
// BIT-EXACT: f64x2 arithmetic is per-lane IEEE-identical, $math.log_v/exp_v are the per-lane mirrors
|
|
4990
|
+
// of the scalar polys, and the conditional accumulate adds bitselect(f(x), 0, mask) — exactly the
|
|
4991
|
+
// scalar add-or-skip. The speculatively-evaluated transcendental of a masked-out lane is discarded
|
|
4992
|
+
// (the helpers never trap). Gated behind cfg.experimentalOuterStrip; only fires when an inner loop
|
|
4993
|
+
// carries a transcendental (the latency-bound work SIMD actually accelerates — cheap-arithmetic
|
|
4994
|
+
// pixel loops are left to the scalar JIT, which already pipelines independent iterations).
|
|
4995
|
+
function tryIteratedReduce(blockNode, fnLocals, freshIdRef, enabled) {
|
|
4996
|
+
if (!enabled) return null
|
|
4997
|
+
const outer = matchOuterPixelLoop(blockNode)
|
|
4998
|
+
if (!outer) return null
|
|
4999
|
+
const { oLabel, loopNode, preamble, pixelIVs, pxVar, widthBound, pivType, obody, oExit } = outer
|
|
5000
|
+
|
|
5001
|
+
const innerIdxs = []
|
|
5002
|
+
for (let i = 0; i < obody.length; i++) {
|
|
5003
|
+
const s = obody[i]
|
|
5004
|
+
if (isArr(s) && s[0] === 'block' && s.slice(1).some(c => isArr(c) && c[0] === 'loop')) innerIdxs.push(i)
|
|
5005
|
+
}
|
|
5006
|
+
if (!innerIdxs.length) return null
|
|
5007
|
+
const lastInner = innerIdxs[innerIdxs.length - 1]
|
|
5008
|
+
const innerSet = new Set(innerIdxs)
|
|
5009
|
+
|
|
5010
|
+
const impureCall = (n) => isArr(n) && ((n[0] === 'call' && typeof n[1] === 'string' && !n[1].startsWith('$math.')) || n.some(impureCall))
|
|
5011
|
+
if (obody.some(impureCall)) return null
|
|
5012
|
+
|
|
5013
|
+
const id = freshIdRef.next++
|
|
5014
|
+
const nm = (s) => `$__ir${id}_${s}`
|
|
5015
|
+
const laneMap = new Map() // f64 per-pixel local → its v128 shadow
|
|
5016
|
+
const shadowOf = (v) => { let s = laneMap.get(v); if (!s) { s = nm(v.replace(/\W/g, '')); laneMap.set(v, s) } return s }
|
|
5017
|
+
let sawHeavy = false // a transcendental lifted inside a loop → SIMD is worth it
|
|
5018
|
+
|
|
5019
|
+
const bump = (n, k) => k === 0 ? n
|
|
5020
|
+
: (isArr(n) && n[0] === 'local.get' && pivType.has(n[1])) ? [pivType.get(n[1]) + '.add', n, [pivType.get(n[1]) + '.const', k]]
|
|
5021
|
+
: (isArr(n) ? n.map(c => bump(c, k)) : n)
|
|
5022
|
+
const rampOf = (piv) => pivType.get(piv) === 'f64'
|
|
5023
|
+
? ['f64x2.replace_lane', 1, ['f64x2.splat', ['local.get', piv]], ['f64.add', ['local.get', piv], ['f64.const', 1]]]
|
|
5024
|
+
: ['f64x2.replace_lane', 1, ['f64x2.splat', ['f64.convert_i32_s', ['local.get', piv]]], ['f64.convert_i32_s', ['i32.add', ['local.get', piv], ['i32.const', 1]]]]
|
|
5025
|
+
const readsName = (n, name) => isArr(n) && ((n[0] === 'local.get' && n[1] === name) || n.some(c => readsName(c, name)))
|
|
5026
|
+
// Lane-invariant: reads no per-pixel lane local and no pixel IV → identical value in both lanes.
|
|
5027
|
+
const laneInvariant = (n) => !isArr(n) ? true
|
|
5028
|
+
: n[0] === 'local.get' ? !(laneMap.has(n[1]) || pivType.has(n[1]))
|
|
5029
|
+
: n.slice(1).every(laneInvariant)
|
|
5030
|
+
|
|
5031
|
+
// Build the f64x2 form of `cond ? x : y` from already-lifted arms `x`,`y` and the raw `cond`.
|
|
5032
|
+
// A lane-INVARIANT cond (same both lanes — e.g. seq[si]<1) → a v128-typed scalar branch; a
|
|
5033
|
+
// per-lane f64 compare → bitselect (x where cond, y elsewhere).
|
|
5034
|
+
const liftSelect = (x, y, cond) => {
|
|
5035
|
+
if (!x || !y) return null
|
|
5036
|
+
if (isArr(cond) && cond[0] === 'i32.ne' && isI32Const(cond[2]) && cond[2][1] === 0) cond = cond[1]
|
|
5037
|
+
if (laneInvariant(cond)) return ['if', ['result', 'v128'], cond, ['then', x], ['else', y]]
|
|
5038
|
+
const cmp = isArr(cond) && cond.length === 3 ? CMP_LANE[cond[0]] : null
|
|
5039
|
+
if (!cmp) return null
|
|
5040
|
+
const ca = lift(cond[1]), cb = lift(cond[2])
|
|
5041
|
+
return (ca && cb) ? ['v128.bitselect', x, y, [cmp, ca, cb]] : null
|
|
5042
|
+
}
|
|
5043
|
+
// Lift an f64 expression to f64x2 (null = not liftable).
|
|
5044
|
+
const lift = (n) => {
|
|
5045
|
+
if (!isArr(n)) return null
|
|
5046
|
+
const op = n[0]
|
|
5047
|
+
if (op === 'f64.const') return ['f64x2.splat', n]
|
|
5048
|
+
if (op === 'local.get') {
|
|
5049
|
+
const v = n[1]
|
|
5050
|
+
if (laneMap.has(v)) return ['local.get', laneMap.get(v)]
|
|
5051
|
+
if (pivType.get(v) === 'f64') return rampOf(v)
|
|
5052
|
+
if (writesName(loopNode, v)) return null
|
|
5053
|
+
return ['f64x2.splat', n]
|
|
5054
|
+
}
|
|
5055
|
+
if (op === 'f64.convert_i32_s' && isArr(n[1]) && n[1][0] === 'local.get' && pivType.get(n[1][1]) === 'i32') return rampOf(n[1][1])
|
|
5056
|
+
if (op === 'global.get') return writesName(loopNode, n[1]) ? null : ['f64x2.splat', n]
|
|
5057
|
+
if (LOAD_OPS[op] === 'f64') {
|
|
5058
|
+
const addr = typeof n[1] === 'string' && n[1].startsWith('offset=') ? n[2] : n[1]
|
|
5059
|
+
if (readsName(addr, pxVar) || [...laneMap.keys()].some(lv => readsName(addr, lv))) return null // per-lane gather: unsupported
|
|
5060
|
+
return ['f64x2.splat', n]
|
|
5061
|
+
}
|
|
5062
|
+
if (op === 'call') {
|
|
5063
|
+
const v2 = PPC_CALL2[n[1]]
|
|
5064
|
+
if (!v2) return null
|
|
5065
|
+
if (n.length === 3) { const a = lift(n[2]); if (!a) return null; sawHeavy = true; return ['call', v2, a] }
|
|
5066
|
+
if (n.length === 4) { const a = lift(n[2]), b = lift(n[3]); if (!a || !b) return null; sawHeavy = true; return ['call', v2, a, b] }
|
|
5067
|
+
return null
|
|
5068
|
+
}
|
|
5069
|
+
if (op === 'if') {
|
|
5070
|
+
if (!isArr(n[1]) || n[1][0] !== 'result' || n[1][1] !== 'f64') return null
|
|
5071
|
+
const thenN = n[3], elseN = n[4]
|
|
5072
|
+
if (!isArr(thenN) || thenN[0] !== 'then' || thenN.length !== 2) return null
|
|
5073
|
+
if (!isArr(elseN) || elseN[0] !== 'else' || elseN.length !== 2) return null
|
|
5074
|
+
let cond = n[2]
|
|
5075
|
+
if (isArr(cond) && cond[0] === 'i32.ne' && isI32Const(cond[2]) && cond[2][1] === 0) cond = cond[1]
|
|
5076
|
+
return liftSelect(lift(thenN[1]), lift(elseN[1]), cond)
|
|
5077
|
+
}
|
|
5078
|
+
// jz lowers `cond ? A : B` to a `select` (A if cond else B). Same two cases as the `if` form:
|
|
5079
|
+
// lane-invariant cond → a scalar v128-typed branch; per-lane f64 compare → bitselect.
|
|
5080
|
+
if (op === 'select' && n.length === 4) return liftSelect(lift(n[1]), lift(n[2]), n[3])
|
|
5081
|
+
if (LANE_PURE.f64.has(op)) {
|
|
5082
|
+
const ks = n.slice(1).map(lift)
|
|
5083
|
+
return ks.some(k => k === null) ? null : [LANE_PURE.f64.get(op).simd, ...ks]
|
|
5084
|
+
}
|
|
5085
|
+
return null
|
|
5086
|
+
}
|
|
5087
|
+
|
|
5088
|
+
// Lift one inner-loop body statement → its lifted form(s), or null to bail.
|
|
5089
|
+
const liftInnerStmt = (s, innerIV) => {
|
|
5090
|
+
if (matchInc1(s) === innerIV || matchIncN(s)?.name === innerIV) return [s] // IV bump: scalar
|
|
5091
|
+
if (isArr(s) && s[0] === 'local.set' && typeof s[1] === 'string' && s.length === 3) {
|
|
5092
|
+
const name = s[1], rhs = s[2]
|
|
5093
|
+
if (fnLocals.get(name) !== 'f64') return laneInvariant(rhs) ? [s] : null // scalar i32 counter
|
|
5094
|
+
const lifted = lift(rhs) // recurrence (rhs reads name) resolves to the shadow — fine
|
|
5095
|
+
return lifted ? [['local.set', shadowOf(name), lifted]] : null
|
|
5096
|
+
}
|
|
5097
|
+
// Lane-invariant scalar `if` (counter wraparound `if(si>=N) si=0`) → keep scalar.
|
|
5098
|
+
if (isArr(s) && s[0] === 'if' && laneInvariant(s[1]) &&
|
|
5099
|
+
s.slice(2).every(arm => isArr(arm) && (arm[0] === 'then' || arm[0] === 'else') &&
|
|
5100
|
+
arm.slice(1).every(st => isArr(st) && st[0] === 'local.set' && fnLocals.get(st[1]) !== 'f64'))) return [s]
|
|
5101
|
+
// Per-lane conditional accumulate `if(cond) acc = acc + E` → acc += bitselect(liftE, 0, mask).
|
|
5102
|
+
if (isArr(s) && s[0] === 'if' && s.length === 3 && isArr(s[2]) && s[2][0] === 'then' && s[2].length === 2) {
|
|
5103
|
+
const st = s[2][1]
|
|
5104
|
+
if (isArr(st) && st[0] === 'local.set' && st.length === 3 && fnLocals.get(st[1]) === 'f64' && laneMap.has(st[1]) &&
|
|
5105
|
+
isArr(st[2]) && st[2][0] === 'f64.add' && isLocalGet(st[2][1], st[1])) {
|
|
5106
|
+
const cond = s[1], cmp = isArr(cond) && cond.length === 3 ? CMP_LANE[cond[0]] : null
|
|
5107
|
+
if (!cmp || laneInvariant(cond)) return null // need a per-lane mask
|
|
5108
|
+
const liftE = lift(st[2][2]), ca = lift(cond[1]), cb = lift(cond[2])
|
|
5109
|
+
if (!liftE || !ca || !cb) return null
|
|
5110
|
+
const sh = laneMap.get(st[1])
|
|
5111
|
+
return [['local.set', sh, ['f64x2.add', ['local.get', sh], ['v128.bitselect', liftE, ['f64x2.splat', ['f64.const', 0]], [cmp, ca, cb]]]]]
|
|
5112
|
+
}
|
|
5113
|
+
}
|
|
5114
|
+
return null
|
|
5115
|
+
}
|
|
5116
|
+
|
|
5117
|
+
const liftInnerLoop = (block) => {
|
|
5118
|
+
const ibl = matchBlockLoop(block, { allowPreamble: true })
|
|
5119
|
+
if (!ibl || ibl.preamble.length) return null
|
|
5120
|
+
const lifted = []
|
|
5121
|
+
for (const s of ibl.body) { const out = liftInnerStmt(s, ibl.incVar); if (!out) return null; lifted.push(...out) }
|
|
5122
|
+
return ['block', ibl.blockLabel, ['loop', ibl.loopLabel, ibl.loopNode[2], ...lifted, ibl.loopNode[ibl.incIdx], ['br', ibl.loopLabel]]]
|
|
5123
|
+
}
|
|
5124
|
+
|
|
5125
|
+
// ---- laneCompute = obody[0..lastInner]: f64 seeds → shadow lift; scalar seeds kept; loops lifted ----
|
|
5126
|
+
const laneCompute = []
|
|
5127
|
+
for (let i = 0; i <= lastInner; i++) {
|
|
5128
|
+
const s = obody[i]
|
|
5129
|
+
if (innerSet.has(i)) { const li = liftInnerLoop(s); if (!li) return null; laneCompute.push(li); continue }
|
|
5130
|
+
if (isArr(s) && s[0] === 'local.set' && typeof s[1] === 'string' && s.length === 3) {
|
|
5131
|
+
const name = s[1], rhs = s[2]
|
|
5132
|
+
if (fnLocals.get(name) === 'f64') {
|
|
5133
|
+
if (readsName(rhs, name)) return null // self-reading seed = carry across the OUTER loop → reject
|
|
5134
|
+
const lifted = lift(rhs); if (!lifted) return null
|
|
5135
|
+
laneCompute.push(['local.set', shadowOf(name), lifted])
|
|
5136
|
+
} else { if (!laneInvariant(rhs)) return null; laneCompute.push(s) } // scalar counter seed
|
|
5137
|
+
continue
|
|
5138
|
+
}
|
|
5139
|
+
return null
|
|
5140
|
+
}
|
|
5141
|
+
if (!sawHeavy || !laneMap.size) return null // no transcendental reduction → leave to the scalar JIT
|
|
5142
|
+
|
|
5143
|
+
// ---- epilogue = obody[lastInner+1..]: colour pack+store, run scalar per lane ----
|
|
5144
|
+
const epilogue = obody.slice(lastInner + 1)
|
|
5145
|
+
let hasStore = false
|
|
5146
|
+
const findStore = (n) => { if (!isArr(n)) return; if (STORE_OPS[n[0]]) hasStore = true; n.forEach(findStore) }
|
|
5147
|
+
epilogue.forEach(findStore)
|
|
5148
|
+
if (!hasStore) return null
|
|
5149
|
+
const epiWritten = new Set()
|
|
5150
|
+
const wr = (n) => { if (!isArr(n)) return; const st = (n[0] === 'local.set' || n[0] === 'local.tee') && typeof n[1] === 'string'; if (st) epiWritten.add(n[1]); for (const c of (st ? n.slice(2) : n.slice(1))) wr(c) }
|
|
5151
|
+
epilogue.forEach(wr)
|
|
5152
|
+
const epiReadSet = new Set(); const rd = (n) => { if (!isArr(n)) return; if (n[0] === 'local.get') epiReadSet.add(n[1]); else for (const c of n) rd(c) }
|
|
5153
|
+
epilogue.forEach(rd)
|
|
5154
|
+
for (const v of epiReadSet) if (writesName(loopNode, v) && !laneMap.has(v) && !epiWritten.has(v) && !pivType.has(v)) return null
|
|
5155
|
+
const epiReads = [...laneMap.keys()].filter(v => epiReadSet.has(v))
|
|
5156
|
+
if (!epiReads.length) return null
|
|
5157
|
+
|
|
5158
|
+
// ============================ emit ============================
|
|
5159
|
+
const newLocalDecls = [...new Set(laneMap.values())].map(n => ['local', n, 'v128'])
|
|
5160
|
+
const epiLane = (k) => [
|
|
5161
|
+
...epiReads.map(v => ['local.set', v, ['f64x2.extract_lane', k, ['local.get', laneMap.get(v)]]]),
|
|
5162
|
+
...epilogue.map(s => bump(s, k)),
|
|
5163
|
+
]
|
|
5164
|
+
const sOut = nm('ob'), sOl = nm('ol')
|
|
5165
|
+
const simdOuter = ['block', sOut, ['loop', sOl,
|
|
5166
|
+
['br_if', sOut, ['i32.eqz', [oExit.cmpOp, bump(['local.get', pxVar], 1), widthBound]]],
|
|
5167
|
+
...laneCompute, ...epiLane(0), ...epiLane(1),
|
|
5168
|
+
...pixelIVs.map(p => ['local.set', p.name, [p.type + '.add', ['local.get', p.name], [p.type + '.const', 2]]]),
|
|
5169
|
+
['br', sOl]]]
|
|
5170
|
+
const wrapper = ['block', nm('w'), ...preamble, simdOuter, ['block', oLabel, loopNode]]
|
|
5171
|
+
return { wrapper, newLocalDecls }
|
|
5172
|
+
}
|
|
5173
|
+
|
|
5174
|
+
// ---- Integer convolution column-strip-mine (tryConvColumn, experimental) ---------------------
|
|
5175
|
+
//
|
|
5176
|
+
// The int8 quantized convolution / dense-MAC kernel (conv2d): an OUTER output-pixel loop (ox)
|
|
5177
|
+
// whose body — after the inner receptive-field loops fully unroll at speed — is a straight-line
|
|
5178
|
+
// f64 reduction acc = bias + Σ inp[…+ox]·wt[…] over int8 taps, then requantize (acc>>SHIFT),
|
|
5179
|
+
// ReLU-clamp, and store one uint8. jz accumulates in f64, but every product is int8×int8 (≤ 16129)
|
|
5180
|
+
// and the sum fits i32, so the f64 carries an EXACT integer. That lets us strip-mine the column
|
|
5181
|
+
// loop 8-wide as pure integer SIMD: 8 adjacent outputs (ox..ox+7) in lanes. Per tap, the per-pixel
|
|
5182
|
+
// input gather inp[base+ox] is 8 CONTIGUOUS bytes — `v128.load64_zero` + `i16x8.extend_low_i8x16`
|
|
5183
|
+
// — and the (pixel-invariant) weight broadcasts via `i16x8.splat`; `i16x8.mul` forms 8 products
|
|
5184
|
+
// (each fits i16), widened (`i32x4.extend_low/high_i16x8_s`) into two i32x4 accumulators so 36 taps
|
|
5185
|
+
// never overflow. Requant + clamp + store run scalar per lane; the kept scalar loop is the <8 tail.
|
|
5186
|
+
//
|
|
5187
|
+
// BIT-EXACT: integer arithmetic reorders nothing — each lane's i32 sum equals the scalar f64's exact
|
|
5188
|
+
// integer, and ToInt32(acc)>>SHIFT == lane>>SHIFT. Gated behind cfg.experimentalOuterStrip. ~5×
|
|
5189
|
+
// over the scalar reduction (the serial f64 add-chain is latency-bound; 8 columns hide it).
|
|
5190
|
+
function tryConvColumn(blockNode, fnLocals, freshIdRef, enabled) {
|
|
5191
|
+
if (!enabled) return null
|
|
5192
|
+
const outer = matchOuterPixelLoop(blockNode)
|
|
5193
|
+
if (!outer) return null
|
|
5194
|
+
const { oLabel, loopNode, preamble, pixelIVs, pxVar, widthBound, pivType, obody, oExit } = outer
|
|
5195
|
+
if (pivType.get(pxVar) !== 'i32') return null // strip-mine an integer column
|
|
5196
|
+
for (const s of obody) if (isArr(s) && s[0] === 'block' && s.slice(1).some(c => isArr(c) && c[0] === 'loop')) return null // body must be unrolled (no inner loop)
|
|
5197
|
+
const impureCall = (n) => isArr(n) && ((n[0] === 'call' && typeof n[1] === 'string' && !n[1].startsWith('$math.')) || n.some(impureCall))
|
|
5198
|
+
if (obody.some(impureCall)) return null
|
|
5199
|
+
const readsName = (n, name) => isArr(n) && ((n[0] === 'local.get' && n[1] === name) || n.some(c => readsName(c, name)))
|
|
5200
|
+
|
|
5201
|
+
// Locals whose value depends on the column IV (transitively) — these address the per-pixel gather.
|
|
5202
|
+
const oxDep = new Set([pxVar])
|
|
5203
|
+
const allSets = []
|
|
5204
|
+
const collectSets = (n) => { if (!isArr(n)) return; if ((n[0] === 'local.set' || n[0] === 'local.tee') && typeof n[1] === 'string') allSets.push([n[1], n[2]]); for (const c of n.slice(1)) collectSets(c) }
|
|
5205
|
+
obody.forEach(collectSets)
|
|
5206
|
+
for (let changed = true; changed;) { changed = false; for (const [name, rhs] of allSets) if (!oxDep.has(name) && [...oxDep].some(d => readsName(rhs, d))) { oxDep.add(name); changed = true } }
|
|
5207
|
+
const isGatherAddr = (addr) => [...oxDep].some(d => readsName(addr, d))
|
|
5208
|
+
|
|
5209
|
+
// A byte tap operand: convert_i32_{s,u}(i32.load8_{s,u}(addr)). Returns { load, addr, signed }.
|
|
5210
|
+
const matchByteLoad = (n) => {
|
|
5211
|
+
if (!isArr(n) || (n[0] !== 'f64.convert_i32_s' && n[0] !== 'f64.convert_i32_u') || !isArr(n[1])) return null
|
|
5212
|
+
const ld = n[1]
|
|
5213
|
+
if (ld[0] !== 'i32.load8_s' && ld[0] !== 'i32.load8_u') return null
|
|
5214
|
+
const addr = (typeof ld[1] === 'string' && ld[1].startsWith('offset=')) ? ld[2] : ld[1]
|
|
5215
|
+
return { load: ld, addr, signed: ld[0] === 'i32.load8_s' }
|
|
5216
|
+
}
|
|
5217
|
+
const load64 = (ld) => (typeof ld[1] === 'string' && ld[1].startsWith('offset=')) ? ['v128.load64_zero', ld[1], ld[2]] : ['v128.load64_zero', ld[1]]
|
|
5218
|
+
// Lift a single product addend `inp·wt` (exactly one side gathers on ox) to an i16x8 of 8 products.
|
|
5219
|
+
const liftProduct = (prod) => {
|
|
5220
|
+
if (!isArr(prod) || prod[0] !== 'f64.mul') return null
|
|
5221
|
+
const a = matchByteLoad(prod[1]), b = matchByteLoad(prod[2])
|
|
5222
|
+
if (!a || !b) return null
|
|
5223
|
+
const ag = isGatherAddr(a.addr), bg = isGatherAddr(b.addr)
|
|
5224
|
+
const g = ag && !bg ? a : bg && !ag ? b : null // exactly one per-pixel gather
|
|
5225
|
+
if (!g) return null
|
|
5226
|
+
const inv = g === a ? b : a
|
|
5227
|
+
const gI16 = [g.signed ? 'i16x8.extend_low_i8x16_s' : 'i16x8.extend_low_i8x16_u', load64(g.load)]
|
|
5228
|
+
return ['i16x8.mul', gI16, ['i16x8.splat', inv.load]] // splat the invariant weight (fits i16)
|
|
5229
|
+
}
|
|
5230
|
+
|
|
5231
|
+
// THE accumulator: an f64 local written as `acc = acc + product` (either operand order). Its FIRST
|
|
5232
|
+
// write is the init — a plain invariant `acc = bias`, or (bias folded into the first tap by the
|
|
5233
|
+
// reassociator) `acc = bias + product`.
|
|
5234
|
+
const macAddend = (rhs, name) => (isArr(rhs) && rhs[0] === 'f64.add') ? (isLocalGet(rhs[1], name) ? rhs[2] : isLocalGet(rhs[2], name) ? rhs[1] : null) : null
|
|
5235
|
+
let accName = null
|
|
5236
|
+
for (const [name, rhs] of allSets) if (fnLocals.get(name) === 'f64' && macAddend(rhs, name) != null) { if (accName && accName !== name) return null; accName = name }
|
|
5237
|
+
if (!accName) return null
|
|
5238
|
+
const accIdx = []
|
|
5239
|
+
for (let i = 0; i < obody.length; i++) { const s = obody[i]; if (isArr(s) && s[0] === 'local.set' && s[1] === accName && s.length === 3) accIdx.push(i) }
|
|
5240
|
+
if (accIdx.length < 4) return null
|
|
5241
|
+
const initIdx = accIdx[0], initRhs = obody[initIdx][2]
|
|
5242
|
+
if (readsName(initRhs, accName)) return null // first write must not read acc
|
|
5243
|
+
|
|
5244
|
+
const id = freshIdRef.next++
|
|
5245
|
+
const nm = (s) => `$__cv${id}_${s}`
|
|
5246
|
+
const loV = nm('lo'), hiV = nm('hi'), pV = nm('p')
|
|
5247
|
+
const splatI32 = (e) => ['i32x4.splat', (isArr(e) && (e[0] === 'f64.convert_i32_s' || e[0] === 'f64.convert_i32_u')) ? e[1] : ['i32.trunc_sat_f64_s', e]]
|
|
5248
|
+
const accStmts = (prod) => [
|
|
5249
|
+
['local.set', pV, prod],
|
|
5250
|
+
['local.set', loV, ['i32x4.add', ['local.get', loV], ['i32x4.extend_low_i16x8_s', ['local.get', pV]]]],
|
|
5251
|
+
['local.set', hiV, ['i32x4.add', ['local.get', hiV], ['i32x4.extend_high_i16x8_s', ['local.get', pV]]]],
|
|
5252
|
+
]
|
|
5253
|
+
// Init → lo/hi seeded to the invariant bias, plus the folded first tap (if the bias was fused in).
|
|
5254
|
+
const initStmts = () => {
|
|
5255
|
+
if (isArr(initRhs) && initRhs[0] === 'f64.add') {
|
|
5256
|
+
const pA = liftProduct(initRhs[1]), pB = liftProduct(initRhs[2])
|
|
5257
|
+
const bias = pA && !pB ? initRhs[2] : pB && !pA ? initRhs[1] : null
|
|
5258
|
+
const prod = pA && !pB ? pA : pB && !pA ? pB : null
|
|
5259
|
+
if (!prod || isGatherAddr(bias)) return null
|
|
5260
|
+
return [['local.set', loV, splatI32(bias)], ['local.set', hiV, splatI32(bias)], ...accStmts(prod)]
|
|
5261
|
+
}
|
|
5262
|
+
if (isGatherAddr(initRhs)) return null // plain seed must be loop-invariant
|
|
5263
|
+
return [['local.set', loV, splatI32(initRhs)], ['local.set', hiV, splatI32(initRhs)]]
|
|
5264
|
+
}
|
|
5265
|
+
|
|
5266
|
+
// Build the SIMD body: keep scalar address setup; init→lo/hi seed; each MAC→i16x8 product → lo/hi.
|
|
5267
|
+
const lastMac = accIdx[accIdx.length - 1]
|
|
5268
|
+
const laneCompute = []
|
|
5269
|
+
for (let i = 0; i <= lastMac; i++) {
|
|
5270
|
+
const s = obody[i]
|
|
5271
|
+
if (i === initIdx) { const init = initStmts(); if (!init) return null; laneCompute.push(...init); continue }
|
|
5272
|
+
if (isArr(s) && s[0] === 'local.set' && s[1] === accName) {
|
|
5273
|
+
const addend = macAddend(s[2], accName)
|
|
5274
|
+
if (addend == null) return null // an acc write that isn't acc+product
|
|
5275
|
+
const prod = liftProduct(addend); if (!prod) return null
|
|
5276
|
+
laneCompute.push(...accStmts(prod)); continue
|
|
5277
|
+
}
|
|
5278
|
+
if (readsName(s, accName)) return null // scalar setup must not touch acc
|
|
5279
|
+
laneCompute.push(s)
|
|
5280
|
+
}
|
|
5281
|
+
|
|
5282
|
+
// Epilogue (requant + clamp + store) runs scalar per lane: acc ← the lane's i32 column sum.
|
|
5283
|
+
const epilogue = obody.slice(lastMac + 1)
|
|
5284
|
+
let hasStore = false
|
|
5285
|
+
const findStore = (n) => { if (!isArr(n)) return; if (STORE_OPS[n[0]]) hasStore = true; n.forEach(findStore) }
|
|
5286
|
+
epilogue.forEach(findStore)
|
|
5287
|
+
if (!hasStore) return null
|
|
5288
|
+
const bump = (n, k) => k === 0 ? n
|
|
5289
|
+
: (isArr(n) && n[0] === 'local.get' && pivType.has(n[1])) ? [pivType.get(n[1]) + '.add', n, [pivType.get(n[1]) + '.const', k]]
|
|
5290
|
+
: (isArr(n) ? n.map(c => bump(c, k)) : n)
|
|
5291
|
+
const epiLane = (k) => [
|
|
5292
|
+
['local.set', accName, ['f64.convert_i32_s', ['i32x4.extract_lane', k & 3, ['local.get', k < 4 ? loV : hiV]]]],
|
|
5293
|
+
...epilogue.map(s => bump(s, k)),
|
|
5294
|
+
]
|
|
5295
|
+
|
|
5296
|
+
const newLocalDecls = [['local', loV, 'v128'], ['local', hiV, 'v128'], ['local', pV, 'v128']]
|
|
5297
|
+
const sOut = nm('ob'), sOl = nm('ol')
|
|
5298
|
+
// Guard requires 8 columns available (ox+7 < width); the kept scalar loop finishes the <8 tail.
|
|
5299
|
+
const simdOuter = ['block', sOut, ['loop', sOl,
|
|
5300
|
+
['br_if', sOut, ['i32.eqz', [oExit.cmpOp, bump(['local.get', pxVar], 7), widthBound]]],
|
|
5301
|
+
...laneCompute, ...epiLane(0), ...epiLane(1), ...epiLane(2), ...epiLane(3), ...epiLane(4), ...epiLane(5), ...epiLane(6), ...epiLane(7),
|
|
5302
|
+
...pixelIVs.map(p => ['local.set', p.name, [p.type + '.add', ['local.get', p.name], [p.type + '.const', 8]]]),
|
|
5303
|
+
['br', sOl]]]
|
|
5304
|
+
const wrapper = ['block', nm('w'), ...preamble, simdOuter, ['block', oLabel, loopNode]]
|
|
5305
|
+
return { wrapper, newLocalDecls }
|
|
5306
|
+
}
|
|
5307
|
+
|
|
4221
5308
|
// ---- Mixed-lane tone-map (tryToneMap, experimental) ------------------------
|
|
4222
5309
|
//
|
|
4223
5310
|
// Vectorizes the log-tonemap TAIL shared by fern / bifurcation / attractors:
|
|
@@ -4269,14 +5356,31 @@ function matchInfCanonTone(sel) {
|
|
|
4269
5356
|
return tr ? tr.inner : null
|
|
4270
5357
|
}
|
|
4271
5358
|
|
|
4272
|
-
//
|
|
4273
|
-
//
|
|
4274
|
-
|
|
5359
|
+
// Loads tryToneMap accepts as the f64-island input. `i32.load` is the original (stride-4, no
|
|
5360
|
+
// widening). Narrow typed-array reads (Uint8/Int8/Uint16/Int16) are widened to the low two i32x4
|
|
5361
|
+
// lanes before the f64x2.convert — exactly the F/B/T `dist` term over an 8-bit/16-bit buffer.
|
|
5362
|
+
// `shift` = the address stride exponent (0/1/2). `over` = extra ELEMENTS the widening load reads
|
|
5363
|
+
// past its lane pair (u8 reads 4 bytes via load32_zero for 2 lanes) — the SIMD bound is shrunk by
|
|
5364
|
+
// it so the read never runs off the array; the scalar tail finishes the remainder. The widen step
|
|
5365
|
+
// signedness matches the load op, so the i32x4 lanes hold the exact scalar value.
|
|
5366
|
+
const TONE_LOAD = {
|
|
5367
|
+
'i32.load': { shift: 2, widen: null, over: 0 },
|
|
5368
|
+
'i32.load8_u': { shift: 0, widen: ['v128.load32_zero', ['i16x8.extend_low_i8x16_u', 'i32x4.extend_low_i16x8_u']], over: 2 },
|
|
5369
|
+
'i32.load8_s': { shift: 0, widen: ['v128.load32_zero', ['i16x8.extend_low_i8x16_s', 'i32x4.extend_low_i16x8_s']], over: 2 },
|
|
5370
|
+
'i32.load16_u': { shift: 1, widen: ['v128.load32_zero', ['i32x4.extend_low_i16x8_u']], over: 0 },
|
|
5371
|
+
'i32.load16_s': { shift: 1, widen: ['v128.load32_zero', ['i32x4.extend_low_i16x8_s']], over: 0 },
|
|
5372
|
+
}
|
|
5373
|
+
|
|
5374
|
+
// `base + (i << shift)` (shift 0 ⇒ bare `base + i`) with `base` a loop-invariant array pointer.
|
|
5375
|
+
// The address shape for a load/store at its element stride: the u32 store uses shift 2 (stride-4 ⇒
|
|
5376
|
+
// load64_zero/i64.store cover exactly 2 consecutive u32); a narrow load uses its own stride (0/1).
|
|
5377
|
+
function matchToneAddrShift(addr, ind, shift) {
|
|
4275
5378
|
if (!isArr(addr) || addr[0] !== 'i32.add' || addr.length !== 3) return null
|
|
4276
5379
|
const pair = (baseN, offN) => {
|
|
4277
5380
|
if (!isArr(baseN) || (baseN[0] !== 'local.get' && baseN[0] !== 'global.get')) return null
|
|
4278
5381
|
if (baseN[0] === 'local.get' && baseN[1] === ind) return null
|
|
4279
|
-
if (
|
|
5382
|
+
if (shift === 0) return isLocalGet(offN, ind) ? baseN : null
|
|
5383
|
+
if (isArr(offN) && offN[0] === 'i32.shl' && offN.length === 3 && isLocalGet(offN[1], ind) && constNum(offN[2]) === shift) return baseN
|
|
4280
5384
|
return null
|
|
4281
5385
|
}
|
|
4282
5386
|
return pair(addr[1], addr[2]) || pair(addr[2], addr[1])
|
|
@@ -4321,14 +5425,20 @@ function tryToneMap(bl, fnLocals, freshIdRef, enabled) {
|
|
|
4321
5425
|
// island signature (`f64.convert_i32_*`). Any other-width load/store declines. The
|
|
4322
5426
|
// f64-convert requirement is what distinguishes this from a plain i32 map (tryVectorize,
|
|
4323
5427
|
// which runs earlier and already owns those).
|
|
4324
|
-
let hasConvert = false, storeCount = 0, loadCount = 0, ok = true
|
|
5428
|
+
let hasConvert = false, storeCount = 0, loadCount = 0, overread = 0, ok = true
|
|
4325
5429
|
const scan = (n) => {
|
|
4326
5430
|
if (!ok || !isArr(n)) return
|
|
4327
5431
|
const o = n[0]
|
|
4328
5432
|
if (o === 'f64.convert_i32_s' || o === 'f64.convert_i32_u') hasConvert = true
|
|
4329
|
-
if (o === 'i32.store') { storeCount++; if (!
|
|
4330
|
-
|
|
4331
|
-
if (
|
|
5433
|
+
if (o === 'i32.store') { storeCount++; if (!matchToneAddrShift(n[1], incVar, 2)) ok = false; scan(n[2]); return }
|
|
5434
|
+
const ld = TONE_LOAD[o]
|
|
5435
|
+
if (ld && n.length === 2) { // i32 (stride-4) or a narrow typed-array read at its own stride
|
|
5436
|
+
loadCount++
|
|
5437
|
+
if (!matchToneAddrShift(n[1], incVar, ld.shift)) { ok = false; return }
|
|
5438
|
+
if (ld.over > overread) overread = ld.over
|
|
5439
|
+
return
|
|
5440
|
+
}
|
|
5441
|
+
if (LOAD_OPS[o] || STORE_OPS[o]) { ok = false; return } // any other-width memop → not this shape
|
|
4332
5442
|
for (let i = 1; i < n.length; i++) scan(n[i])
|
|
4333
5443
|
}
|
|
4334
5444
|
for (const s of body) scan(s)
|
|
@@ -4371,10 +5481,16 @@ function tryToneMap(bl, fnLocals, freshIdRef, enabled) {
|
|
|
4371
5481
|
}
|
|
4372
5482
|
}
|
|
4373
5483
|
|
|
4374
|
-
const
|
|
4375
|
-
|
|
5484
|
+
const newLanedLocals = new Map() // origName → laneName (bare string; see getOrAllocLanedLocal)
|
|
5485
|
+
// SAME field set + ORDER as the ctx in tryVectorize / tryReduceVectorize / tryRampMap. The
|
|
5486
|
+
// self-host kernel infers ONE struct layout per shared callee, and `liftFail` is shared with
|
|
5487
|
+
// liftExprV — so every ctx reaching it MUST have the identical shape, or the inferred layout is
|
|
5488
|
+
// wrong for some and field reads corrupt (this is the exact regression 11657cf fixed; the
|
|
5489
|
+
// tone-map's old 3-field ctx re-broke the ENTIRE self-host vectorizer). tryToneMap itself only
|
|
5490
|
+
// reads fail/failReason/extraLocals, but the unused fields must still be present, in order.
|
|
5491
|
+
const ctx = { laneType: 'f64', incVar, rampVar: null, rampTemp: null, widenLoads: false, localKind, fnLocals: null, newLanedLocals, extraLocals: [], freshIdRef, fail: false, failReason: null }
|
|
4376
5492
|
const toneSetBefore = new Set() // lane locals already assigned (conditional-merge gate)
|
|
4377
|
-
const laned = (name) => { let
|
|
5493
|
+
const laned = (name) => { let ln = newLanedLocals.get(name); if (!ln) { ln = `${name}__v`; newLanedLocals.set(name, ln) } return ln }
|
|
4378
5494
|
const freshMask = () => { const mt = `$__mask${freshIdRef.next++}`; ctx.extraLocals.push(['local', mt, 'v128']); return mt }
|
|
4379
5495
|
|
|
4380
5496
|
// The ctx-using lifters are NESTED function declarations that CAPTURE the state above (like
|
|
@@ -4412,7 +5528,13 @@ function tryToneMap(bl, fnLocals, freshIdRef, enabled) {
|
|
|
4412
5528
|
}
|
|
4413
5529
|
const tr = matchTruncF64(expr) // f64 → i32 `|0` bridge
|
|
4414
5530
|
if (tr) { const a = liftV(tr.inner); if (ctx.fail) return null; return [tr.signed ? 'i32x4.trunc_sat_f64x2_s_zero' : 'i32x4.trunc_sat_f64x2_u_zero', a] }
|
|
4415
|
-
if (op
|
|
5531
|
+
if (TONE_LOAD[op] && expr.length === 2) { // i32 → load64_zero (low 2 lanes); narrow → widen to i32x4 low lanes
|
|
5532
|
+
const w = TONE_LOAD[op].widen
|
|
5533
|
+
if (!w) return ['v128.load64_zero', expr[1]] // address kept scalar
|
|
5534
|
+
let v = [w[0], expr[1]]
|
|
5535
|
+
for (let i = 0; i < w[1].length; i++) v = [w[1][i], v]
|
|
5536
|
+
return v
|
|
5537
|
+
}
|
|
4416
5538
|
if (op === 'i32.const') return ['i32x4.splat', expr]
|
|
4417
5539
|
if (op === 'f64.const') return ['f64x2.splat', expr]
|
|
4418
5540
|
if (op === 'local.get' && typeof expr[1] === 'string') {
|
|
@@ -4421,6 +5543,15 @@ function tryToneMap(bl, fnLocals, freshIdRef, enabled) {
|
|
|
4421
5543
|
if (kind === 'invariant') return [fnLocals.get(name) === 'f64' ? 'f64x2.splat' : 'i32x4.splat', expr]
|
|
4422
5544
|
return liftFail(ctx, `tonemap: ${name} address/induction var used as lane data`)
|
|
4423
5545
|
}
|
|
5546
|
+
if (op === 'local.tee' && typeof expr[1] === 'string' && expr.length === 3) {
|
|
5547
|
+
// `let v = X` reused in the same statement folds to `(local.tee $v X)`; lift it to set the
|
|
5548
|
+
// lane local AND yield it (bit-exact — the scalar tee sets $v and returns the same value).
|
|
5549
|
+
const name = expr[1]
|
|
5550
|
+
if (localKind.get(name) !== 'lane') return liftFail(ctx, `tonemap: tee of non-lane ${name}`)
|
|
5551
|
+
const v = liftV(expr[2]); if (ctx.fail) return null
|
|
5552
|
+
toneSetBefore.add(name)
|
|
5553
|
+
return ['local.tee', laned(name), v]
|
|
5554
|
+
}
|
|
4424
5555
|
if (op === 'call' && PPC_CALL2[expr[1]]) { // transcendental → its 2-wide mirror
|
|
4425
5556
|
const args = []
|
|
4426
5557
|
for (let i = 2; i < expr.length; i++) { const a = liftV(expr[i]); if (ctx.fail) return null; args.push(a) }
|
|
@@ -4557,11 +5688,15 @@ function tryToneMap(bl, fnLocals, freshIdRef, enabled) {
|
|
|
4557
5688
|
...lifted,
|
|
4558
5689
|
['local.set', incVar, ['i32.add', ['local.get', incVar], ['i32.const', LANES]]],
|
|
4559
5690
|
['br', simdLoop]]]
|
|
4560
|
-
|
|
5691
|
+
// A widening narrow load reads more elements than its lane pair (u8: 4 bytes for 2 lanes), so shrink
|
|
5692
|
+
// the SIMD bound by that over-read — the widest load stays in-bounds and the scalar tail finishes the
|
|
5693
|
+
// remainder. (A negative bound from a tiny array just leaves the whole loop scalar — safe.)
|
|
5694
|
+
const boundExprAdj = overread > 0 ? ['i32.sub', boundExpr, ['i32.const', overread]] : boundExpr
|
|
5695
|
+
const boundSetup = ['local.set', simdBoundName, ['i32.and', boundExprAdj, ['i32.const', -LANES]]]
|
|
4561
5696
|
const wrapper = ['block', ...preamble.map(cloneNode), boundSetup, simdBlock, bl.blockNode]
|
|
4562
5697
|
const newLocalDecls = [
|
|
4563
5698
|
['local', simdBoundName, 'i32'],
|
|
4564
|
-
...[...newLanedLocals.values()].map(
|
|
5699
|
+
...[...newLanedLocals.values()].map(laneName => ['local', laneName, 'v128']),
|
|
4565
5700
|
...ctx.extraLocals,
|
|
4566
5701
|
]
|
|
4567
5702
|
return { wrapper, newLocalDecls }
|
|
@@ -4573,7 +5708,12 @@ function tryToneMap(bl, fnLocals, freshIdRef, enabled) {
|
|
|
4573
5708
|
* Walk a function looking for vectorizable (block (loop)) pairs, in-place.
|
|
4574
5709
|
* Adds new locals to the function header.
|
|
4575
5710
|
*/
|
|
4576
|
-
|
|
5711
|
+
// opts gates the recognizer set + lift variants (all default-safe so a bare
|
|
5712
|
+
// `vectorizeLaneLocal(fn)` is the conservative scalar-preserving pass):
|
|
5713
|
+
// multiAcc, relaxedFma, blurMP, whyNot, stencil, outerStrip, pureFuncMap, toneMap.
|
|
5714
|
+
export function vectorizeLaneLocal(fn, opts = {}) {
|
|
5715
|
+
const { multiAcc = false, relaxedFma = false, blurMP = true, whyNot = false,
|
|
5716
|
+
stencil = false, outerStrip = false, pureFuncMap = null, toneMap = false, slp = false } = opts
|
|
4577
5717
|
if (!isArr(fn) || fn[0] !== 'func') return
|
|
4578
5718
|
const bodyStart = findBodyStart(fn)
|
|
4579
5719
|
if (bodyStart < 0) return
|
|
@@ -4594,7 +5734,15 @@ export function vectorizeLaneLocal(fn, multiAcc = false, relaxedFma = false, blu
|
|
|
4594
5734
|
const freshIdRef = { next: 0 }
|
|
4595
5735
|
const newLocalDeclsAll = []
|
|
4596
5736
|
|
|
5737
|
+
// Hoist loop-invariant partial products out of unrolled dot reductions (rust/LLVM's
|
|
5738
|
+
// mat4 prologue trick). Reassociates the float sum, so tied to the relaxedFma tier;
|
|
5739
|
+
// runs BEFORE the dot-pair vectorizer so a hoisted dot drops below DOT_UNROLL steps
|
|
5740
|
+
// and stays scalar (faster here than the pack/extract SIMD form — see the lab).
|
|
5741
|
+
if (relaxedFma) hoistReductionInvariantsIn(fn, fnLocals, freshIdRef, newLocalDeclsAll)
|
|
4597
5742
|
vectorizeStraightLineF64DotPairsIn(fn, fnLocals, freshIdRef, newLocalDeclsAll, relaxedFma)
|
|
5743
|
+
// SLP within-iteration store pairs. Sound only with no aliasing typed-array view
|
|
5744
|
+
// in the module (else a shifted view could reorder-hazard the packed read/write).
|
|
5745
|
+
if (slp && !ctx.features.typedView) slpStorePairsIn(fn, fnLocals, freshIdRef, newLocalDeclsAll, slpGetCounts(fn))
|
|
4598
5746
|
|
|
4599
5747
|
// Walk body recursively. Process inner-most matches first (post-order)
|
|
4600
5748
|
// so we don't try to vectorize an outer loop whose inner is the lane-local one.
|
|
@@ -4612,7 +5760,13 @@ export function vectorizeLaneLocal(fn, multiAcc = false, relaxedFma = false, blu
|
|
|
4612
5760
|
// recognizers (divergent-escape, ramp-map, blur, channel-reduce, per-pixel)
|
|
4613
5761
|
// do their own matching on the raw node. Order is preserved exactly — it is
|
|
4614
5762
|
// load-bearing (first match wins).
|
|
4615
|
-
|
|
5763
|
+
// allowInlinedLi: accept an inlined LICM preamble (`$__inl*___li*`) too — jz's
|
|
5764
|
+
// LICM hoists ToInt32/casts of loop-invariant params just before the loop (e.g.
|
|
5765
|
+
// `a[i] & m` with a runtime `m`), and after inlining the snap is renamed off the
|
|
5766
|
+
// bare `$__li*` form. The preamble is pure & loop-invariant by construction
|
|
5767
|
+
// (hasSideEffect-guarded) and cloned ahead of the SIMD block, so this only widens
|
|
5768
|
+
// which loops the recognizers see, never changes a lifted result.
|
|
5769
|
+
const bl = matchBlockLoop(node, { allowPreamble: true, allowInlinedLi: true })
|
|
4616
5770
|
let r = tryDivergentEscapeVectorize(node, fnLocals, freshIdRef)
|
|
4617
5771
|
?? tryMemCopyFill(bl, fnLocals, freshIdRef)
|
|
4618
5772
|
?? tryVectorize(bl, fnLocals, freshIdRef)
|
|
@@ -4625,6 +5779,8 @@ export function vectorizeLaneLocal(fn, multiAcc = false, relaxedFma = false, blu
|
|
|
4625
5779
|
?? tryByteScan(bl, fnLocals, freshIdRef)
|
|
4626
5780
|
?? tryPerPixelColor(node, fnLocals, freshIdRef, pureFuncMap)
|
|
4627
5781
|
?? tryOuterStrip(node, fnLocals, freshIdRef, outerStrip)
|
|
5782
|
+
?? tryIteratedReduce(node, fnLocals, freshIdRef, outerStrip)
|
|
5783
|
+
?? tryConvColumn(node, fnLocals, freshIdRef, outerStrip)
|
|
4628
5784
|
?? tryToneMap(bl, fnLocals, freshIdRef, toneMap)
|
|
4629
5785
|
// --why-not-simd: a canonical loop-shaped candidate that no SIMD pass took.
|
|
4630
5786
|
// Reported BEFORE the scalar strength-reduce fallback (which fires on most
|
|
@@ -4644,8 +5800,10 @@ export function vectorizeLaneLocal(fn, multiAcc = false, relaxedFma = false, blu
|
|
|
4644
5800
|
}
|
|
4645
5801
|
}
|
|
4646
5802
|
_whyNotActive = whyNot
|
|
5803
|
+
_relaxF32 = relaxedFma
|
|
4647
5804
|
for (let i = bodyStart; i < fn.length; i++) walk(fn, i)
|
|
4648
5805
|
_whyNotActive = false
|
|
5806
|
+
_relaxF32 = false
|
|
4649
5807
|
|
|
4650
5808
|
if (newLocalDeclsAll.length) {
|
|
4651
5809
|
// Sibling loops (and the straight-line dot pass) can each lift the SAME source
|