@scriptc/compiler 0.4.0 → 0.4.2

package/dist/backend/llvm/emit-statements.js

@@ -15,20 +15,24 @@ import { emitDiscardedSplice } from "./expr-containers.js";
 import { InternalCompilerError } from "../../errors.js";
 import { emitDenseArrayGet, emitDenseArrayPop, emitDenseArraySet } from "./dense-array-access.js";
 import { emitBorrowedInput } from "./borrowed-inputs.js";
-import { f64Lit, llvmCommentText, sizeToDouble } from "./common.js";
+import { f64Lit, llvmCommentText, loadArrCount, sizeToDouble } from "./common.js";
 import { RESULT_DST_ID, RESULT_ENTRY_PREFIX } from "./record-results.js";
-import { matchIntegerArrayForLoop, matchIntegerBytesForLoop, matchIntegerCountedForLoop, } from "../../ir/integer-loops.js";
+import { matchIntegerArrayForLoop, matchIntegerBytesForLoop, matchIntegerCountedForLoop, matchIntegerPairedForLoop, } from "../../ir/integer-loops.js";
 import { emitByteWindowGuard, matchByteWindow } from "./byte-windows.js";
 import { countedDenseShape, emitDenseLoopGuard, matchDenseArrayLoop } from "./dense-array-loops.js";
-import { emitCountedLoopLimit, emitCountedLoopSafety } from "./counted-loops.js";
+import { emitCountedLoopLimit, emitCountedLoopSafety, emitPairedLoopSafety, } from "./counted-loops.js";
 import { emitBytesLoopVersionGuard } from "./bytes-loop-versions.js";
 import { mangleLocal } from "../mangle.js";
 import { LlvmUnsupportedError } from "./unsupported.js";
+import { everyExprChild, everyStmtChild } from "../../ir/traverse.js";
+import { widenInteger } from "./integer-values.js";
 import { SCALAR_UNION_RESULT } from "./int32-abi.js";
 import { emitLiteralSwitch } from "./switch-dispatch.js";
-import { SPECULATIVE_RANGE, emitSafeIntegerCheck, emitSpeculativeInteger, planSpeculativeLoop, speculativeReads, speculativeWriteTarget, } from "./speculative-integers.js";
+import { SPECULATIVE_RANGE, emitSafeIntegerCheck, emitSpeculativeInteger, planProvenIntegerLoop, planSpeculativeLoop, speculativeReads, speculativeWriteTarget, } from "./speculative-integers.js";
 import { emitConcatInputs, emitMixedConcat, numberPart, stringParts, } from "./string-construction.js";
 import { emitStringInputs } from "./string-lifetimes.js";
+/** A callback loop version copies the loop; keep the copies small. */
+const MAX_CALLBACK_VERSION_COST = 400;
 export function emitStmtBody(emitter, s) {
     const B = emitter.B;
     emitter.frames.push([]);
@@ -216,6 +220,7 @@ export function emitStmtBody(emitter, s) {
                 emitter.toFrame(v);
             B.line(`store ${emitter.llType(b.type)} ${v.name}, ptr ${b.slot}`);
             emitter.storeIntegerView(s.localId, v, s.init);
+            storeIntegerTemp(emitter, s.localId, b, v);
             if (isRefCounted(b.type)) {
                 emitter.scopes[emitter.scopes.length - 1].push({ slot: b.slot, type: b.type });
                 emitter.noteLastUseMove(s, b);
@@ -386,7 +391,7 @@ export function emitStmtBody(emitter, s) {
             const resize = B.newLabel("arr.len.set"), done = B.newLabel("arr.len.done");
             B.line(`${lenPtr} = getelementptr inbounds %ScrArr, ptr ${arr.name}, i32 0, i32 1`);
             emitter.markMemoryPointer(lenPtr, "array:header");
-            B.line(`${len} = load ${emitter.sizeType}, ptr ${lenPtr}${emitter.fieldAliasAttachment(lenPtr)}${emitter.arrayLengthRange}`);
+            loadArrCount(emitter, len, lenPtr, emitter.fieldAliasAttachment(lenPtr));
             B.line(`${lenNumber} = ${sizeToDouble(emitter.sizeType)} ${emitter.sizeType} ${len} to double`);
             B.line(`${same} = fcmp oeq double ${length.name}, ${lenNumber}`);
             B.condBr(same, done, resize);
@@ -443,7 +448,7 @@ export function emitStmtBody(emitter, s) {
             const v = emitter.emitExpr(s.value);
             if (s.arr.type.kind !== "bytes")
                 throw new InternalCompilerError("llvm emitter bug: bytesSet on non-bytes");
-            emitter.emitBytesSet(s.arr.type.elem, arr.name, idx, v, s.index, s.value, emitter.bytesBounds.has(s));
+            emitter.emitBytesSet(s.arr.type.elem, arr.name, idx, v, s.index, s.value, emitter.bytesBounds.has(s), emitter.bytesOwned.has(s));
             break;
         }
         case "fieldSet":
@@ -481,7 +486,7 @@ export function emitStmtBody(emitter, s) {
             else if (emitter.publishedTypes?.recordGuarded(s.shapeId) === true)
                 emitter.emitPublishedFieldGuard(obj.name, null, s.field);
             const { ptr, type, inline } = s.kind === "fieldSet"
-                ? emitter.classFieldPtr(obj.name, s.className, s.field)
+                ? emitter.classFieldPtr(obj.name, s.className, s.field, "compressed")
                 : emitter.recordFieldPtr(obj.name, s.shapeId, s.field);
             // A result-slot entry fills zeroed caller storage, each field once.
             if (s.kind === "recordSet" &&
@@ -704,6 +709,11 @@ export function emitStmtBody(emitter, s) {
             break;
         }
         case "while": {
+            const callbacks = callbackVersionBindings(emitter, s);
+            if (callbacks) {
+                emitCallbackVersionedLoop(emitter, s, callbacks);
+                break;
+            }
             const workerBudget = emitter.workerLoopBudget(s);
             const le = B.newLabel("loop.e");
             emitter.emitLoopCopies(s, true, le, (exit) => {
@@ -712,6 +722,7 @@ export function emitStmtBody(emitter, s) {
                 const lp = workerBudget !== null ? B.newLabel("loop.p") : lc;
                 B.br(lc);
                 B.startBlock(lc);
+                enterLoopHeader(emitter, lc);
                 B.condBr(emitter.emitCondition(s.cond), lb, exit);
                 B.startBlock(lb);
                 emitter.jumpTargets.push({
@@ -736,6 +747,11 @@ export function emitStmtBody(emitter, s) {
             break;
         }
         case "doWhile": {
+            const callbacks = callbackVersionBindings(emitter, s);
+            if (callbacks) {
+                emitCallbackVersionedLoop(emitter, s, callbacks);
+                break;
+            }
             const workerBudget = emitter.workerLoopBudget(s);
             // Body first (runs at least once); continue jumps to the CONDITION.
             const le = B.newLabel("loop.e");
@@ -744,6 +760,7 @@ export function emitStmtBody(emitter, s) {
                 const lc = B.newLabel("loop.c");
                 B.br(lb);
                 B.startBlock(lb);
+                enterLoopHeader(emitter, lb);
                 emitter.jumpTargets.push({
                     kind: "loop",
                     brkLabel: exit,
@@ -764,6 +781,11 @@ export function emitStmtBody(emitter, s) {
             break;
         }
         case "for": {
+            const callbacks = callbackVersionBindings(emitter, s);
+            if (callbacks) {
+                emitCallbackVersionedLoop(emitter, s, callbacks);
+                break;
+            }
             // The init's scope wraps the whole loop (break/continue must NOT
             // release it — scopeDepth captured after the push, C parity).
             const integerArrayLoop = emitter.debug === null
@@ -774,7 +796,13 @@ export function emitStmtBody(emitter, s) {
             const countedLoop = !integerLoop && emitter.countedLoopsEnabled
                 ? matchIntegerCountedForLoop(s, emitter.numericLocals, emitter.integerRanges, emitter.unaliasedArrays)
                 : null;
-            const window = countedLoop && emitter.byteWindowsEnabled
+            // Two counters closing in on each other (`lo++, hi--`) take the same
+            // integer storage as a counted loop's one.
+            const pairedLoop = !integerLoop && !countedLoop && emitter.countedLoopsEnabled
+                ? matchIntegerPairedForLoop(s, emitter.numericLocals, emitter.integerRanges)
+                : null;
+            // Byte windows and dense loops reason about one counter.
+            const window = countedLoop && countedLoop.followers.length === 0 && emitter.byteWindowsEnabled
                 ? matchByteWindow(s, countedLoop, emitter.numericLocals, emitter.captureIds, emitter.integerRanges, emitter.byteWindowEntries.get(s) ?? new Set())
                 : null;
             const denseShape = !emitter.denseLoopsEnabled
@@ -785,7 +813,7 @@ export function emitStmtBody(emitter, s) {
                         length: integerArrayLoop.limitReceiver,
                         range: { min: 0, max: 4294967294 },
                     }
-                    : countedLoop && !countedLoop.guarded
+                    : countedLoop && !countedLoop.guarded && countedLoop.followers.length === 0
                         ? countedDenseShape(s, countedLoop)
                         : null;
             const dense = denseShape && !window
@@ -836,15 +864,22 @@ export function emitStmtBody(emitter, s) {
                 B.startBlock(done);
                 break;
             }
-            const version = emitter.bytesLoopVersionsEnabled &&
+            const versionable = emitter.bytesLoopVersionsEnabled &&
                 emitter.bytesLoopVersioning.size < MAX_BYTES_LOOP_VERSION_DEPTH &&
-                !emitter.bytesLoopVersioning.has(s)
-                ? emitter.matchBytesLoopVersion(s, countedLoop)
-                : null;
+                !emitter.bytesLoopVersioning.has(s);
+            // A loop to a typed array's or an array's length keeps its integer
+            // induction. Its counted form only gives the version guard the
+            // counter's iteration space, so the guard also covers accesses of
+            // other typed arrays at the counter, such as `x[i]` and `y[i]` in
+            // `for (let i = 0; i < out.length; i++) out[i] = x[i] * y[i]`.
+            const versionCounted = versionable && !countedLoop && integerLoop && emitter.countedLoopsEnabled
+                ? matchIntegerCountedForLoop(s, emitter.numericLocals, emitter.integerRanges, emitter.unaliasedArrays)
+                : countedLoop;
+            const version = versionable ? emitter.matchBytesLoopVersion(s, versionCounted) : null;
             if (version) {
                 const fast = B.newLabel("bytes.version.fast"), slow = B.newLabel("bytes.version.slow"), done = B.newLabel("bytes.version.done");
                 emitBytesLoopVersionGuard(emitter, version, fast, slow);
-                const originalBounds = emitter.bytesBounds, originalRanges = emitter.integerRanges;
+                const originalBounds = emitter.bytesBounds, originalOwned = emitter.bytesOwned, originalRanges = emitter.integerRanges;
                 emitter.bytesLoopVersioning.add(s);
                 try {
                     B.startBlock(slow);
@@ -852,6 +887,8 @@ export function emitStmtBody(emitter, s) {
                     B.br(done);
                     B.startBlock(fast);
                     emitter.bytesBounds = new Set([...originalBounds, ...version.bounds]);
+                    if (version.owned.size > 0)
+                        emitter.bytesOwned = new Set([...originalOwned, ...version.owned]);
                     if (version.ranges.size > 0)
                         emitter.integerRanges = new Map([...originalRanges, ...version.ranges]);
                     if (version.counted?.guarded)
@@ -861,6 +898,7 @@ export function emitStmtBody(emitter, s) {
                 }
                 finally {
                     emitter.bytesBounds = originalBounds;
+                    emitter.bytesOwned = originalOwned;
                     emitter.integerRanges = originalRanges;
                     emitter.provenCountedGuards.delete(s);
                     emitter.bytesLoopVersioning.delete(s);
@@ -869,8 +907,10 @@ export function emitStmtBody(emitter, s) {
                 break;
             }
             let countedJoin = null;
-            if (countedLoop?.guarded && !emitter.provenCountedGuards.has(s)) {
-                const safe = emitCountedLoopSafety(emitter, countedLoop);
+            if ((countedLoop?.guarded && !emitter.provenCountedGuards.has(s)) || pairedLoop?.guarded) {
+                const safe = countedLoop
+                    ? emitCountedLoopSafety(emitter, countedLoop)
+                    : emitPairedLoopSafety(emitter, pairedLoop);
                 const fast = B.newLabel("counted.fast");
                 const slow = B.newLabel("counted.slow");
                 countedJoin = B.newLabel("counted.done");
@@ -885,6 +925,8 @@ export function emitStmtBody(emitter, s) {
             emitter.scopes.push([]);
             let integerSlot = null;
             let countedLimit = null;
+            const pairedSlots = [];
+            const followerSlots = [];
             if (integerLoop) {
                 integerSlot = B.slot();
                 B.entryAllocas.push(`${integerSlot} = alloca ${emitter.sizeType} ; integer induction ${emitter.currentLocals.get(integerLoop.localId).name}`);
@@ -923,8 +965,40 @@ export function emitStmtBody(emitter, s) {
                         ? emitter.denseLoop.range
                         : countedLoop.range,
                 });
+                for (const follower of countedLoop.followers) {
+                    const slot = B.slot();
+                    B.entryAllocas.push(`${slot} = alloca i64 ; integer induction ${emitter.currentLocals.get(follower.localId).name}`);
+                    const value = emitter.emitExpr(follower.start);
+                    const followerStart = B.tmp();
+                    B.line(`${followerStart} = fptosi double ${value.name} to i64`);
+                    B.line(`store i64 ${followerStart}, ptr ${slot}`);
+                    followerSlots.push(slot);
+                    emitter.integerLoopBindings.set(follower.localId, {
+                        slot,
+                        type: "i64",
+                        signed: true,
+                        range: follower.range,
+                    });
+                }
                 countedLimit = emitCountedLoopLimit(emitter, countedLoop);
             }
+            else if (pairedLoop) {
+                for (const counter of [pairedLoop.low, pairedLoop.high]) {
+                    const slot = B.slot();
+                    B.entryAllocas.push(`${slot} = alloca i64 ; integer induction ${emitter.currentLocals.get(counter.localId).name}`);
+                    const value = emitter.emitExpr(counter.start);
+                    const start = B.tmp();
+                    B.line(`${start} = fptosi double ${value.name} to i64`);
+                    B.line(`store i64 ${start}, ptr ${slot}`);
+                    pairedSlots.push(slot);
+                    emitter.integerLoopBindings.set(counter.localId, {
+                        slot,
+                        type: "i64",
+                        signed: true,
+                        range: pairedLoop.range,
+                    });
+                }
+            }
             else if (s.init) {
                 // A multi-declarator head shares the loop's scope. Emitting its
                 // IR block as an ordinary block would release captured/ref locals
@@ -966,6 +1040,7 @@ export function emitStmtBody(emitter, s) {
                 const lu = s.update || capturedLets.length > 0 || workerBudget !== null ? B.newLabel("loop.u") : lc;
                 B.br(lc);
                 B.startBlock(lc);
+                enterLoopHeader(emitter, lc);
                 if (integerLoop && integerSlot) {
                     const receiver = emitter.emitStableReceiver(integerLoop.limitReceiver, []);
                     const lenPtr = B.tmp();
@@ -973,10 +1048,25 @@ export function emitStmtBody(emitter, s) {
                     const index = B.tmp();
                     const inBounds = B.tmp();
                     B.line(`${lenPtr} = getelementptr inbounds %${integerArrayLoop ? "ScrArr" : "ScrBytes"}, ptr ${receiver.name}, i64 0, i32 1`);
-                    emitter.markMemoryPointer(lenPtr, integerArrayLoop ? "array:header" : "bytes:header");
-                    B.line(`${len} = load ${emitter.sizeType}, ptr ${lenPtr}${emitter.fieldAliasAttachment(lenPtr)}`);
                     B.line(`${index} = load ${emitter.sizeType}, ptr ${integerSlot}`);
-                    B.line(`${inBounds} = icmp ult ${emitter.sizeType} ${index}, ${len}`);
+                    if (integerArrayLoop) {
+                        // The counter starts at most 2^32 - 1 and steps only while it is
+                        // below the length, so it fits the i32 length field: compare in
+                        // 32 bits, which lets the compare read the field directly.
+                        emitter.markMemoryPointer(lenPtr, "array:header");
+                        B.line(`${len} = load i32, ptr ${lenPtr}${emitter.fieldAliasAttachment(lenPtr)}`);
+                        let narrow = index;
+                        if (emitter.sizeType === "i64") {
+                            narrow = B.tmp();
+                            B.line(`${narrow} = trunc i64 ${index} to i32`);
+                        }
+                        B.line(`${inBounds} = icmp ult i32 ${narrow}, ${len}`);
+                    }
+                    else {
+                        emitter.markMemoryPointer(lenPtr, "bytes:header");
+                        B.line(`${len} = load ${emitter.sizeType}, ptr ${lenPtr}${emitter.fieldAliasAttachment(lenPtr)}`);
+                        B.line(`${inBounds} = icmp ult ${emitter.sizeType} ${index}, ${len}`);
+                    }
                     B.condBr(inBounds, lb, exit);
                 }
                 else if (countedLoop && integerSlot && countedLimit) {
@@ -986,6 +1076,13 @@ export function emitStmtBody(emitter, s) {
                     B.line(`${inBounds} = icmp ${countedLoop.step > 0 ? "slt" : "sgt"} i64 ${index}, ${countedLimit}`);
                     B.condBr(inBounds, lb, exit);
                 }
+                else if (pairedLoop) {
+                    const low = B.tmp(), high = B.tmp(), open = B.tmp();
+                    B.line(`${low} = load i64, ptr ${pairedSlots[0]}`);
+                    B.line(`${high} = load i64, ptr ${pairedSlots[1]}`);
+                    B.line(`${open} = icmp ${pairedLoop.inclusive ? "sle" : "slt"} i64 ${low}, ${high}`);
+                    B.condBr(open, lb, exit);
+                }
                 else if (s.cond)
                     B.condBr(emitter.emitCondition(s.cond), lb, exit);
                 else
@@ -1020,6 +1117,22 @@ export function emitStmtBody(emitter, s) {
                         B.line(`${old} = load i64, ptr ${integerSlot}`);
                         B.line(`${next} = add nsw i64 ${old}, ${countedLoop.step}`);
                         B.line(`store i64 ${next}, ptr ${integerSlot}`);
+                        countedLoop.followers.forEach((follower, i) => {
+                            const was = B.tmp();
+                            const now = B.tmp();
+                            B.line(`${was} = load i64, ptr ${followerSlots[i]}`);
+                            B.line(`${now} = add nsw i64 ${was}, ${follower.step}`);
+                            B.line(`store i64 ${now}, ptr ${followerSlots[i]}`);
+                        });
+                    }
+                    else if (pairedLoop) {
+                        [pairedLoop.low, pairedLoop.high].forEach((counter, i) => {
+                            const old = B.tmp();
+                            const next = B.tmp();
+                            B.line(`${old} = load i64, ptr ${pairedSlots[i]}`);
+                            B.line(`${next} = add nsw i64 ${old}, ${counter.step}`);
+                            B.line(`store i64 ${next}, ptr ${pairedSlots[i]}`);
+                        });
                     }
                     else if (s.update)
                         emitter.emitStmt(s.update);
@@ -1030,8 +1143,15 @@ export function emitStmtBody(emitter, s) {
             emitter.releaseScope(emitter.scopes.pop());
             if (integerLoop)
                 emitter.integerLoopBindings.delete(integerLoop.localId);
-            if (countedLoop)
+            if (countedLoop) {
                 emitter.integerLoopBindings.delete(countedLoop.localId);
+                for (const follower of countedLoop.followers)
+                    emitter.integerLoopBindings.delete(follower.localId);
+            }
+            if (pairedLoop) {
+                emitter.integerLoopBindings.delete(pairedLoop.low.localId);
+                emitter.integerLoopBindings.delete(pairedLoop.high.localId);
+            }
             if (integerArrayLoop)
                 emitter.integerArrayBindings.delete(integerArrayLoop.localId);
             if (countedJoin) {
@@ -1088,7 +1208,7 @@ export function emitStmtBody(emitter, s) {
                 // The inline form of scr_arr_len, like the `length` intrinsic.
                 B.line(`${lenPtr} = getelementptr inbounds %ScrArr, ptr ${arr.name}, i32 0, i32 1`);
                 emitter.markMemoryPointer(lenPtr, "array:header");
-                B.line(`${rawLen} = load ${emitter.sizeType}, ptr ${lenPtr}${emitter.fieldAliasAttachment(lenPtr)}${emitter.arrayLengthRange}`);
+                loadArrCount(emitter, rawLen, lenPtr, emitter.fieldAliasAttachment(lenPtr));
                 B.line(`${len} = ${sizeToDouble(emitter.sizeType)} ${emitter.sizeType} ${rawLen} to double`);
                 B.line(`${inBounds} = fcmp olt double ${i}, ${len}`);
                 cur = i;
@@ -1401,6 +1521,40 @@ export function emitStmtBody(emitter, s) {
  * Abrupt completions inside tryBody/catchBody ride the finallyStack:
  * returns inline copies at the return site, while break/continue use the
  * shared region walk before branching to their resolved target. */
+/** A constant number local whose initializer has an exact integer form
+ * (such as `const j = i * 4` in a typed-array loop version) keeps that
+ * integer in an i64 slot beside the double, so reads such as `px[j + 1]`
+ * index with integer arithmetic instead of converting the double back.
+ * Every other declaration of the local drops the integer: the same
+ * declaration is emitted once per loop copy, and a copy must not read the
+ * integer another copy stored. */
+function storeIntegerTemp(emitter, localId, b, v) {
+    const local = b.local;
+    if (!emitter.integerTempsEnabled ||
+        b.kind !== "local" ||
+        !local ||
+        local.type.kind !== "f64" ||
+        local.mutable ||
+        local.tdz === true ||
+        emitter.captureIds.has(localId) ||
+        !v.integer) {
+        emitter.integerTemps.delete(localId);
+        return;
+    }
+    const B = emitter.B;
+    let slot = emitter.integerTemps.get(localId)?.slot;
+    if (slot === undefined) {
+        slot = B.slot();
+        B.entryAllocas.push(`${slot} = alloca i64 ; integer ${local.name}`);
+    }
+    B.line(`store i64 ${widenInteger(emitter, v.integer)}, ptr ${slot}`);
+    emitter.integerTemps.set(localId, {
+        slot,
+        type: "i64",
+        signed: true,
+        range: v.integer.range,
+    });
+}
 export function emitTryCatch(emitter, s) {
     const B = emitter.B;
     const hasCatch = s.catchBody !== null;
@@ -1653,6 +1807,99 @@ export function emitSwitch(emitter, s) {
     B.br(end);
     B.startBlock(end);
 }
+/** The loop copies a callback loop version needs: the bindings the loop
+ * calls that the constant-callback census resolves to one function, each
+ * with that function's direct entry. Only small loops of optimized
+ * synchronous functions, and only bindings declared before the loop,
+ * qualify: the guard reads each binding before the loop runs. Versions do
+ * not nest: a loop inside either copy of a version (including the
+ * versioned loop itself, emitted again for each copy) keeps its calls as
+ * they are, so a loop nest is emitted at most twice. */
+function callbackVersionBindings(emitter, s) {
+    if (!emitter.callbackVersionsEnabled ||
+        emitter.currentConstantCallbacks.size === 0 ||
+        emitter.callbackVersioning.size > 0)
+        return null;
+    const found = new Map();
+    // Bindings the loop declares or writes (a callback declared in its body).
+    const bound = new Set();
+    let cost = 0;
+    const expr = (e) => {
+        if (++cost > MAX_CALLBACK_VERSION_COST)
+            return false;
+        if (e.kind === "assignExpr")
+            bound.add(e.localId);
+        if (e.kind === "callValue" && e.callee.kind === "varRef" && !found.has(e.callee.localId)) {
+            const local = emitter.currentLocals.get(e.callee.localId);
+            const target = local && local.tdz !== true ? emitter.knownCallbackTarget(e.callee) : undefined;
+            if (target)
+                found.set(e.callee.localId, { ref: e.callee, direct: target.direct });
+        }
+        return everyExprChild(e, expr, stmt);
+    };
+    const stmt = (x) => {
+        if (++cost > MAX_CALLBACK_VERSION_COST)
+            return false;
+        if (x.kind === "varDecl" || x.kind === "assign" || x.kind === "forOf")
+            bound.add(x.localId);
+        else if (x.kind === "tryCatch" && x.catchLocalId !== null)
+            bound.add(x.catchLocalId);
+        return everyStmtChild(x, expr, stmt);
+    };
+    if (!stmt(s))
+        return null;
+    const bindings = [...found.values()].filter(({ ref }) => !bound.has(ref.localId));
+    return bindings.length > 0 ? bindings : null;
+}
+/** A callback loop version. The guard tests, once before the loop, that
+ * each binding's closure holds its known function: a closure's code
+ * pointer is written only when the closure is built, and the census
+ * resolves only bindings the function never rebinds. The copy where the
+ * guard holds calls those functions directly, with no indirect fallback
+ * that could rewrite the closure or what it captures, so LLVM can keep a
+ * captured scalar the callback updates in a register. The other copy is the
+ * ordinary loop. */
+function emitCallbackVersionedLoop(emitter, s, bindings) {
+    const B = emitter.B;
+    let ok = "true";
+    for (const { ref, direct } of bindings) {
+        // A borrowed read: the guard only compares the code pointer.
+        const closure = emitter.emitReadReceiver(ref).name;
+        const fnp = B.tmp(), fn = B.tmp(), same = B.tmp();
+        B.line(`${fnp} = getelementptr inbounds %ScrClosure, ptr ${closure}, i64 0, i32 1`);
+        B.line(`${fn} = load ptr, ptr ${fnp}`);
+        B.line(`${same} = icmp eq ptr ${fn}, ${direct}`);
+        if (ok === "true")
+            ok = same;
+        else {
+            const both = B.tmp();
+            B.line(`${both} = and i1 ${ok}, ${same}`);
+            ok = both;
+        }
+    }
+    const fast = B.newLabel("callback.version.fast"), slow = B.newLabel("callback.version.slow"), done = B.newLabel("callback.version.done");
+    B.condBr(ok, fast, slow);
+    emitter.callbackVersioning.add(s);
+    try {
+        B.startBlock(slow);
+        emitter.emitStmt(s);
+        B.br(done);
+        B.startBlock(fast);
+        for (const { ref } of bindings)
+            emitter.knownCallbacks.add(ref.localId);
+        emitter.emitStmt(s);
+        B.br(done);
+    }
+    finally {
+        for (const { ref } of bindings)
+            emitter.knownCallbacks.delete(ref.localId);
+        emitter.callbackVersioning.delete(s);
+    }
+    B.startBlock(done);
+}
+/** Entries in a row whose fast copy bailed before a speculative loop site
+ * stops trying it. */
+const SPECULATION_BAILS = 8;
 /** Emit a loop's condition, body and update through `emitCopy`, twice
  * when speculative-integers.ts plans a fast integer copy. Both copies
  * share the loop's setup and exit; the ordinary copy comes first so the
@@ -1671,7 +1918,11 @@ export function emitLoopCopies(emitter, loop, allowed, exit, emitCopy) {
         }, (id) => emitter.speculativeLocal(id) || emitter.speculativeInvariantGlobal(id))
         : null;
     if (!plan) {
-        emitCopy(exit);
+        const proven = allowed ? planProvenLoop(emitter, loop) : null;
+        if (proven)
+            emitProvenLoop(emitter, proven, exit, emitCopy);
+        else
+            emitCopy(exit);
         return;
     }
     const B = emitter.B;
@@ -1692,24 +1943,68 @@ export function emitLoopCopies(emitter, loop, allowed, exit, emitCopy) {
             ok = both;
         }
     }
+    // Each entry whose fast copy bails counts at the loop site, and a fast
+    // copy that completes a speculated write and leaves the loop clears the
+    // count. After SPECULATION_BAILS bailing entries in a row, the site runs
+    // the ordinary copy only. The count is a hint: both copies give the same
+    // results, and a race between workers loses at most an update.
+    const bails = `@scr.spec.bails.${emitter.speculativeLoopSites++}`;
+    emitter.declare(`${bails} = internal global i8 0`);
+    const count = B.tmp(), trying = B.tmp();
+    B.line(`${count} = load atomic i8, ptr ${bails} unordered, align 1`);
+    B.line(`${trying} = icmp ult i8 ${count}, ${SPECULATION_BAILS}`);
+    if (ok === "true")
+        ok = trying;
+    else {
+        const both = B.tmp();
+        B.line(`${both} = and i1 ${ok}, ${trying}`);
+        ok = both;
+    }
     const fast = B.newLabel("spec.fast"), slow = B.newLabel("spec.slow");
     B.condBr(ok, fast, slow);
     const resume = new Map();
     for (const s of plan.writes)
         resume.set(s, { label: B.newLabel("spec.resume"), placed: false });
+    // A failed check finishes its iteration in a third copy, entered only
+    // there: it re-executes the statement and the rest of the body and the
+    // update, then enters the ordinary copy through its header. Jumping
+    // into the middle of the ordinary copy instead would give that loop a
+    // second entry, and LLVM does not optimize an irreducible loop.
+    const points = [...resume.values()];
+    let header = null;
     B.startBlock(slow);
+    emitter.speculationHeader = (label) => {
+        header = label;
+    };
+    try {
+        emitCopy(exit);
+    }
+    finally {
+        emitter.speculationHeader = null;
+    }
+    if (header === null)
+        throw new InternalCompilerError("llvm emitter bug: speculative loop copy without a header");
+    B.startBlock(B.newLabel("spec.tail"));
     emitter.speculationResume = resume;
+    emitter.speculationHeader = () => {
+        B.br(header);
+        B.startBlock(B.newLabel("spec.unused"));
+    };
     try {
         emitCopy(exit);
     }
     finally {
         emitter.speculationResume = null;
+        emitter.speculationHeader = null;
     }
     B.startBlock(fast);
-    if (![...resume.values()].every((r) => r.placed)) {
+    if (!points.every((r) => r.placed)) {
         B.br(slow);
         return;
     }
+    const ran = B.slot();
+    B.entryAllocas.push(`${ran} = alloca i1 ; speculated write completed`);
+    B.line(`store i1 false, ptr ${ran}`);
     for (const id of plan.locals.keys()) {
         const shadow = B.slot();
         B.entryAllocas.push(`${shadow} = alloca i64 ; speculative integer ${emitter.currentLocals.get(id)?.name ?? id}`);
@@ -1747,17 +2042,89 @@ export function emitLoopCopies(emitter, loop, allowed, exit, emitCopy) {
         writes: plan.writes,
         resume: new Map([...resume].map(([s, r]) => [s, r.label])),
         deferred,
+        bails,
+        ran,
     };
     try {
-        if (deferred.size === 0)
-            emitCopy(exit);
-        else {
-            const leave = B.newLabel("spec.leave");
-            emitCopy(leave);
-            B.startBlock(leave);
-            emitter.writeBackSpeculativeGlobals(deferred);
-            B.br(exit);
-        }
+        const leave = B.newLabel("spec.leave");
+        emitCopy(leave);
+        B.startBlock(leave);
+        emitter.writeBackDeferred(deferred);
+        // An entry that never ran a speculated write (a loop of no iterations)
+        // says nothing about the site.
+        const wrote = B.tmp(), left = B.tmp(), counted = B.tmp(), both = B.tmp(), clear = B.newLabel("spec.clear");
+        B.line(`${wrote} = load i1, ptr ${ran}`);
+        B.line(`${left} = load atomic i8, ptr ${bails} unordered, align 1`);
+        B.line(`${counted} = icmp ne i8 ${left}, 0`);
+        B.line(`${both} = and i1 ${wrote}, ${counted}`);
+        B.condBr(both, clear, exit);
+        B.startBlock(clear);
+        B.line(`store atomic i8 0, ptr ${bails} unordered, align 1`);
+        B.br(exit);
+    }
+    finally {
+        emitter.speculation = null;
+        for (const id of plan.locals.keys())
+            emitter.integerLoopBindings.delete(id);
+    }
+}
+/** At the top of a loop copy's header: lets a speculative loop find its
+ * ordinary copy's header, and keeps its third copy out of the loop. */
+function enterLoopHeader(emitter, header) {
+    const dispatch = emitter.speculationHeader;
+    emitter.speculationHeader = null;
+    dispatch?.(header);
+}
+/** A proven integer loop (speculative-integers.ts) for this loop, or null.
+ * A shadowed local's double is stale until the loop is left, which only
+ * the loop's exit, a return and an exception do: the loop is planned only
+ * where an exception leaves the function (no enclosing try or finally, no
+ * coroutine frame), so no handler reads the stale double. */
+function planProvenLoop(emitter, loop) {
+    if (loop.kind !== "for" ||
+        !emitter.speculativeLoopsEnabled ||
+        emitter.speculation !== null ||
+        emitter.speculationResume !== null ||
+        emitter.tryStack.length > 0 ||
+        emitter.finallyStack.length > 0 ||
+        emitter.currentCoro !== null)
+        return null;
+    const cursors = emitter.integerLoopRanges.get(loop);
+    if (!cursors)
+        return null;
+    return planProvenIntegerLoop(loop, cursors, (id) => emitter.currentLocals.has(id) && emitter.speculativeLocal(id), (id) => emitter.integerLoopBindings.has(id) || emitter.int32Locals.has(id), (e) => {
+        const range = emitter.integerRanges.get(e);
+        return (range !== undefined &&
+            range !== null &&
+            range.min >= -Number.MAX_SAFE_INTEGER &&
+            range.max <= Number.MAX_SAFE_INTEGER);
+    });
+}
+/** Emit a proven integer loop: its shadowed locals start from their
+ * doubles (integers in their loop ranges on entry), its planned writes
+ * compute integers into the shadows only, and leaving the loop through its
+ * exit writes each double back. */
+function emitProvenLoop(emitter, plan, exit, emitCopy) {
+    const B = emitter.B;
+    for (const [id, range] of plan.locals) {
+        const shadow = B.slot();
+        B.entryAllocas.push(`${shadow} = alloca i64 ; integer ${emitter.currentLocals.get(id)?.name ?? id}`);
+        const value = B.tmp(), integer = B.tmp();
+        B.line(`${value} = load double, ptr ${emitter.binding(id).slot}`);
+        B.line(`${integer} = fptosi double ${value} to i64`);
+        B.line(`store i64 ${integer}, ptr ${shadow}`);
+        emitter.integerLoopBindings.set(id, { slot: shadow, type: "i64", signed: true, range });
+    }
+    const deferred = new Set();
+    for (const id of plan.locals.keys())
+        deferred.add(id);
+    emitter.speculation = { writes: plan.writes, resume: new Map(), deferred };
+    try {
+        const leave = B.newLabel("integer.leave");
+        emitCopy(leave);
+        B.startBlock(leave);
+        emitter.writeBackDeferred(deferred);
+        B.br(exit);
     }
     finally {
         emitter.speculation = null;
@@ -1767,7 +2134,8 @@ export function emitLoopCopies(emitter, loop, allowed, exit, emitCopy) {
 }
 /** A speculated write in the fast copy: compute the value as a safe
  * integer, store it to the shadow and its exact double to the local.
- * Any failed check re-executes the statement in the ordinary copy. */
+ * Any failed check re-executes the statement in the ordinary copy. A
+ * proven integer loop's write (no `resume`) has nothing to check. */
 export function emitSpeculativeWrite(emitter, s, resume) {
     const B = emitter.B;
     const target = speculativeWriteTarget(s);
@@ -1776,7 +2144,7 @@ export function emitSpeculativeWrite(emitter, s, resume) {
     let bails = false;
     emitter.declare("declare i1 @llvm.expect.i1(i1, i1)");
     const bail = (condition) => {
-        if (condition === "false")
+        if (condition === "false" || resume === undefined)
             return;
         bails = true;
         const expected = B.tmp();
@@ -1808,12 +2176,14 @@ export function emitSpeculativeWrite(emitter, s, resume) {
         bail(outside);
     }
     else
-        result = emitSpeculativeInteger(emitter, target.value, bail, reads);
+        result = emitSpeculativeInteger(emitter, target.value, bail, reads, resume === undefined);
     // Speculable values read only numbers past this point.
     if (emitter.frames.pop().length > 0)
         throw new InternalCompilerError("llvm emitter bug: speculative write owns a temporary");
     B.line(`store i64 ${result}, ptr ${binding.slot}`);
-    const deferred = emitter.speculation.deferred;
+    const { deferred, bails: site, ran } = emitter.speculation;
+    if (ran !== undefined)
+        B.line(`store i1 true, ptr ${ran}`);
     if (!deferred.has(target.localId)) {
         const number = B.tmp();
         B.line(`${number} = sitofp i64 ${result} to double`);
@@ -1824,7 +2194,14 @@ export function emitSpeculativeWrite(emitter, s, resume) {
     const done = B.newLabel("spec.done");
     B.br(done);
     B.startBlock(failed);
-    emitter.writeBackSpeculativeGlobals(deferred);
+    emitter.writeBackDeferred(deferred);
+    if (site !== undefined) {
+        const count = B.tmp(), next = B.tmp();
+        emitter.declare("declare i8 @llvm.uadd.sat.i8(i8, i8)");
+        B.line(`${count} = load atomic i8, ptr ${site} unordered, align 1`);
+        B.line(`${next} = call i8 @llvm.uadd.sat.i8(i8 ${count}, i8 1)`);
+        B.line(`store atomic i8 ${next}, ptr ${site} unordered, align 1`);
+    }
     B.br(resume);
     B.startBlock(done);
 }