@scriptc/compiler 0.4.1 → 0.4.2

package/dist/backend/llvm/emit-statements.js

@@ -17,18 +17,22 @@ import { emitDenseArrayGet, emitDenseArrayPop, emitDenseArraySet } from "./dense
 import { emitBorrowedInput } from "./borrowed-inputs.js";
 import { f64Lit, llvmCommentText, loadArrCount, sizeToDouble } from "./common.js";
 import { RESULT_DST_ID, RESULT_ENTRY_PREFIX } from "./record-results.js";
-import { matchIntegerArrayForLoop, matchIntegerBytesForLoop, matchIntegerCountedForLoop, } from "../../ir/integer-loops.js";
+import { matchIntegerArrayForLoop, matchIntegerBytesForLoop, matchIntegerCountedForLoop, matchIntegerPairedForLoop, } from "../../ir/integer-loops.js";
 import { emitByteWindowGuard, matchByteWindow } from "./byte-windows.js";
 import { countedDenseShape, emitDenseLoopGuard, matchDenseArrayLoop } from "./dense-array-loops.js";
-import { emitCountedLoopLimit, emitCountedLoopSafety } from "./counted-loops.js";
+import { emitCountedLoopLimit, emitCountedLoopSafety, emitPairedLoopSafety, } from "./counted-loops.js";
 import { emitBytesLoopVersionGuard } from "./bytes-loop-versions.js";
 import { mangleLocal } from "../mangle.js";
 import { LlvmUnsupportedError } from "./unsupported.js";
+import { everyExprChild, everyStmtChild } from "../../ir/traverse.js";
+import { widenInteger } from "./integer-values.js";
 import { SCALAR_UNION_RESULT } from "./int32-abi.js";
 import { emitLiteralSwitch } from "./switch-dispatch.js";
 import { SPECULATIVE_RANGE, emitSafeIntegerCheck, emitSpeculativeInteger, planProvenIntegerLoop, planSpeculativeLoop, speculativeReads, speculativeWriteTarget, } from "./speculative-integers.js";
 import { emitConcatInputs, emitMixedConcat, numberPart, stringParts, } from "./string-construction.js";
 import { emitStringInputs } from "./string-lifetimes.js";
+/** A callback loop version copies the loop; keep the copies small. */
+const MAX_CALLBACK_VERSION_COST = 400;
 export function emitStmtBody(emitter, s) {
     const B = emitter.B;
     emitter.frames.push([]);
@@ -216,6 +220,7 @@ export function emitStmtBody(emitter, s) {
                 emitter.toFrame(v);
             B.line(`store ${emitter.llType(b.type)} ${v.name}, ptr ${b.slot}`);
             emitter.storeIntegerView(s.localId, v, s.init);
+            storeIntegerTemp(emitter, s.localId, b, v);
             if (isRefCounted(b.type)) {
                 emitter.scopes[emitter.scopes.length - 1].push({ slot: b.slot, type: b.type });
                 emitter.noteLastUseMove(s, b);
@@ -443,7 +448,7 @@ export function emitStmtBody(emitter, s) {
             const v = emitter.emitExpr(s.value);
             if (s.arr.type.kind !== "bytes")
                 throw new InternalCompilerError("llvm emitter bug: bytesSet on non-bytes");
-            emitter.emitBytesSet(s.arr.type.elem, arr.name, idx, v, s.index, s.value, emitter.bytesBounds.has(s));
+            emitter.emitBytesSet(s.arr.type.elem, arr.name, idx, v, s.index, s.value, emitter.bytesBounds.has(s), emitter.bytesOwned.has(s));
             break;
         }
         case "fieldSet":
@@ -481,7 +486,7 @@ export function emitStmtBody(emitter, s) {
             else if (emitter.publishedTypes?.recordGuarded(s.shapeId) === true)
                 emitter.emitPublishedFieldGuard(obj.name, null, s.field);
             const { ptr, type, inline } = s.kind === "fieldSet"
-                ? emitter.classFieldPtr(obj.name, s.className, s.field)
+                ? emitter.classFieldPtr(obj.name, s.className, s.field, "compressed")
                 : emitter.recordFieldPtr(obj.name, s.shapeId, s.field);
             // A result-slot entry fills zeroed caller storage, each field once.
             if (s.kind === "recordSet" &&
@@ -704,6 +709,11 @@ export function emitStmtBody(emitter, s) {
             break;
         }
         case "while": {
+            const callbacks = callbackVersionBindings(emitter, s);
+            if (callbacks) {
+                emitCallbackVersionedLoop(emitter, s, callbacks);
+                break;
+            }
             const workerBudget = emitter.workerLoopBudget(s);
             const le = B.newLabel("loop.e");
             emitter.emitLoopCopies(s, true, le, (exit) => {
@@ -712,6 +722,7 @@ export function emitStmtBody(emitter, s) {
                 const lp = workerBudget !== null ? B.newLabel("loop.p") : lc;
                 B.br(lc);
                 B.startBlock(lc);
+                enterLoopHeader(emitter, lc);
                 B.condBr(emitter.emitCondition(s.cond), lb, exit);
                 B.startBlock(lb);
                 emitter.jumpTargets.push({
@@ -736,6 +747,11 @@ export function emitStmtBody(emitter, s) {
             break;
         }
         case "doWhile": {
+            const callbacks = callbackVersionBindings(emitter, s);
+            if (callbacks) {
+                emitCallbackVersionedLoop(emitter, s, callbacks);
+                break;
+            }
             const workerBudget = emitter.workerLoopBudget(s);
             // Body first (runs at least once); continue jumps to the CONDITION.
             const le = B.newLabel("loop.e");
@@ -744,6 +760,7 @@ export function emitStmtBody(emitter, s) {
                 const lc = B.newLabel("loop.c");
                 B.br(lb);
                 B.startBlock(lb);
+                enterLoopHeader(emitter, lb);
                 emitter.jumpTargets.push({
                     kind: "loop",
                     brkLabel: exit,
@@ -764,6 +781,11 @@ export function emitStmtBody(emitter, s) {
             break;
         }
         case "for": {
+            const callbacks = callbackVersionBindings(emitter, s);
+            if (callbacks) {
+                emitCallbackVersionedLoop(emitter, s, callbacks);
+                break;
+            }
             // The init's scope wraps the whole loop (break/continue must NOT
             // release it — scopeDepth captured after the push, C parity).
             const integerArrayLoop = emitter.debug === null
@@ -774,7 +796,13 @@ export function emitStmtBody(emitter, s) {
             const countedLoop = !integerLoop && emitter.countedLoopsEnabled
                 ? matchIntegerCountedForLoop(s, emitter.numericLocals, emitter.integerRanges, emitter.unaliasedArrays)
                 : null;
-            const window = countedLoop && emitter.byteWindowsEnabled
+            // Two counters closing in on each other (`lo++, hi--`) take the same
+            // integer storage as a counted loop's one.
+            const pairedLoop = !integerLoop && !countedLoop && emitter.countedLoopsEnabled
+                ? matchIntegerPairedForLoop(s, emitter.numericLocals, emitter.integerRanges)
+                : null;
+            // Byte windows and dense loops reason about one counter.
+            const window = countedLoop && countedLoop.followers.length === 0 && emitter.byteWindowsEnabled
                 ? matchByteWindow(s, countedLoop, emitter.numericLocals, emitter.captureIds, emitter.integerRanges, emitter.byteWindowEntries.get(s) ?? new Set())
                 : null;
             const denseShape = !emitter.denseLoopsEnabled
@@ -785,7 +813,7 @@ export function emitStmtBody(emitter, s) {
                         length: integerArrayLoop.limitReceiver,
                         range: { min: 0, max: 4294967294 },
                     }
-                    : countedLoop && !countedLoop.guarded
+                    : countedLoop && !countedLoop.guarded && countedLoop.followers.length === 0
                         ? countedDenseShape(s, countedLoop)
                         : null;
             const dense = denseShape && !window
@@ -836,15 +864,22 @@ export function emitStmtBody(emitter, s) {
                 B.startBlock(done);
                 break;
             }
-            const version = emitter.bytesLoopVersionsEnabled &&
+            const versionable = emitter.bytesLoopVersionsEnabled &&
                 emitter.bytesLoopVersioning.size < MAX_BYTES_LOOP_VERSION_DEPTH &&
-                !emitter.bytesLoopVersioning.has(s)
-                ? emitter.matchBytesLoopVersion(s, countedLoop)
-                : null;
+                !emitter.bytesLoopVersioning.has(s);
+            // A loop to a typed array's or an array's length keeps its integer
+            // induction. Its counted form only gives the version guard the
+            // counter's iteration space, so the guard also covers accesses of
+            // other typed arrays at the counter, such as `x[i]` and `y[i]` in
+            // `for (let i = 0; i < out.length; i++) out[i] = x[i] * y[i]`.
+            const versionCounted = versionable && !countedLoop && integerLoop && emitter.countedLoopsEnabled
+                ? matchIntegerCountedForLoop(s, emitter.numericLocals, emitter.integerRanges, emitter.unaliasedArrays)
+                : countedLoop;
+            const version = versionable ? emitter.matchBytesLoopVersion(s, versionCounted) : null;
             if (version) {
                 const fast = B.newLabel("bytes.version.fast"), slow = B.newLabel("bytes.version.slow"), done = B.newLabel("bytes.version.done");
                 emitBytesLoopVersionGuard(emitter, version, fast, slow);
-                const originalBounds = emitter.bytesBounds, originalRanges = emitter.integerRanges;
+                const originalBounds = emitter.bytesBounds, originalOwned = emitter.bytesOwned, originalRanges = emitter.integerRanges;
                 emitter.bytesLoopVersioning.add(s);
                 try {
                     B.startBlock(slow);
@@ -852,6 +887,8 @@ export function emitStmtBody(emitter, s) {
                     B.br(done);
                     B.startBlock(fast);
                     emitter.bytesBounds = new Set([...originalBounds, ...version.bounds]);
+                    if (version.owned.size > 0)
+                        emitter.bytesOwned = new Set([...originalOwned, ...version.owned]);
                     if (version.ranges.size > 0)
                         emitter.integerRanges = new Map([...originalRanges, ...version.ranges]);
                     if (version.counted?.guarded)
@@ -861,6 +898,7 @@ export function emitStmtBody(emitter, s) {
                 }
                 finally {
                     emitter.bytesBounds = originalBounds;
+                    emitter.bytesOwned = originalOwned;
                     emitter.integerRanges = originalRanges;
                     emitter.provenCountedGuards.delete(s);
                     emitter.bytesLoopVersioning.delete(s);
@@ -869,8 +907,10 @@ export function emitStmtBody(emitter, s) {
                 break;
             }
             let countedJoin = null;
-            if (countedLoop?.guarded && !emitter.provenCountedGuards.has(s)) {
-                const safe = emitCountedLoopSafety(emitter, countedLoop);
+            if ((countedLoop?.guarded && !emitter.provenCountedGuards.has(s)) || pairedLoop?.guarded) {
+                const safe = countedLoop
+                    ? emitCountedLoopSafety(emitter, countedLoop)
+                    : emitPairedLoopSafety(emitter, pairedLoop);
                 const fast = B.newLabel("counted.fast");
                 const slow = B.newLabel("counted.slow");
                 countedJoin = B.newLabel("counted.done");
@@ -885,6 +925,8 @@ export function emitStmtBody(emitter, s) {
             emitter.scopes.push([]);
             let integerSlot = null;
             let countedLimit = null;
+            const pairedSlots = [];
+            const followerSlots = [];
             if (integerLoop) {
                 integerSlot = B.slot();
                 B.entryAllocas.push(`${integerSlot} = alloca ${emitter.sizeType} ; integer induction ${emitter.currentLocals.get(integerLoop.localId).name}`);
@@ -923,8 +965,40 @@ export function emitStmtBody(emitter, s) {
                         ? emitter.denseLoop.range
                         : countedLoop.range,
                 });
+                for (const follower of countedLoop.followers) {
+                    const slot = B.slot();
+                    B.entryAllocas.push(`${slot} = alloca i64 ; integer induction ${emitter.currentLocals.get(follower.localId).name}`);
+                    const value = emitter.emitExpr(follower.start);
+                    const followerStart = B.tmp();
+                    B.line(`${followerStart} = fptosi double ${value.name} to i64`);
+                    B.line(`store i64 ${followerStart}, ptr ${slot}`);
+                    followerSlots.push(slot);
+                    emitter.integerLoopBindings.set(follower.localId, {
+                        slot,
+                        type: "i64",
+                        signed: true,
+                        range: follower.range,
+                    });
+                }
                 countedLimit = emitCountedLoopLimit(emitter, countedLoop);
             }
+            else if (pairedLoop) {
+                for (const counter of [pairedLoop.low, pairedLoop.high]) {
+                    const slot = B.slot();
+                    B.entryAllocas.push(`${slot} = alloca i64 ; integer induction ${emitter.currentLocals.get(counter.localId).name}`);
+                    const value = emitter.emitExpr(counter.start);
+                    const start = B.tmp();
+                    B.line(`${start} = fptosi double ${value.name} to i64`);
+                    B.line(`store i64 ${start}, ptr ${slot}`);
+                    pairedSlots.push(slot);
+                    emitter.integerLoopBindings.set(counter.localId, {
+                        slot,
+                        type: "i64",
+                        signed: true,
+                        range: pairedLoop.range,
+                    });
+                }
+            }
             else if (s.init) {
                 // A multi-declarator head shares the loop's scope. Emitting its
                 // IR block as an ordinary block would release captured/ref locals
@@ -966,6 +1040,7 @@ export function emitStmtBody(emitter, s) {
                 const lu = s.update || capturedLets.length > 0 || workerBudget !== null ? B.newLabel("loop.u") : lc;
                 B.br(lc);
                 B.startBlock(lc);
+                enterLoopHeader(emitter, lc);
                 if (integerLoop && integerSlot) {
                     const receiver = emitter.emitStableReceiver(integerLoop.limitReceiver, []);
                     const lenPtr = B.tmp();
@@ -1001,6 +1076,13 @@ export function emitStmtBody(emitter, s) {
                     B.line(`${inBounds} = icmp ${countedLoop.step > 0 ? "slt" : "sgt"} i64 ${index}, ${countedLimit}`);
                     B.condBr(inBounds, lb, exit);
                 }
+                else if (pairedLoop) {
+                    const low = B.tmp(), high = B.tmp(), open = B.tmp();
+                    B.line(`${low} = load i64, ptr ${pairedSlots[0]}`);
+                    B.line(`${high} = load i64, ptr ${pairedSlots[1]}`);
+                    B.line(`${open} = icmp ${pairedLoop.inclusive ? "sle" : "slt"} i64 ${low}, ${high}`);
+                    B.condBr(open, lb, exit);
+                }
                 else if (s.cond)
                     B.condBr(emitter.emitCondition(s.cond), lb, exit);
                 else
@@ -1035,6 +1117,22 @@ export function emitStmtBody(emitter, s) {
                         B.line(`${old} = load i64, ptr ${integerSlot}`);
                         B.line(`${next} = add nsw i64 ${old}, ${countedLoop.step}`);
                         B.line(`store i64 ${next}, ptr ${integerSlot}`);
+                        countedLoop.followers.forEach((follower, i) => {
+                            const was = B.tmp();
+                            const now = B.tmp();
+                            B.line(`${was} = load i64, ptr ${followerSlots[i]}`);
+                            B.line(`${now} = add nsw i64 ${was}, ${follower.step}`);
+                            B.line(`store i64 ${now}, ptr ${followerSlots[i]}`);
+                        });
+                    }
+                    else if (pairedLoop) {
+                        [pairedLoop.low, pairedLoop.high].forEach((counter, i) => {
+                            const old = B.tmp();
+                            const next = B.tmp();
+                            B.line(`${old} = load i64, ptr ${pairedSlots[i]}`);
+                            B.line(`${next} = add nsw i64 ${old}, ${counter.step}`);
+                            B.line(`store i64 ${next}, ptr ${pairedSlots[i]}`);
+                        });
                     }
                     else if (s.update)
                         emitter.emitStmt(s.update);
@@ -1045,8 +1143,15 @@ export function emitStmtBody(emitter, s) {
             emitter.releaseScope(emitter.scopes.pop());
             if (integerLoop)
                 emitter.integerLoopBindings.delete(integerLoop.localId);
-            if (countedLoop)
+            if (countedLoop) {
                 emitter.integerLoopBindings.delete(countedLoop.localId);
+                for (const follower of countedLoop.followers)
+                    emitter.integerLoopBindings.delete(follower.localId);
+            }
+            if (pairedLoop) {
+                emitter.integerLoopBindings.delete(pairedLoop.low.localId);
+                emitter.integerLoopBindings.delete(pairedLoop.high.localId);
+            }
             if (integerArrayLoop)
                 emitter.integerArrayBindings.delete(integerArrayLoop.localId);
             if (countedJoin) {
@@ -1416,6 +1521,40 @@ export function emitStmtBody(emitter, s) {
  * Abrupt completions inside tryBody/catchBody ride the finallyStack:
  * returns inline copies at the return site, while break/continue use the
  * shared region walk before branching to their resolved target. */
+/** A constant number local whose initializer has an exact integer form
+ * (such as `const j = i * 4` in a typed-array loop version) keeps that
+ * integer in an i64 slot beside the double, so reads such as `px[j + 1]`
+ * index with integer arithmetic instead of converting the double back.
+ * Every other declaration of the local drops the integer: the same
+ * declaration is emitted once per loop copy, and a copy must not read the
+ * integer another copy stored. */
+function storeIntegerTemp(emitter, localId, b, v) {
+    const local = b.local;
+    if (!emitter.integerTempsEnabled ||
+        b.kind !== "local" ||
+        !local ||
+        local.type.kind !== "f64" ||
+        local.mutable ||
+        local.tdz === true ||
+        emitter.captureIds.has(localId) ||
+        !v.integer) {
+        emitter.integerTemps.delete(localId);
+        return;
+    }
+    const B = emitter.B;
+    let slot = emitter.integerTemps.get(localId)?.slot;
+    if (slot === undefined) {
+        slot = B.slot();
+        B.entryAllocas.push(`${slot} = alloca i64 ; integer ${local.name}`);
+    }
+    B.line(`store i64 ${widenInteger(emitter, v.integer)}, ptr ${slot}`);
+    emitter.integerTemps.set(localId, {
+        slot,
+        type: "i64",
+        signed: true,
+        range: v.integer.range,
+    });
+}
 export function emitTryCatch(emitter, s) {
     const B = emitter.B;
     const hasCatch = s.catchBody !== null;
@@ -1668,6 +1807,99 @@ export function emitSwitch(emitter, s) {
     B.br(end);
     B.startBlock(end);
 }
+/** The loop copies a callback loop version needs: the bindings the loop
+ * calls that the constant-callback census resolves to one function, each
+ * with that function's direct entry. Only small loops of optimized
+ * synchronous functions, and only bindings declared before the loop,
+ * qualify: the guard reads each binding before the loop runs. Versions do
+ * not nest: a loop inside either copy of a version (including the
+ * versioned loop itself, emitted again for each copy) keeps its calls as
+ * they are, so a loop nest is emitted at most twice. */
+function callbackVersionBindings(emitter, s) {
+    if (!emitter.callbackVersionsEnabled ||
+        emitter.currentConstantCallbacks.size === 0 ||
+        emitter.callbackVersioning.size > 0)
+        return null;
+    const found = new Map();
+    // Bindings the loop declares or writes (a callback declared in its body).
+    const bound = new Set();
+    let cost = 0;
+    const expr = (e) => {
+        if (++cost > MAX_CALLBACK_VERSION_COST)
+            return false;
+        if (e.kind === "assignExpr")
+            bound.add(e.localId);
+        if (e.kind === "callValue" && e.callee.kind === "varRef" && !found.has(e.callee.localId)) {
+            const local = emitter.currentLocals.get(e.callee.localId);
+            const target = local && local.tdz !== true ? emitter.knownCallbackTarget(e.callee) : undefined;
+            if (target)
+                found.set(e.callee.localId, { ref: e.callee, direct: target.direct });
+        }
+        return everyExprChild(e, expr, stmt);
+    };
+    const stmt = (x) => {
+        if (++cost > MAX_CALLBACK_VERSION_COST)
+            return false;
+        if (x.kind === "varDecl" || x.kind === "assign" || x.kind === "forOf")
+            bound.add(x.localId);
+        else if (x.kind === "tryCatch" && x.catchLocalId !== null)
+            bound.add(x.catchLocalId);
+        return everyStmtChild(x, expr, stmt);
+    };
+    if (!stmt(s))
+        return null;
+    const bindings = [...found.values()].filter(({ ref }) => !bound.has(ref.localId));
+    return bindings.length > 0 ? bindings : null;
+}
+/** A callback loop version. The guard tests, once before the loop, that
+ * each binding's closure holds its known function: a closure's code
+ * pointer is written only when the closure is built, and the census
+ * resolves only bindings the function never rebinds. The copy where the
+ * guard holds calls those functions directly, with no indirect fallback
+ * that could rewrite the closure or what it captures, so LLVM can keep a
+ * captured scalar the callback updates in a register. The other copy is the
+ * ordinary loop. */
+function emitCallbackVersionedLoop(emitter, s, bindings) {
+    const B = emitter.B;
+    let ok = "true";
+    for (const { ref, direct } of bindings) {
+        // A borrowed read: the guard only compares the code pointer.
+        const closure = emitter.emitReadReceiver(ref).name;
+        const fnp = B.tmp(), fn = B.tmp(), same = B.tmp();
+        B.line(`${fnp} = getelementptr inbounds %ScrClosure, ptr ${closure}, i64 0, i32 1`);
+        B.line(`${fn} = load ptr, ptr ${fnp}`);
+        B.line(`${same} = icmp eq ptr ${fn}, ${direct}`);
+        if (ok === "true")
+            ok = same;
+        else {
+            const both = B.tmp();
+            B.line(`${both} = and i1 ${ok}, ${same}`);
+            ok = both;
+        }
+    }
+    const fast = B.newLabel("callback.version.fast"), slow = B.newLabel("callback.version.slow"), done = B.newLabel("callback.version.done");
+    B.condBr(ok, fast, slow);
+    emitter.callbackVersioning.add(s);
+    try {
+        B.startBlock(slow);
+        emitter.emitStmt(s);
+        B.br(done);
+        B.startBlock(fast);
+        for (const { ref } of bindings)
+            emitter.knownCallbacks.add(ref.localId);
+        emitter.emitStmt(s);
+        B.br(done);
+    }
+    finally {
+        for (const { ref } of bindings)
+            emitter.knownCallbacks.delete(ref.localId);
+        emitter.callbackVersioning.delete(s);
+    }
+    B.startBlock(done);
+}
+/** Entries in a row whose fast copy bailed before a speculative loop site
+ * stops trying it. */
+const SPECULATION_BAILS = 8;
 /** Emit a loop's condition, body and update through `emitCopy`, twice
  * when speculative-integers.ts plans a fast integer copy. Both copies
  * share the loop's setup and exit; the ordinary copy comes first so the
@@ -1711,24 +1943,68 @@ export function emitLoopCopies(emitter, loop, allowed, exit, emitCopy) {
             ok = both;
         }
     }
+    // Each entry whose fast copy bails counts at the loop site, and a fast
+    // copy that completes a speculated write and leaves the loop clears the
+    // count. After SPECULATION_BAILS bailing entries in a row, the site runs
+    // the ordinary copy only. The count is a hint: both copies give the same
+    // results, and a race between workers loses at most an update.
+    const bails = `@scr.spec.bails.${emitter.speculativeLoopSites++}`;
+    emitter.declare(`${bails} = internal global i8 0`);
+    const count = B.tmp(), trying = B.tmp();
+    B.line(`${count} = load atomic i8, ptr ${bails} unordered, align 1`);
+    B.line(`${trying} = icmp ult i8 ${count}, ${SPECULATION_BAILS}`);
+    if (ok === "true")
+        ok = trying;
+    else {
+        const both = B.tmp();
+        B.line(`${both} = and i1 ${ok}, ${trying}`);
+        ok = both;
+    }
     const fast = B.newLabel("spec.fast"), slow = B.newLabel("spec.slow");
     B.condBr(ok, fast, slow);
     const resume = new Map();
     for (const s of plan.writes)
         resume.set(s, { label: B.newLabel("spec.resume"), placed: false });
+    // A failed check finishes its iteration in a third copy, entered only
+    // there: it re-executes the statement and the rest of the body and the
+    // update, then enters the ordinary copy through its header. Jumping
+    // into the middle of the ordinary copy instead would give that loop a
+    // second entry, and LLVM does not optimize an irreducible loop.
+    const points = [...resume.values()];
+    let header = null;
     B.startBlock(slow);
+    emitter.speculationHeader = (label) => {
+        header = label;
+    };
+    try {
+        emitCopy(exit);
+    }
+    finally {
+        emitter.speculationHeader = null;
+    }
+    if (header === null)
+        throw new InternalCompilerError("llvm emitter bug: speculative loop copy without a header");
+    B.startBlock(B.newLabel("spec.tail"));
     emitter.speculationResume = resume;
+    emitter.speculationHeader = () => {
+        B.br(header);
+        B.startBlock(B.newLabel("spec.unused"));
+    };
     try {
         emitCopy(exit);
     }
     finally {
         emitter.speculationResume = null;
+        emitter.speculationHeader = null;
     }
     B.startBlock(fast);
-    if (![...resume.values()].every((r) => r.placed)) {
+    if (!points.every((r) => r.placed)) {
         B.br(slow);
         return;
     }
+    const ran = B.slot();
+    B.entryAllocas.push(`${ran} = alloca i1 ; speculated write completed`);
+    B.line(`store i1 false, ptr ${ran}`);
     for (const id of plan.locals.keys()) {
         const shadow = B.slot();
         B.entryAllocas.push(`${shadow} = alloca i64 ; speculative integer ${emitter.currentLocals.get(id)?.name ?? id}`);
@@ -1766,17 +2042,25 @@ export function emitLoopCopies(emitter, loop, allowed, exit, emitCopy) {
         writes: plan.writes,
         resume: new Map([...resume].map(([s, r]) => [s, r.label])),
         deferred,
+        bails,
+        ran,
     };
     try {
-        if (deferred.size === 0)
-            emitCopy(exit);
-        else {
-            const leave = B.newLabel("spec.leave");
-            emitCopy(leave);
-            B.startBlock(leave);
-            emitter.writeBackDeferred(deferred);
-            B.br(exit);
-        }
+        const leave = B.newLabel("spec.leave");
+        emitCopy(leave);
+        B.startBlock(leave);
+        emitter.writeBackDeferred(deferred);
+        // An entry that never ran a speculated write (a loop of no iterations)
+        // says nothing about the site.
+        const wrote = B.tmp(), left = B.tmp(), counted = B.tmp(), both = B.tmp(), clear = B.newLabel("spec.clear");
+        B.line(`${wrote} = load i1, ptr ${ran}`);
+        B.line(`${left} = load atomic i8, ptr ${bails} unordered, align 1`);
+        B.line(`${counted} = icmp ne i8 ${left}, 0`);
+        B.line(`${both} = and i1 ${wrote}, ${counted}`);
+        B.condBr(both, clear, exit);
+        B.startBlock(clear);
+        B.line(`store atomic i8 0, ptr ${bails} unordered, align 1`);
+        B.br(exit);
     }
     finally {
         emitter.speculation = null;
@@ -1784,6 +2068,13 @@ export function emitLoopCopies(emitter, loop, allowed, exit, emitCopy) {
             emitter.integerLoopBindings.delete(id);
     }
 }
+/** At the top of a loop copy's header: lets a speculative loop find its
+ * ordinary copy's header, and keeps its third copy out of the loop. */
+function enterLoopHeader(emitter, header) {
+    const dispatch = emitter.speculationHeader;
+    emitter.speculationHeader = null;
+    dispatch?.(header);
+}
 /** A proven integer loop (speculative-integers.ts) for this loop, or null.
  * A shadowed local's double is stale until the loop is left, which only
  * the loop's exit, a return and an exception do: the loop is planned only
@@ -1890,7 +2181,9 @@ export function emitSpeculativeWrite(emitter, s, resume) {
     if (emitter.frames.pop().length > 0)
         throw new InternalCompilerError("llvm emitter bug: speculative write owns a temporary");
     B.line(`store i64 ${result}, ptr ${binding.slot}`);
-    const deferred = emitter.speculation.deferred;
+    const { deferred, bails: site, ran } = emitter.speculation;
+    if (ran !== undefined)
+        B.line(`store i1 true, ptr ${ran}`);
     if (!deferred.has(target.localId)) {
         const number = B.tmp();
         B.line(`${number} = sitofp i64 ${result} to double`);
@@ -1902,6 +2195,13 @@ export function emitSpeculativeWrite(emitter, s, resume) {
     B.br(done);
     B.startBlock(failed);
     emitter.writeBackDeferred(deferred);
+    if (site !== undefined) {
+        const count = B.tmp(), next = B.tmp();
+        emitter.declare("declare i8 @llvm.uadd.sat.i8(i8, i8)");
+        B.line(`${count} = load atomic i8, ptr ${site} unordered, align 1`);
+        B.line(`${next} = call i8 @llvm.uadd.sat.i8(i8 ${count}, i8 1)`);
+        B.line(`store atomic i8 ${next}, ptr ${site} unordered, align 1`);
+    }
     B.br(resume);
     B.startBlock(done);
 }