@scriptc/compiler 0.4.3 → 0.4.5

package/dist/backend/llvm/emit-statements.js

@@ -11,7 +11,7 @@ import { allocateLocalUnion, storeLocalUnion } from "./local-union-storage.js";
 import { boxAccess, boxNewCall, elemAccess } from "./shapes.js";
 import { dynHoleRead } from "./loop-array-borrows.js";
 import { endsWithJump, matchStringSelfConcat } from "../../ir/analysis.js";
-import { charAtUnitRead, emitDiscardedSplice } from "./expr-containers.js";
+import { charAtUnitRead, emitBorrowedMapGet, emitDiscardedSplice } from "./expr-containers.js";
 import { InternalCompilerError } from "../../errors.js";
 import { emitDenseArrayGet, emitDenseArrayPop, emitDenseArraySet, SCR_ELEM_I32, } from "./dense-array-access.js";
 import { emitBorrowedInput } from "./borrowed-inputs.js";
@@ -22,7 +22,7 @@ import { resultSlotStore } from "../result-slots.js";
 import { matchIntegerArrayForLoop, matchIntegerBytesForLoop, matchIntegerCountedForLoop, matchIntegerPairedForLoop, } from "../../ir/integer-loops.js";
 import { emitByteWindowGuard, matchByteWindow } from "./byte-windows.js";
 import { countedDenseShape, emitDenseLoopGuard, matchDenseArrayLoop, pairedDenseShape, } from "./dense-array-loops.js";
-import { emitCountedLoopLimit, emitCountedLoopSafety, emitPairedLoopSafety, } from "./counted-loops.js";
+import { emitCountedLoopLimit, emitCountedLoopSafety, emitCountedStart, emitPairedLoopSafety, } from "./counted-loops.js";
 import { emitBytesLoopVersionGuard } from "./bytes-loop-versions.js";
 import { bytesHeaderAlias } from "./bytes-alias.js";
 import { mangleLocal } from "../mangle.js";
@@ -35,6 +35,7 @@ import { SPECULATIVE_RANGE, emitSafeIntegerCheck, emitSpeculativeInteger, planPr
 import { emitConcatInputs, emitMixedConcat, numberPart, stringParts, } from "./string-construction.js";
 import { emitStringInputs } from "./string-lifetimes.js";
 import { LOOPS } from "./loop-state.js";
+import { loopAccessesTypedArrays } from "../loop-polls.js";
 /** A callback loop version copies the loop; keep the copies small. */
 const MAX_CALLBACK_VERSION_COST = 400;
 export function emitStmtBody(emitter, s) {
@@ -47,6 +48,17 @@ export function emitStmtBody(emitter, s) {
                 break;
             }
             const b = emitter.binding(s.localId);
+            if (emitter.borrowedMapGets.has(s.localId) &&
+                b.kind === "local" &&
+                s.init?.kind === "mapIntrinsic") {
+                // The map entry's own reference: no retain now, no release at scope
+                // exit (findMapReadBorrows).
+                const borrowed = emitBorrowedMapGet(emitter, s.init);
+                if (borrowed) {
+                    B.line(`store ptr ${borrowed.name}, ptr ${b.slot}`);
+                    break;
+                }
+            }
             if (emitter.walkBorrows.has(s.localId) && b.kind === "local") {
                 // A borrowed walk pointer: no retain now, no release at scope exit.
                 const v = s.init === null ? "null" : emitter.emitReadReceiver(s.init).name;
@@ -396,6 +408,10 @@ export function emitStmtBody(emitter, s) {
                 s.length.value === 0 &&
                 emitter.publishedTypes?.containerGuarded(s.arr.type) !== true)
                 break;
+            // The literal it follows was constructed with this capacity
+            // (reservedArrayLength).
+            if (emitter.consumedReserves.delete(s))
+                break;
             const arr = emitBorrowedInput(emitter, s.arr);
             const length = emitter.emitExpr(s.length);
             if (s.arr.type.kind !== "array")
@@ -480,7 +496,7 @@ export function emitStmtBody(emitter, s) {
             const v = emitter.emitExpr(s.value);
             if (s.arr.type.kind !== "bytes")
                 throw new InternalCompilerError("llvm emitter bug: bytesSet on non-bytes");
-            emitter.emitBytesSet(s.arr.type.elem, arr.name, idx, v, s.index, s.value, emitter.bytesBounds.has(s), emitter.bytesOwned.has(s));
+            emitter.emitBytesSet(s.arr.type.elem, arr.name, idx, v, s.index, s.value, emitter.bytesBounds.has(s), emitter.bytesOwned.has(s), emitter.bytesAliasScopes.get(s));
             break;
         }
         case "fieldSet":
@@ -871,7 +887,7 @@ export function emitStmtBody(emitter, s) {
                 LOOPS.of(emitter).denseLoopsEnabled = false;
                 try {
                     B.startBlock(slow);
-                    emitter.emitStmt(s);
+                    emitSlowCopy(emitter, s);
                     B.br(done);
                     B.startBlock(fast);
                     LOOPS.of(emitter).denseLoop = dense;
@@ -893,7 +909,7 @@ export function emitStmtBody(emitter, s) {
                 LOOPS.of(emitter).byteWindowsEnabled = false;
                 try {
                     B.startBlock(slow);
-                    emitter.emitStmt(s);
+                    emitSlowCopy(emitter, s);
                     B.br(done);
                     B.startBlock(fast);
                     emitter.integerRanges = window.ranges;
@@ -939,7 +955,7 @@ export function emitStmtBody(emitter, s) {
                 B.condBr(safe, fast, slow);
                 B.startBlock(slow);
                 LOOPS.of(emitter).countedLoopsEnabled = false;
-                emitter.emitStmt(s);
+                emitSlowCopy(emitter, s);
                 LOOPS.of(emitter).countedLoopsEnabled = true;
                 B.br(countedJoin);
                 B.startBlock(fast);
@@ -975,9 +991,7 @@ export function emitStmtBody(emitter, s) {
             else if (countedLoop) {
                 integerSlot = B.slot();
                 B.entryAllocas.push(`${integerSlot} = alloca i64 ; integer induction ${emitter.currentLocals.get(countedLoop.localId).name}`);
-                const startNumber = emitter.emitExpr(countedLoop.start);
-                const start = B.tmp();
-                B.line(`${start} = fptosi double ${startNumber.name} to i64`);
+                const start = emitCountedStart(emitter, countedLoop.start);
                 B.line(`store i64 ${start}, ptr ${integerSlot}`);
                 emitter.integerLoopBindings.set(countedLoop.localId, {
                     slot: integerSlot,
@@ -990,9 +1004,7 @@ export function emitStmtBody(emitter, s) {
                 for (const follower of countedLoop.followers) {
                     const slot = B.slot();
                     B.entryAllocas.push(`${slot} = alloca i64 ; integer induction ${emitter.currentLocals.get(follower.localId).name}`);
-                    const value = emitter.emitExpr(follower.start);
-                    const followerStart = B.tmp();
-                    B.line(`${followerStart} = fptosi double ${value.name} to i64`);
+                    const followerStart = emitCountedStart(emitter, follower.start);
                     B.line(`store i64 ${followerStart}, ptr ${slot}`);
                     followerSlots.push(slot);
                     emitter.integerLoopBindings.set(follower.localId, {
@@ -1008,9 +1020,7 @@ export function emitStmtBody(emitter, s) {
                 for (const counter of [pairedLoop.low, pairedLoop.high]) {
                     const slot = B.slot();
                     B.entryAllocas.push(`${slot} = alloca i64 ; integer induction ${emitter.currentLocals.get(counter.localId).name}`);
-                    const value = emitter.emitExpr(counter.start);
-                    const start = B.tmp();
-                    B.line(`${start} = fptosi double ${value.name} to i64`);
+                    const start = emitCountedStart(emitter, counter.start);
                     B.line(`store i64 ${start}, ptr ${slot}`);
                     pairedSlots.push(slot);
                     const dense = LOOPS.of(emitter).denseLoop;
@@ -1057,9 +1067,61 @@ export function emitStmtBody(emitter, s) {
                     B.line(`store ptr ${fresh}, ptr ${slot}`);
                 }
             };
-            const workerBudget = emitter.workerLoopBudget(s);
+            // A counted loop over typed arrays, or a loop to a typed array's
+            // length, polls between blocks of WORKER_POLL_BLOCK iterations instead
+            // of counting a budget in its body: the inner loop over a block then
+            // has no exit besides its own bound, which LLVM can vectorize. A typed
+            // array's length is fixed, so each block can take it as its limit when
+            // the loop does not assign the array's binding.
+            const stripMined = capturedLets.length === 0 &&
+                integerSlot !== null &&
+                ((countedLoop !== null &&
+                    countedLimit !== null &&
+                    Math.abs(countedLoop.step) * WORKER_POLL_BLOCK <= 2 ** 52) ||
+                    (integerLoop !== null && integerArrayLoop === null && integerLoop.stableReceiver)) &&
+                emitter.workerLoopPolls(s) &&
+                loopAccessesTypedArrays(s);
+            const workerBudget = stripMined ? null : emitter.workerLoopBudget(s);
+            // One slot for every copy of the loop: a speculative copy resumes an
+            // iteration in another copy's body, which then reads this bound.
+            let blockBound = null;
+            if (stripMined) {
+                blockBound = B.slot();
+                B.entryAllocas.push(`${blockBound} = alloca ${countedLoop ? "i64" : emitter.sizeType} ; cancellation poll block bound`);
+            }
             freshenBindings();
             emitter.emitLoopCopies(s, capturedLets.length === 0, le, (exit) => {
+                if (stripMined) {
+                    const counter = countedLoop
+                        ? {
+                            slot: integerSlot,
+                            type: "i64",
+                            signed: true,
+                            step: countedLoop.step,
+                            followers: countedLoop.followers.map((f, i) => ({
+                                slot: followerSlots[i],
+                                step: f.step,
+                            })),
+                            limit: () => countedLimit,
+                        }
+                        : {
+                            slot: integerSlot,
+                            type: emitter.sizeType,
+                            signed: false,
+                            step: 1,
+                            followers: [],
+                            limit: () => {
+                                const receiver = emitter.emitStableReceiver(integerLoop.limitReceiver, []);
+                                const lenPtr = B.tmp();
+                                const len = B.tmp();
+                                B.line(`${lenPtr} = getelementptr inbounds %ScrBytes, ptr ${receiver.name}, i64 0, i32 1`);
+                                B.line(`${len} = load ${emitter.sizeType}, ptr ${lenPtr}${bytesHeaderAlias(emitter)}`);
+                                return len;
+                            },
+                        };
+                    emitStripMinedCountedLoop(emitter, s, counter, blockBound, exit);
+                    return;
+                }
                 const lc = B.newLabel("loop.c");
                 const lb = B.newLabel("loop.b");
                 const lu = s.update || capturedLets.length > 0 || workerBudget !== null ? B.newLabel("loop.u") : lc;
@@ -1906,7 +1968,7 @@ function emitCallbackVersionedLoop(emitter, s, bindings) {
     emitter.callbackVersioning.add(s);
     try {
         B.startBlock(slow);
-        emitter.emitStmt(s);
+        emitSlowCopy(emitter, s);
         B.br(done);
         B.startBlock(fast);
         for (const { ref } of bindings)
@@ -1921,41 +1983,144 @@ function emitCallbackVersionedLoop(emitter, s, bindings) {
     }
     B.startBlock(done);
 }
+/** The copy of a versioned loop that runs when its guard fails. Its loops
+ * get no speculative or int32 kind copies, which would emit each loop again
+ * for a path that rarely runs. Proven integer loops, which emit one copy,
+ * are kept. A typed-array version whose guard requires thread-owned
+ * storage does not use it (emitVersionedLoop). */
+function emitSlowCopy(emitter, s) {
+    const copies = emitter.loopCopiesEnabled;
+    emitter.loopCopiesEnabled = false;
+    try {
+        emitter.emitStmt(s);
+    }
+    finally {
+        emitter.loopCopiesEnabled = copies;
+    }
+}
 /** Entries in a row whose fast copy bailed before a speculative loop site
  * stops trying it. */
 const SPECULATION_BAILS = 8;
+/** The attachment of each scoped access in a version's fast copy: the
+ * scope of its own view, and no alias with the scopes of the other views,
+ * whose bytes the guard proved disjoint. The scopes are named after the
+ * function and the version, so every copy of a loop has its own, and
+ * written inline (string-named scopes need no numbered metadata). Also
+ * the scopes, which each fast copy declares on entry (declareAliasScopes). */
+function bytesAliasScopes(emitter, scopes) {
+    const version = emitter.aliasScopedVersions++;
+    const name = emitter.currentFunctionName.replace(/[^A-Za-z0-9_.$%]/g, "_");
+    const domain = '!{!"scriptc typed-array views"}';
+    const scope = (receiver) => `!{!"${name}.${version}.${receiver}", ${domain}}`;
+    const attachments = new Map();
+    for (const receiver of scopes.receivers) {
+        const others = scopes.receivers.filter((other) => other !== receiver).map(scope);
+        attachments.set(receiver, `, !alias.scope !{${scope(receiver)}}, !noalias !{${others.join(", ")}}`);
+    }
+    const result = new Map();
+    for (const [node, receiver] of scopes.nodes)
+        result.set(node, attachments.get(receiver));
+    return { attachments: result, scopes: scopes.receivers.map(scope) };
+}
+/** Declares a version's scopes where a fast copy starts. The guard proves
+ * the views disjoint for this entry only: an enclosing loop may pass other
+ * views on its next pass (`[cur, next] = [next, cur]`). A declared scope
+ * holds from its declaration, and LLVM gives each copy of the declaration
+ * its own scopes when it unrolls or duplicates the enclosing code, so the
+ * no-alias facts of one entry never apply to the accesses of another. */
+function declareAliasScopes(emitter, scopes) {
+    if (scopes.length === 0)
+        return;
+    emitter.declare("declare void @llvm.experimental.noalias.scope.decl(metadata)");
+    for (const scope of scopes)
+        emitter.B.line(`call void @llvm.experimental.noalias.scope.decl(metadata !{${scope}})`);
+}
 /** Emit a loop's condition, body and update through `emitCopy`, twice
  * when speculative-integers.ts plans a fast integer copy. Both copies
  * share the loop's setup and exit; the ordinary copy comes first so the
  * fast copy is only emitted when every resume point exists. */
 /** A typed-array loop version: the guard, then the checked copy, which
  * runs whenever the guard fails, and the copy whose accesses the guard
- * proves in bounds (or thread-owned). */
+ * proves in bounds (or thread-owned). In worker programs a version that
+ * proves both also has a copy for views with shared storage, in bounds but
+ * not owned. */
 function emitVersionedLoop(emitter, s, version) {
     const B = emitter.B;
     const fast = B.newLabel("bytes.version.fast"), slow = B.newLabel("bytes.version.slow"), done = B.newLabel("bytes.version.done");
-    emitBytesLoopVersionGuard(emitter, version, fast, slow);
-    const originalBounds = emitter.bytesBounds, originalOwned = emitter.bytesOwned, originalRanges = emitter.integerRanges;
+    const unitsFast = version.unitsReceiver !== null ? B.newLabel("bytes.version.fast.units") : undefined;
+    const sharedFast = version.sharedCopy && unitsFast === undefined
+        ? B.newLabel("bytes.version.fast.shared")
+        : undefined;
+    const guard = emitBytesLoopVersionGuard(emitter, version, fast, slow, unitsFast, sharedFast);
+    const units = guard?.units ?? null;
+    const originalBounds = emitter.bytesBounds, originalOwned = emitter.bytesOwned, originalRanges = emitter.integerRanges, originalUnitReads = emitter.stringUnitReads, originalLengthReads = emitter.stringLengthReads, originalScopes = emitter.bytesAliasScopes;
     LOOPS.of(emitter).bytesLoopVersioning.add(s);
     try {
         B.startBlock(slow);
-        emitter.emitStmt(s);
+        // A guard that requires thread-owned storage fails for every view of
+        // shared memory, which worker programs pass routinely: that checked
+        // copy keeps its loops' speculative and int32 kind copies.
+        if (version.ownedReceivers.length > 0)
+            emitter.emitStmt(s);
+        else
+            emitSlowCopy(emitter, s);
         B.br(done);
-        B.startBlock(fast);
-        emitter.bytesBounds = new Set([...originalBounds, ...version.bounds]);
         if (version.owned.size > 0)
             emitter.bytesOwned = new Set([...originalOwned, ...version.owned]);
         if (version.ranges.size > 0)
             emitter.integerRanges = new Map([...originalRanges, ...version.ranges]);
+        // Both fast copies run under the guard's span test, so they share the
+        // version's scopes, and each declares them on entry.
+        const scoped = version.scopes && emitter.debug === null ? bytesAliasScopes(emitter, version.scopes) : null;
+        if (scoped)
+            emitter.bytesAliasScopes = new Map([...originalScopes, ...scoped.attachments]);
         if (version.counted?.guarded || version.paired?.loop.guarded)
             LOOPS.of(emitter).provenCountedGuards.add(s);
+        B.startBlock(fast);
+        declareAliasScopes(emitter, scoped?.scopes ?? []);
+        emitter.bytesBounds = new Set([...originalBounds, ...version.bounds]);
+        // Both fast copies read the string's length the guard took.
+        const length = guard?.length;
+        if (length != null && version.unitLengths.size > 0)
+            emitter.stringLengthReads = new Map([
+                ...originalLengthReads,
+                ...[...version.unitLengths].map((node) => [node, length]),
+            ]);
         emitter.emitStmt(s);
         B.br(done);
+        if (sharedFast !== undefined) {
+            // The bounds hold and a view has shared storage: the accesses skip
+            // their bounds checks and stay shared-safe, without alias scopes
+            // (which worker programs give only thread-owned views).
+            B.startBlock(sharedFast);
+            emitter.bytesOwned = originalOwned;
+            emitter.bytesAliasScopes = originalScopes;
+            emitter.emitStmt(s);
+            B.br(done);
+        }
+        if (unitsFast !== undefined && units !== null) {
+            // The string reads take the receiver's UTF-16 units.
+            B.startBlock(unitsFast);
+            declareAliasScopes(emitter, scoped?.scopes ?? []);
+            emitter.bytesBounds = new Set([
+                ...originalBounds,
+                ...[...version.bounds].filter((node) => !version.stringReads.has(node)),
+            ]);
+            emitter.stringUnitReads = new Map([
+                ...originalUnitReads,
+                ...[...version.stringReads].map((node) => [node, units]),
+            ]);
+            emitter.emitStmt(s);
+            B.br(done);
+        }
     }
     finally {
         emitter.bytesBounds = originalBounds;
         emitter.bytesOwned = originalOwned;
         emitter.integerRanges = originalRanges;
+        emitter.stringUnitReads = originalUnitReads;
+        emitter.stringLengthReads = originalLengthReads;
+        emitter.bytesAliasScopes = originalScopes;
         LOOPS.of(emitter).provenCountedGuards.delete(s);
         LOOPS.of(emitter).bytesLoopVersioning.delete(s);
     }
@@ -1964,6 +2129,7 @@ function emitVersionedLoop(emitter, s, version) {
 export function emitLoopCopies(emitter, loop, allowed, exit, emitCopy) {
     const plan = allowed &&
         emitter.speculativeLoopsEnabled &&
+        emitter.loopCopiesEnabled &&
         emitter.speculation === null &&
         emitter.speculationResume === null
         ? planSpeculativeLoop(loop, (id) => emitter.speculativeLocal(id), (id) => emitter.integerLoopBindings.has(id) || emitter.int32Locals.has(id), (e) => {
@@ -1972,7 +2138,7 @@ export function emitLoopCopies(emitter, loop, allowed, exit, emitCopy) {
                 range !== null &&
                 range.min >= -Number.MAX_SAFE_INTEGER &&
                 range.max <= Number.MAX_SAFE_INTEGER);
-        }, (id) => emitter.speculativeLocal(id) || emitter.speculativeInvariantGlobal(id))
+        }, (id) => emitter.speculativeLocal(id) || emitter.speculativeInvariantGlobal(id), emitter.speculationContext())
         : null;
     if (!plan) {
         const ordinary = (exit) => {
@@ -1984,6 +2150,7 @@ export function emitLoopCopies(emitter, loop, allowed, exit, emitCopy) {
         };
         const kinds = allowed &&
             emitter.speculativeLoopsEnabled &&
+            emitter.loopCopiesEnabled &&
             emitter.int32Slots.checkedInt32Arrays.size > 0 &&
             emitter.int32Kinds === null &&
             emitter.speculation === null &&
@@ -2030,6 +2197,18 @@ export function emitLoopCopies(emitter, loop, allowed, exit, emitCopy) {
         B.line(`${both} = and i1 ${ok}, ${trying}`);
         ok = both;
     }
+    // Hoisted leaves live in locals of their own, which the resume copy of
+    // their write reads (see SpeculativeValue).
+    for (const { leaves } of plan.values.values())
+        for (const leaf of leaves) {
+            emitter.currentLocals.set(leaf.localId, {
+                id: leaf.localId,
+                name: "speculated leaf",
+                type: leaf.expr.type,
+                mutable: true,
+            });
+            B.entryAllocas.push(`${emitter.binding(leaf.localId).slot} = alloca double ; speculated leaf`);
+        }
     const fast = B.newLabel("spec.fast"), slow = B.newLabel("spec.slow");
     B.condBr(ok, fast, slow);
     const resume = new Map();
@@ -2056,6 +2235,7 @@ export function emitLoopCopies(emitter, loop, allowed, exit, emitCopy) {
         throw new InternalCompilerError("llvm emitter bug: speculative loop copy without a header");
     B.startBlock(B.newLabel("spec.tail"));
     emitter.speculationResume = resume;
+    emitter.speculationResumeValues = plan.values;
     emitter.speculationHeader = () => {
         B.br(header);
         B.startBlock(B.newLabel("spec.unused"));
@@ -2065,6 +2245,7 @@ export function emitLoopCopies(emitter, loop, allowed, exit, emitCopy) {
     }
     finally {
         emitter.speculationResume = null;
+        emitter.speculationResumeValues = null;
         emitter.speculationHeader = null;
     }
     B.startBlock(fast);
@@ -2114,6 +2295,7 @@ export function emitLoopCopies(emitter, loop, allowed, exit, emitCopy) {
         deferred,
         bails,
         ran,
+        values: plan.values,
     };
     try {
         const leave = B.newLabel("spec.leave");
@@ -2254,6 +2436,80 @@ function emitInt32KindLoop(emitter, plan, exit, ordinary) {
 }
 /** At the top of a loop copy's header: lets a speculative loop find its
  * ordinary copy's header, and keeps its third copy out of the loop. */
+/** Iterations between cancellation polls of a strip-mined counted loop. */
+const WORKER_POLL_BLOCK = 64;
+/** A counted `for` loop of a worker program as blocks of WORKER_POLL_BLOCK
+ * iterations, with the cancellation poll between blocks: the outer header
+ * tests the counter against the loop's limit, each block runs the body up
+ * to the nearer of the limit and WORKER_POLL_BLOCK steps on, and the poll
+ * follows the block. It polls as often as the budget it replaces, and the
+ * inner loop's only exit besides `break` and `return` is its bound. */
+function emitStripMinedCountedLoop(emitter, s, counter, boundSlot, exit) {
+    const B = emitter.B;
+    const { slot, type, signed, step } = counter;
+    const ascending = step > 0;
+    const compare = !signed ? "ult" : ascending ? "slt" : "sgt";
+    const saturating = `llvm.${signed ? "sadd" : "uadd"}.sat.${type}`;
+    const nearer = `llvm.${!signed ? "umin" : ascending ? "smin" : "smax"}.${type}`;
+    const wrap = signed ? "nsw" : "nuw";
+    const lc = B.newLabel("loop.c");
+    const block = B.newLabel("loop.block");
+    const bc = B.newLabel("loop.bc");
+    const lb = B.newLabel("loop.b");
+    const lu = B.newLabel("loop.u");
+    const poll = B.newLabel("loop.poll");
+    emitter.declare(`declare ${type} @${saturating}(${type}, ${type})`);
+    emitter.declare(`declare ${type} @${nearer}(${type}, ${type})`);
+    B.br(lc);
+    B.startBlock(lc);
+    enterLoopHeader(emitter, lc);
+    const start = B.tmp();
+    const open = B.tmp();
+    B.line(`${start} = load ${type}, ptr ${slot}`);
+    const limit = counter.limit();
+    B.line(`${open} = icmp ${compare} ${type} ${start}, ${limit}`);
+    B.condBr(open, block, exit);
+    B.startBlock(block);
+    const stepped = B.tmp();
+    const bound = B.tmp();
+    B.line(`${stepped} = call ${type} @${saturating}(${type} ${start}, ${type} ${step * WORKER_POLL_BLOCK})`);
+    B.line(`${bound} = call ${type} @${nearer}(${type} ${stepped}, ${type} ${limit})`);
+    B.line(`store ${type} ${bound}, ptr ${boundSlot}`);
+    B.br(bc);
+    B.startBlock(bc);
+    const index = B.tmp();
+    const end = B.tmp();
+    const inBlock = B.tmp();
+    B.line(`${index} = load ${type}, ptr ${slot}`);
+    B.line(`${end} = load ${type}, ptr ${boundSlot}`);
+    B.line(`${inBlock} = icmp ${compare} ${type} ${index}, ${end}`);
+    B.condBr(inBlock, lb, poll);
+    B.startBlock(lb);
+    emitter.jumpTargets.push({
+        kind: "loop",
+        brkLabel: exit,
+        contLabel: lu,
+        labels: s.labels,
+        frameDepth: emitter.frames.length,
+        scopeDepth: emitter.scopes.length,
+        finallyDepth: emitter.finallyStack.length,
+    });
+    emitter.emitBlock(s.body);
+    emitter.jumpTargets.pop();
+    B.br(lu);
+    B.startBlock(lu);
+    for (const { slot: stepSlot, step: by } of [{ slot, step }, ...counter.followers]) {
+        const old = B.tmp();
+        const next = B.tmp();
+        B.line(`${old} = load ${type}, ptr ${stepSlot}`);
+        B.line(`${next} = add ${wrap} ${type} ${old}, ${by}`);
+        B.line(`store ${type} ${next}, ptr ${stepSlot}`);
+    }
+    B.br(bc);
+    B.startBlock(poll);
+    emitter.emitWorkerPoll();
+    B.br(lc);
+}
 function enterLoopHeader(emitter, header) {
     const dispatch = emitter.speculationHeader;
     emitter.speculationHeader = null;
@@ -2265,7 +2521,7 @@ function enterLoopHeader(emitter, header) {
  * where an exception leaves the function (no enclosing try or finally, no
  * coroutine frame), so no handler reads the stale double. */
 function planProvenLoop(emitter, loop) {
-    if (loop.kind !== "for" ||
+    if (loop.kind === "doWhile" ||
         !emitter.speculativeLoopsEnabled ||
         emitter.speculation !== null ||
         emitter.speculationResume !== null ||
@@ -2341,9 +2597,22 @@ export function emitSpeculativeWrite(emitter, s, resume) {
     // at once: a bailing path then owns nothing, and reads have no effect
     // whose reordering before the arithmetic could be observed.
     const reads = new Map();
-    if (target.value !== null) {
+    const rewritten = emitter.speculation.values?.get(s);
+    const value = rewritten?.value ?? target.value;
+    if (rewritten !== undefined) {
+        // Hoisted leaves run first, in the write's order, and keep their
+        // doubles for the resume copy.
+        emitter.frames.push([]);
+        for (const leaf of rewritten.leaves) {
+            const evaluated = emitter.emitExpr(leaf.expr);
+            B.line(`store double ${evaluated.name}, ptr ${emitter.binding(leaf.localId).slot}`);
+            reads.set(leaf.ref, evaluated);
+        }
+        emitter.releaseTemps(emitter.frames.pop());
+    }
+    if (value !== null) {
         emitter.frames.push([]);
-        for (const read of speculativeReads(target.value))
+        for (const read of speculativeReads(value))
             reads.set(read, emitter.emitExpr(read));
         emitter.releaseTemps(emitter.frames.pop());
     }
@@ -2360,7 +2629,7 @@ export function emitSpeculativeWrite(emitter, s, resume) {
         bail(outside);
     }
     else
-        result = emitSpeculativeInteger(emitter, target.value, bail, reads, resume === undefined);
+        result = emitSpeculativeInteger(emitter, value, bail, reads, resume === undefined);
     // Speculable values read only numbers past this point.
     if (emitter.frames.pop().length > 0)
         throw new InternalCompilerError("llvm emitter bug: speculative write owns a temporary");