@scriptc/compiler 0.4.3 → 0.4.5

package/dist/backend/llvm/emitter.js

@@ -9,22 +9,25 @@ import { CYC_BUF, CYC_HDR, FIRST_EMITTED_CYC_KIND, inlineAllocSupported, RECORD_
 import { youngNurseryRequested } from "../young-nursery.js";
 import { llvmBytes as llBytes, wtf8Bytes } from "../literals.js";
 import { InternalCompilerError } from "../../errors.js";
+import { serializeModuleParts } from "../../ir/serialize.js";
 import { createHash } from "node:crypto";
 import { endsWithJump, matchFieldSelfConcat, readsArraySlot, streamTypedRefEligible, undefinedArmTag, } from "../../ir/analysis.js";
 import { loopTestsEveryIteration } from "../loop-polls.js";
-import { isDynTypedRefType, isRefCounted, isUnitType, POINTER_KINDS, RUNTIME_EMITTER_CLASS, RUNTIME_EVENT_CLASS, RUNTIME_EVENT_TARGET_CLASS, RUNTIME_ERROR_CLASSES, RUNTIME_STREAM_CLASSES, typeEquals, typeKey, VOID, classJsName, } from "../../ir/ir.js";
+import { isDynTypedRefType, isRefCounted, isUnitType, POINTER_KINDS, RUNTIME_EMITTER_CLASS, RUNTIME_EVENT_CLASS, RUNTIME_EVENT_TARGET_CLASS, RUNTIME_ERROR_CLASSES, RUNTIME_STREAM_CLASSES, typeEquals, typeKey, VOID, classJsName, classValueAbiEquals, } from "../../ir/ir.js";
 import { moduleRuntimeFeatures } from "../../ir/runtime-features.js";
 import { captureFreeFunctions, observesStacks } from "../stack-observations.js";
 import { findInitializedByteLoopBindings } from "./byte-windows.js";
 import { matchBytesLoopVersion } from "./bytes-loop-versions.js";
 import { globalSlotAlias } from "./bytes-alias.js";
 import { materializeSplitPiece, } from "./split-loops.js";
-import { scalarizeNumericRecords } from "../../ir/scalar-records.js";
+import { scalarizeLiteralResults, scalarizeNumericRecords } from "../../ir/scalar-records.js";
 import { addRecordResultEntries, resultEntryName } from "./record-results.js";
 import { specializeCallbackCalls } from "../../ir/callback-specialization.js";
 import { EmitJournal, registerReplay, renameChunk, replayJournal, } from "./emit-journal.js";
 import { specializeNumericCalls } from "../../ir/numeric-call-specialization.js";
 import { versionLoopBoundParameters } from "../../ir/parameter-versions.js";
+import { inlineExpressionFunctions } from "../../ir/expression-inlining.js";
+import { pairRowSums } from "../../ir/row-pairs.js";
 import { allocatedLiteral, fillAllocationLength, int32ArrayKey, selectInt32Arrays, } from "../../ir/int32-arrays.js";
 import { everyExpr, everyExprChild, everyStmtChild, everyStmtList } from "../../ir/traverse.js";
 import { INT32_RANGE, withinInt32, } from "../../ir/integer-ranges.js";
@@ -32,13 +35,15 @@ import { unshareNodes } from "../../ir/unshare.js";
 import { runPairedWhileLoopsAsFor } from "../../ir/paired-while-loops.js";
 import { copyStableCaptures } from "../../ir/capture-values.js";
 import { resolveSelfCaptures } from "../../ir/self-captures.js";
-import { analyzeInt32Slots, dynamicFieldStores } from "../../ir/int32-slots.js";
+import { analyzeInt32Slots, dynamicFieldStores, Int32RevisionConflict, Int32Slots, } from "../../ir/int32-slots.js";
 import { analyzeFinalFields } from "../../ir/final-fields.js";
 import { foldConstantGlobals } from "../../ir/constant-globals.js";
 import { dropUnwrittenPropertyBags } from "../../ir/property-bags.js";
 import { freezeInitOnlyGlobals } from "../../ir/init-only-globals.js";
 import { hoistFinalFieldReads } from "../../ir/field-values.js";
 import { holdCharLocalsAsUnits } from "../../ir/char-unit-locals.js";
+import { unitSafeFunctions } from "./string-unit-versions.js";
+import { stringScanInline } from "./string-reads.js";
 import { findInt32Abis, INT32_ABI_SUFFIX, SCALAR_UNION_DUMMY, SCALAR_UNION_RESULT, } from "./int32-abi.js";
 import {} from "./int32-locals.js";
 import { findIntegerClones, NO_INTEGER_CLONES } from "./integer-clones.js";
@@ -47,6 +52,7 @@ import { findInitializerBindings } from "../../ir/initializer-bindings.js";
 import { findConstantNumericTables } from "../../ir/constant-tables.js";
 import { allocateFfiCallbackAdapters, hasForeignFfiCallback, hasRetainedFfiCallback, } from "../ffi-callbacks.js";
 import { computeMayThrow } from "../may-throw.js";
+import { UNWIND_PERSONALITY, UNWIND_RAISE, UNWIND_RAISE_DECLARATION, UNWIND_RESUME, UNWIND_SLOT, UnwindAbiViolation, bridgePlainUnwindCalls, unwindAbiFunctions, } from "./unwind-abi.js";
 import { mangleAsyncInlineAwaitCall, mangleAsyncSpawn, mangleBorrowedFunction, mangleClassObj, mangleFnClosure, mangleFunction, mangleGenSpawn, mangleGlobal, mangleLocal, mangleRecordStruct, mangleWrapper, mangleBorrowingWrapper, } from "../mangle.js";
 import { analyzeCallLifetimes } from "./call-lifetimes.js";
 import { stackObjectStruct, } from "./stack-objects.js";
@@ -62,16 +68,16 @@ import {} from "./map-entry-fusion.js";
 import { emitBorrowedFieldSequence, guardedValue } from "./borrowed-receivers.js";
 import { emitBorrowedInputs } from "./borrowed-inputs.js";
 import { findWalkBorrows } from "./walk-borrows.js";
-import { IrNodeIndex } from "../../ir/node-index.js";
+import { eachNodeOfKinds, IrNodeIndex } from "../../ir/node-index.js";
 import { lastUseRead } from "./local-extents.js";
 import { findReboundParameters } from "./rebound-parameters.js";
 import { ReferenceEffects } from "./reference-effects.js";
 import { ReferenceWrites } from "./reference-writes.js";
 import { LlvmDebugInfo } from "./debug-info.js";
 import { StackCallbacks } from "./stack-callbacks.js";
-import { findAppendedArrayLiterals } from "./appended-array-literals.js";
+import { findAppendedArrayLiterals, reservedArrayLength } from "./appended-array-literals.js";
 import { AmbientReceiverReaders, findConstantCallbacks } from "./constant-callbacks.js";
-import { findLentStringParameters, } from "./string-slices.js";
+import { findLentRecordFields, findLentStringParameters, findRecordFieldProducers, } from "./string-slices.js";
 import { LazyCaptures } from "./lazy-captures.js";
 import { StackCaptures } from "./stack-captures.js";
 import { emitFieldAbsentTest, f64Lit, SCR_ARR_FIELD, SCR_ARR_FROZEN } from "./common.js";
@@ -109,7 +115,9 @@ import { emitNullablePresent, emitUnionPeek, emitUnionTag } from "./union-repr.j
 import { emitConcatInputs, emitMixedConcat, numberPart, stringParts, } from "./string-construction.js";
 import { arrNewCall, boxAccess, elemAccess, boxNewCall, boxReleaseSym, computeTraced, emitInlineRcHelpers, emitNullableRetainWrappers, emitShapeUnionHelpers, FN_ATTRS, inlineRcDecls, llFieldType, mapFlagsOffset, releaseSym, retainSym, traceArg, vAdapters, } from "./shapes.js";
 import { INLINE_LITERAL_BYTES } from "./string-literal-equality.js";
-import { emitFfiCallbackDefs, emitLibDefs, emitModuleParts, emitParts, helperDefs, } from "./emit-module.js";
+import { inlinableNumberFunction, withSpeculativeValue, } from "./speculative-integers.js";
+import { emitFfiCallbackDefs, emitLibDefs, emitModuleParts, emitParts, helperDefs, workerMinFunctions, } from "./emit-module.js";
+import { forkedAnalysis } from "./forked-analysis.js";
 import { emitLoopCopies, emitSpeculativeWrite, emitStmtBody, emitStringSelfConcatAssign, emitSwitch, emitTryCatch, } from "./emit-statements.js";
 import { emitArgPackAndTrampolinePrologue, emitAsyncAwaitCall, emitAsyncScaffolding, emitFunction, emitGenScaffolding, emitWasiFulfill, fiberlessAsyncFunctions, parkingFunctions, } from "./emit-function.js";
 import { analyzeBorrowedReturns, canBorrowCallArgument, canBorrowReceiver, canStackReceiver, emitReadReceiver, fieldAliasMetadata, isStackUnionSource, } from "./receivers.js";
@@ -155,14 +163,53 @@ function versionParameters(mod) {
         return mod;
     return versionLoopBoundParameters(mod, mangleFunction);
 }
-/** The int32 arrays the int32 slot analysis keeps in `mod` (proven and
- * checked), under the inputs LlEmitter gives it (selectInt32Arrays). */
-function provenInt32Arrays(mod) {
+/** The int32 slot analysis of `mod`, with the inputs LlEmitter gives it.
+ * With `revise`, the analysis revises the functions of candidates that drop
+ * (Int32SlotInputs.revise), and `mod` is the module it started from. */
+function analyzeModuleInt32Slots(mod, revise) {
+    const nodeIndex = new IrNodeIndex(mod);
+    const initializerBindings = findInitializerBindings(mod, nodeIndex);
+    const dynamicStores = dynamicFieldStores(mod, undefined, undefined, nodeIndex);
     const slots = analyzeInt32Slots(mod, {
-        dynamicStores: dynamicFieldStores(mod),
-        initializerBindings: findInitializerBindings(mod),
+        dynamicStores,
+        initializerBindings,
+        nodes: nodeIndex,
+        ...(revise === undefined ? {} : { revise }),
     });
-    return new Set([...slots.int32Arrays, ...slots.checkedInt32Arrays]);
+    return { mod, nodeIndex, initializerBindings, dynamicStores, slots };
+}
+/** The candidates an analysis keeps: int32 arrays and checked ones. */
+function keptInt32Arrays(analysis) {
+    return new Set([...analysis.slots.int32Arrays, ...analysis.slots.checkedInt32Arrays]);
+}
+/** An analysis that revised functions (analyzeModuleInt32Slots' `revise`)
+ * as the analysis of `mod`, the module of the functions it ended with: the
+ * node index of `mod`, and its initializer bindings and dynamic stores,
+ * which must be the ones the analysis read. Null when they differ. */
+function revisedAnalysis(analysis, mod) {
+    const nodeIndex = new IrNodeIndex(mod);
+    const initializerBindings = findInitializerBindings(mod, nodeIndex);
+    const dynamicStores = dynamicFieldStores(mod, undefined, undefined, nodeIndex);
+    const read = analysis.initializerBindings;
+    if (initializerBindings.size !== read.size)
+        return null;
+    for (const [name, globals] of initializerBindings) {
+        const other = read.get(name);
+        if (other?.length !== globals.length || globals.some((g, i) => other[i] !== g))
+            return null;
+    }
+    const stores = analysis.dynamicStores;
+    if (dynamicStores.all !== stores.all ||
+        dynamicStores.names.size !== stores.names.size ||
+        [...dynamicStores.names].some((name) => !stores.names.has(name)))
+        return null;
+    return {
+        mod,
+        nodeIndex,
+        initializerBindings,
+        dynamicStores,
+        slots: analysis.slots,
+    };
 }
 /** The functions whose calls the backend may emit as calls of their
  * result-slot entries (record-results.ts). */
@@ -179,26 +226,119 @@ function moduleEmitter(input, options) {
     // Facts are keyed by node identity: emit the same module whether it was
     // lowered in memory or read back from its serialized form.
     const mod = unshareNodes(input);
+    // Each proof selectInt32Arrays asks for analyzes a marked module; the
+    // emitter of the module it returns reuses the last one. A proof that
+    // revised functions analyzed the module of the functions it ended with.
+    const rounds = { last: null, functions: null, revising: true };
+    const proven = (marked, revise) => {
+        rounds.functions = null;
+        if (revise === null || !rounds.revising) {
+            rounds.last = analyzeModuleInt32Slots(marked);
+            return { kept: keptInt32Arrays(rounds.last), functions: null };
+        }
+        const latest = new Map();
+        try {
+            rounds.last = analyzeModuleInt32Slots(marked, (fnName, dropped) => {
+                const fn = revise(fnName, dropped);
+                latest.set(fnName, fn);
+                return fn;
+            });
+        }
+        catch (error) {
+            if (!(error instanceof Int32RevisionConflict))
+                throw error;
+            rounds.last = null;
+            return null;
+        }
+        if (latest.size === 0)
+            return { kept: keptInt32Arrays(rounds.last), functions: null };
+        rounds.functions = marked.functions.map((fn) => latest.get(fn.name) ?? fn);
+        return { kept: keptInt32Arrays(rounds.last), functions: rounds.functions };
+    };
     // Keep source storage intact for debugger inspection in dev builds.
-    return new LlEmitter(options.debugSources === undefined
-        ? selectInt32Arrays(versionParameters(addRecordResultEntries(scalarizeNumericRecords(specializeCallbackCalls(specializeNumericCalls(foldConstantGlobals(copyStableCaptures(resolveSelfCaptures(hoistFinalFieldReads(freezeInitOnlyGlobals(holdCharLocalsAsUnits(dropUnwrittenPropertyBags(runPairedWhileLoopsAsFor(mod)))))))), mangleFunction), mangleFunction)))), provenInt32Arrays, resultEntryTargets)
-        : mod, options);
+    if (options.debugSources !== undefined)
+        return new LlEmitter(mod, options, null, null);
+    const passed = versionParameters(addRecordResultEntries(scalarizeLiteralResults(scalarizeNumericRecords(specializeCallbackCalls(specializeNumericCalls(pairRowSums(inlineExpressionFunctions(foldConstantGlobals(copyStableCaptures(resolveSelfCaptures(hoistFinalFieldReads(freezeInitOnlyGlobals(holdCharLocalsAsUnits(dropUnwrittenPropertyBags(runPairedWhileLoopsAsFor(mod)))))))))), mangleFunction), mangleFunction)))));
+    let emitted = selectInt32Arrays(passed, proven, resultEntryTargets);
+    let prepared = rounds.last;
+    // A proof that revised functions analyzed the module of the functions it
+    // ended with, which is the selected module unless selection fell back to
+    // rounds: then the selected module was proved by its own round.
+    if (prepared !== null && rounds.functions !== null && prepared.mod !== emitted) {
+        const functions = rounds.functions;
+        prepared =
+            emitted !== passed &&
+                emitted.functions.length === functions.length &&
+                emitted.functions.every((fn, i) => fn === functions[i])
+                ? revisedAnalysis(prepared, emitted)
+                : null;
+        if (prepared === null && emitted !== passed) {
+            rounds.revising = false;
+            emitted = selectInt32Arrays(passed, proven, resultEntryTargets);
+            prepared = rounds.last;
+        }
+    }
+    if (process.env["SCRIPTC_INT32_VERIFY"] === "1")
+        verifyInt32Selection(passed, emitted, prepared?.mod === emitted ? prepared : null);
+    return new LlEmitter(emitted, options, null, prepared?.mod === emitted ? prepared : null);
+}
+/** SCRIPTC_INT32_VERIFY=1: throws unless the selection with revisions
+ * (`emitted`, and the analysis the emitter takes over) is the one rounds
+ * without revisions give: the same module, and the same facts. */
+function verifyInt32Selection(passed, emitted, analysis) {
+    let last = null;
+    const reference = selectInt32Arrays(passed, (marked) => {
+        last = analyzeModuleInt32Slots(marked);
+        return { kept: keptInt32Arrays(last), functions: null };
+    }, resultEntryTargets);
+    const text = (mod) => serializeModuleParts(mod).join("");
+    if (text(reference) !== text(emitted))
+        throw new InternalCompilerError("int32 selection: the revised module differs from the rounds'");
+    const facts = (mod, prepared) => (prepared?.mod === mod ? prepared : analyzeModuleInt32Slots(mod)).slots.summary(mod);
+    const expected = facts(reference, last);
+    const actual = facts(emitted, analysis);
+    if (expected !== actual) {
+        const want = expected.split("\n");
+        const got = actual.split("\n");
+        const line = want.findIndex((entry, i) => entry !== got[i]);
+        throw new InternalCompilerError(`int32 selection: the revised analysis differs from the rounds': ${want[line]?.slice(0, 400)} != ${got[line]?.slice(0, 400)}`);
+    }
+}
+/** Emits the module's parts, emitting it again with the functions an
+ * UnwindAbiViolation names on the pending-flag ABI. */
+function emitModuleText(input, options) {
+    for (let excluded = options.unwindExcluded ?? new Set();;) {
+        const emitter = moduleEmitter(input, { ...options, unwindExcluded: excluded });
+        try {
+            return emitter.emitParts();
+        }
+        catch (error) {
+            if (!(error instanceof UnwindAbiViolation))
+                throw error;
+            const names = new Set(excluded);
+            for (const name of emitter.unwindAbi)
+                for (const symbol of [mangleFunction(name), mangleBorrowedFunction(name)])
+                    if (error.symbols.has(symbol) || error.symbols.has(symbol + INT32_ABI_SUFFIX))
+                        names.add(name);
+            if (names.size === excluded.size)
+                throw error;
+            excluded = names;
+        }
+    }
 }
 export function emitLlvmModule(input, options = {}) {
-    return moduleEmitter(input, options).emit();
+    return emitModuleText(input, options).join("\n");
 }
 /** emitLlvmModule's text as parts that concatenate to it, without the copy
  * that joining them makes: a large program's text is 150 MB. */
 export function emitLlvmModuleParts(input, options = {}) {
-    return moduleEmitter(input, options)
-        .emitParts()
-        .flatMap((part, index) => (index === 0 ? [part] : ["\n", part]));
+    return emitModuleText(input, options).flatMap((part, index) => index === 0 ? [part] : ["\n", part]);
 }
 /** Keep large translation units below the host's single-string limit. */
 export function emitLlvmModuleSource(input, options = {}) {
     // IR validation, contracts and the frontend's disposal run after lowering without checks.
     relieveHeapPressure(true);
-    const parts = moduleEmitter(input, options).emitParts();
+    const parts = emitModuleText(input, options);
     const length = parts.reduce((total, part) => total + part.length, Math.max(0, parts.length - 1));
     if (length <= 256 * 1024 * 1024)
         return parts.join("\n");
@@ -208,6 +348,17 @@ export function llStrBytes(text) {
     return llBytes(wtf8Bytes(text));
 }
 const NO_BORROWED = new Set();
+/** The nodes that write a binding by its id (numberGlobalWriters). */
+const GLOBAL_WRITE_KINDS = ["assign", "varDecl", "assignExpr", "incDec"];
+/** Runtime library calls (readsLoopMetrics). */
+const LIB_CALL_KINDS = ["libCall"];
+/** [A-Za-z0-9_] */
+function isIdentifierByte(code) {
+    return ((code >= 48 && code <= 57) ||
+        (code >= 65 && code <= 90) ||
+        (code >= 97 && code <= 122) ||
+        code === 95);
+}
 /** Runtime entry points (and await-call entries) that may park the running
  * execution context, as they appear in emitted function text. */
 export const SUSPENDS = /@(?:scr_(?:await_|module_await\b|promise_await_settled\b|test_sub\b|gen_yield_|async_gen_|fiber_)|sc_a[wd]_)/;
@@ -322,8 +473,9 @@ export class LlEmitter {
     stringSlices = new Map();
     /** Slices written into stack buffers (findScratchStringSlices). */
     scratchStringSlices = new Set();
-    /** String parameters a caller may lend scratch slices to. */
-    lentStringParameters = new Map();
+    /** String parameters, record fields and record producers a caller may
+     * lend scratch slices to. */
+    lendingFacts = { strings: new Map(), records: new Map(), producers: new Map() };
     localStackUnions = new Map();
     localStackObjects = new Map();
     reassignedStackObjects = new Set();
@@ -642,6 +794,7 @@ export class LlEmitter {
     emitLibraryIdentity;
     runtimeAbiMarker;
     detachedExit;
+    targetCpu;
     wasmBoundsChecks;
     /** Interned string literals: UTF-8 text → { symbol, byte length } —
      * first-use order, the runtime ABI’s determinism discipline. */
@@ -729,7 +882,13 @@ export class LlEmitter {
     checkedNarrows;
     callArrayReads = new Map();
     mapReadLifetimes = { locals: new Map(), arguments: new Map() };
+    /** Locals of the current function that hold a Map.get result as the
+     * entry's own reference (findMapReadBorrows, borrowableMapGet). */
+    borrowedMapGets = new Set();
     readNumberMapWrites = new Set();
+    /** The integer views of the current function's number Map/Set keys, by
+     * the key's double operand (noteNumberMapKey, map-number-lookup.ts). */
+    numberMapKeyIntegers = new Map();
     /** The current function's fused string-key Map get-then-set pairs (map-entry-fusion.ts). */
     mapEntryFusion = null;
     callLifetimes;
@@ -799,6 +958,10 @@ export class LlEmitter {
     /** May-throw analysis (computeMayThrow): pending
      * checks are emitted only after calls that can actually raise. */
     mayThrow;
+    /** The functions that propagate exceptions by unwinding (unwind-abi.ts). */
+    unwindAbi;
+    /** Whether the function being emitted is one of unwindAbi. */
+    currentUnwindAbi = false;
     indirectMayThrow;
     /** Worker programs: the functions that poll for termination on entry
      * (workerTerminationPolls); null outside worker programs. */
@@ -837,6 +1000,11 @@ export class LlEmitter {
     /** Locals whose array literal emitStmts left for the `await Promise.all`
      * that reads them (deferredPromiseAllLocal). */
     deferredPromiseAllEntries = new Map();
+    /** Empty array literals that a capacity hint follows directly
+     * (reservedArrayLength), with that hint. Constructing the literal with
+     * the hinted capacity consumes the hint, which then emits nothing. */
+    reservedArrayLengths = new Map();
+    consumedReserves = new Set();
     /** The promise a `return` emitted past its finally regions already
      * produced for its adopting await (borrowed from the return's parked
      * slot); that await takes it instead of evaluating its operand. */
@@ -952,12 +1120,22 @@ export class LlEmitter {
     streamingSplitsEnabled = false;
     initializerBindings;
     /** Whole-program int32 storage and parameter/result facts. */
-    int32Slots;
+    get int32Slots() {
+        return (this.int32SlotsValue ??= this.joinInt32Slots());
+    }
+    int32SlotsValue;
+    /** The join of the int32 slot analysis a forked process computes
+     * (startInt32Slots). */
+    int32SlotsJoin;
     /** The program has no runtime store into a checked-dynamic value
      * (dynamicFieldStores), so live views of classes and records never
      * commit back into their objects. Library builds keep their commits:
      * their values cross a native boundary. */
     liveViewsReadOnly;
+    /** The program freezes, seals or prevents extensions of untyped values
+     * (IrModule.untypedIntegrity): the commits of class views store the
+     * view's integrity state in the instance's integrity word. */
+    untypedIntegrity;
     /** Functions whose direct calls use i32 parameters or results. */
     int32Abis;
     /** Pure number functions with speculative integer clones. */
@@ -987,6 +1165,9 @@ export class LlEmitter {
     integerTempsEnabled = false;
     /** Speculative integer loops (speculative-integers.ts) run in this function. */
     speculativeLoopsEnabled = false;
+    /** False inside the copy of a versioned loop that runs when its guard
+     * fails: loops there get no speculative or int32 kind copies. */
+    loopCopiesEnabled = true;
     /** The fast copy of a speculative integer loop being emitted: its
      * integer-computed writes and where each resumes in the ordinary copy. */
     speculation = null;
@@ -994,6 +1175,37 @@ export class LlEmitter {
     speculativeLoopSites = 0;
     /** The ordinary copy being emitted: resume labels still to place. */
     speculationResume = null;
+    /** The rewritten values of the writes `speculationResume` resumes at. */
+    speculationResumeValues = null;
+    /** Hoisted speculative leaves numbered so far in the function being
+     * emitted (emitFunction resets it): a function's local names must not
+     * depend on which functions the same emitter emitted before it, since
+     * function workers each emit a share of the module. */
+    speculativeLeaves = 0;
+    inlinableFunctions = new Map();
+    /** What speculative loop planning may assume here (SpeculationContext). */
+    speculationContext() {
+        const inlinable = (callee) => {
+            if (!this.inlinableFunctions.has(callee)) {
+                // Unset while its body is checked: a recursive helper is not inlinable.
+                this.inlinableFunctions.set(callee, undefined);
+                const fn = this.fnByName.get(callee);
+                this.inlinableFunctions.set(callee, fn ? inlinableNumberFunction(fn, inlinable) : undefined);
+            }
+            return this.inlinableFunctions.get(callee);
+        };
+        return {
+            inlinable,
+            unaliased: (id) => {
+                const local = this.currentLocals.get(id);
+                return (local !== undefined &&
+                    !local.boxed &&
+                    !this.captureIds.has(id) &&
+                    !this.lazyCaptureBoxes.has(id));
+            },
+            leafId: () => `%spec.leaf.${this.speculativeLeaves++}`,
+        };
+    }
     /** Run once at the top of the next loop copy's header
      * (`enterLoopHeader`): records the ordinary copy's header, or sends the
      * third copy's header there. */
@@ -1031,9 +1243,23 @@ export class LlEmitter {
      * (one-unit tests and switches, charAtUnitRead), with the charAt each
      * stands for: a loop version's proof for the charAt covers its read. */
     charAtUnitReads = new Map();
+    /** String reads a loop version proved in bounds of a string's UTF-16
+     * units, with the units pointer to read (string-unit-versions.ts). */
+    stringUnitReads = new Map();
+    /** `length` reads of a loop version's string, with the i64 UTF-16
+     * length its guard took (bytes-loop-versions.ts). */
+    stringLengthReads = new Map();
     /** Worker programs: typed-array accesses whose views an enclosing version
      * guard proved thread-owned. */
     bytesOwned = new Set();
+    /** The alias-scope attachment of each typed-array element read and write
+     * in the current loop version's fast copy (emitVersionedLoop), and the
+     * one the element access being emitted carries (bytesElementAlias). */
+    bytesAliasScopes = new Map();
+    bytesAccessScope = "";
+    /** Loop versions with alias scopes in the current function; it names
+     * their scopes. */
+    aliasScopedVersions = 0;
     integerViews = new Map();
     /** Module globals the current function owns outright (initializer
      * bindings): their function-local slot, which stands in for the global. */
@@ -1094,18 +1320,26 @@ export class LlEmitter {
     journaled;
     /** Stack-trace mode: functions no capture can include (no frame push). */
     unframedFunctions;
-    constructor(mod, options, journal = null) {
+    constructor(mod, options, journal = null, int32Analysis = null) {
         this.mod = mod;
         this.targetOptions = options;
         this.journal = journal;
         this.journaled =
             journal !== null ||
-                ((options.functionWorkers ?? 1) > 1 &&
-                    options.forkPrimitives !== undefined &&
-                    options.debugSources === undefined);
+                ((options.functionWorkers ?? 1) > 1 && options.forkPrimitives !== undefined);
         // One walk of every body, which the analyses below read instead of
         // walking the bodies themselves.
-        this.nodeIndex = new IrNodeIndex(mod);
+        this.nodeIndex = int32Analysis?.nodeIndex ?? new IrNodeIndex(mod);
+        this.initializerBindings =
+            int32Analysis?.initializerBindings ?? findInitializerBindings(mod, this.nodeIndex);
+        // One walk finds the dynamic field stores every analysis below consults.
+        const dynamicStores = int32Analysis?.dynamicStores ?? dynamicFieldStores(mod, undefined, undefined, this.nodeIndex);
+        // Started before the other analyses, which may run while a forked
+        // process computes it (startInt32Slots).
+        if (int32Analysis !== null)
+            this.int32SlotsValue = int32Analysis.slots;
+        else
+            this.startInt32Slots(dynamicStores);
         this.stackTraces = observesStacks(mod, this.nodeIndex);
         this.nullableUnions = new NullableUnions(mod, options.debugSources === undefined, this.nodeIndex);
         this.mockTimers = this.runtimeFeatures.mockTimers;
@@ -1114,16 +1348,12 @@ export class LlEmitter {
                 ? null
                 : new LlvmDebugInfo(mod.sourceFile, options.debugSources, options.pointerBits, mod.unions, this.nullableUnions.armTypes());
         this.constantNumericTables = findConstantNumericTables(mod);
-        this.initializerBindings = findInitializerBindings(mod, this.nodeIndex);
-        // One walk finds the dynamic field stores every analysis below consults.
-        const dynamicStores = dynamicFieldStores(mod, undefined, undefined, this.nodeIndex);
-        this.int32Slots = analyzeInt32Slots(mod, {
-            dynamicStores,
-            initializerBindings: this.initializerBindings,
-            nodes: this.nodeIndex,
-        });
+        this.untypedIntegrity = mod.untypedIntegrity === true;
         this.liveViewsReadOnly =
-            mod.lib === undefined && !dynamicStores.all && dynamicStores.names.size === 0;
+            mod.lib === undefined &&
+                !this.untypedIntegrity &&
+                !dynamicStores.all &&
+                dynamicStores.names.size === 0;
         this.int32Abis = this.debug === null ? findInt32Abis(mod, this.int32Slots) : new Map();
         this.immortalGlobals = new Set(mod.contracts?.immortalGlobals ?? []);
         this.programLifetime = new Set(mod.contracts?.programLifetime ?? []);
@@ -1159,6 +1389,7 @@ export class LlEmitter {
         this.emitLibraryIdentity = options.emitLibraryIdentity !== false;
         this.runtimeAbiMarker = options.runtimeAbiMarker === true;
         this.detachedExit = options.detachedExit === true;
+        this.targetCpu = options.targetCpu;
         this.wasmBoundsChecks =
             options.wasmBoundsChecks === true ||
                 wasmBoundsChecks({
@@ -1191,6 +1422,16 @@ export class LlEmitter {
             nodes: this.nodeIndex,
         });
         this.mayThrow = mt.fns;
+        this.unwindAbi =
+            options.unwindAbi === true &&
+                !this.wasi &&
+                mod.lib === undefined &&
+                options.debugSources === undefined &&
+                (options.targetTriple
+                    ? /-linux-(?:gnu|musl)/.test(options.targetTriple)
+                    : process.platform === "linux")
+                ? unwindAbiFunctions(mod, this.mayThrow, options.unwindExcluded)
+                : new Set();
         this.workerEntryPolls = mt.workerEntryPolls ?? null;
         this.stackChecks = mt.stackChecks ?? null;
         // A stackless call's eager prefix borrows its caller's stack; the
@@ -1285,9 +1526,15 @@ export class LlEmitter {
         }, (shapeId) => {
             const shape = this.recordsById.get(shapeId);
             return shape !== undefined && !recordDynKeepsIdentity(shape);
-        });
-        this.lentStringParameters =
-            this.debug === null ? findLentStringParameters(this.fnByName, this.callLifetimes) : new Map();
+        }, (wrap) => this.borrowableNullableWrap(wrap) === "ref");
+        if (this.debug === null) {
+            const strings = findLentStringParameters(this.fnByName, this.callLifetimes);
+            this.lendingFacts = {
+                strings,
+                records: findLentRecordFields(this.fnByName, this.callLifetimes, strings),
+                producers: findRecordFieldProducers(this.fnByName),
+            };
+        }
         this.constantCallbacks = findConstantCallbacks(mod, this.callLifetimes, this.nodeIndex);
         // Integer clones signal a stack overflow through their result, which
         // needs the single-threaded guard; worker programs also poll for
@@ -1303,10 +1550,14 @@ export class LlEmitter {
                     ? (fn, index) => this.callLifetimes.borrowed.get(fn)?.has(index) === true
                     : undefined)
                 : NO_INTEGER_CLONES;
+        // Integer clones and their dispatchers call each other through their
+        // own lowering, which keeps plain calls and flag tests.
+        for (const name of this.integerClones.members)
+            this.unwindAbi.delete(name);
         this.stackCallbacks = new StackCallbacks(this.fnByName);
-        this.receiverReaders = new AmbientReceiverReaders(this.fnByName);
+        this.receiverReaders = new AmbientReceiverReaders(this.fnByName, this.nodeIndex);
         this.stackCaptures = new StackCaptures(this.fnByName, this.callLifetimes, this.stackCallbacks);
-        this.lazyCaptures = new LazyCaptures(this.fnByName);
+        this.lazyCaptures = new LazyCaptures(this.fnByName, this.nodeIndex);
         const traced = computeTraced(mod, dynamicStores, cycleMutability);
         this.tracedShapes = traced.shapes;
         this.tracedUnions = traced.unions;
@@ -1395,12 +1646,11 @@ export class LlEmitter {
                 const meta = this.classMeta.get(t.className);
                 return meta !== undefined && programLifetimeClass(this.shapeHost, meta);
             },
-            classSubtypes: (name) => {
-                const target = this.classMetaOf(name);
-                const intervals = classMembershipIntervals(this.classMeta, target.def.name);
-                return [...this.classMeta.values()]
-                    .filter((meta) => intervals.some((range) => range.pre <= meta.pre && meta.pre <= range.post))
-                    .map((meta) => meta.def.name);
+            classSubtypes: (name) => this.classSubtypes(name),
+            classValueSubtypes: (name) => {
+                const target = this.classMetaOf(name).def;
+                return this.classSubtypes(name).filter((sub) => sub !== target.name &&
+                    classValueAbiEquals(this.classMetaOf(sub).def, target, this.fnByName));
             },
             pendingTestLines: (dest) => this.pendingTestLines(dest),
             virtualEntry: (implFn) => this.virtualEntry(implFn),
@@ -1509,6 +1759,36 @@ export class LlEmitter {
         // After the vtable slots: borrowing slots exclude their implementations.
         if (this.debug === null)
             this.analyzeBorrowedReturns();
+        this.int32SlotsValue ??= this.joinInt32Slots();
+    }
+    /** Computes the int32 slot analysis, or starts it on a forked process.
+     * Dev builds read its result only near the end of the constructor
+     * (int32Abis is empty with debug info), so for a large module the
+     * process computes it while the constructor runs the other analyses. */
+    startInt32Slots(dynamicStores) {
+        const mod = this.mod;
+        const nodes = this.nodeIndex;
+        const analyze = () => analyzeInt32Slots(mod, {
+            dynamicStores,
+            initializerBindings: this.initializerBindings,
+            nodes,
+        });
+        const fork = this.targetOptions.forkPrimitives;
+        if (fork === undefined ||
+            this.targetOptions.debugSources === undefined ||
+            (this.targetOptions.functionWorkers ?? 1) < 2 ||
+            mod.functions.length < workerMinFunctions()) {
+            this.int32SlotsValue = analyze();
+            return;
+        }
+        this.int32SlotsJoin = forkedAnalysis(fork, analyze, (slots) => slots.encode(nodes), (text) => Int32Slots.decode(text, mod, nodes));
+    }
+    joinInt32Slots() {
+        const join = this.int32SlotsJoin;
+        if (join === undefined)
+            throw new InternalCompilerError("llvm emitter bug: int32 slots read before analysis");
+        this.int32SlotsJoin = undefined;
+        return join();
     }
     /** The borrowed parameters of a hierarchy's vtable slot (empty: owned). */
     virtualSlotBorrowed(rootName, slotIndex) {
@@ -1534,7 +1814,6 @@ export class LlEmitter {
             return "ptr";
         switch (t.kind) {
             case "f64":
-            case "date":
                 return "double";
             case "bool":
                 return "i1";
@@ -1562,7 +1841,8 @@ export class LlEmitter {
         return this.emitParts().join("\n");
     }
     emitParts() {
-        return emitParts(this);
+        const parts = emitParts(this);
+        return this.unwindAbi.size > 0 ? bridgePlainUnwindCalls(parts) : parts;
     }
     /** The dispose-observer gates the emitted frees and record writers test
      * (alloc.ts): defined after every helper is emitted, because dynCheck
@@ -1638,7 +1918,6 @@ export class LlEmitter {
                 this.declare(`declare void @scr_promise_fulfill_void(ptr)`);
                 return [`  call void @scr_promise_fulfill_void(ptr ${promise})`];
             case "f64":
-            case "date":
                 this.declare(`declare void @scr_promise_fulfill_f64(ptr, double)`);
                 return [`  call void @scr_promise_fulfill_f64(ptr ${promise}, double %r)`];
             case "bool":
@@ -1691,10 +1970,26 @@ export class LlEmitter {
                 this.mangledFunctions.set(mangleBorrowedFunction(fn.name), fn.name);
             }
         }
-        for (const match of text.matchAll(/@(sc_b?f_[A-Za-z0-9_]+)/g)) {
-            const callee = this.mangledFunctions.get(match[1]);
+        // The names `@(sc_b?f_[A-Za-z0-9_]+)` matches, found with indexOf.
+        for (let at = text.indexOf("@sc_"); at >= 0;) {
+            let start = at + 4;
+            if (text.charCodeAt(start) === 98 /* b */)
+                start++;
+            if (text.charCodeAt(start) !== 102 /* f */ || text.charCodeAt(start + 1) !== 95 /* _ */) {
+                at = text.indexOf("@sc_", at + 1);
+                continue;
+            }
+            let end = start + 2;
+            while (end < text.length && isIdentifierByte(text.charCodeAt(end)))
+                end++;
+            if (end === start + 2) {
+                at = text.indexOf("@sc_", at + 1);
+                continue;
+            }
+            const callee = this.mangledFunctions.get(text.slice(at + 1, end));
             if (callee !== undefined)
                 yield callee;
+            at = text.indexOf("@sc_", end);
         }
     }
     /** Chooses the native async functions whose bodies compile to switched
@@ -1879,7 +2174,7 @@ export class LlEmitter {
             return { name: "", type };
         }
         const t = B.tmp();
-        if (type.kind === "f64" || type.kind === "date") {
+        if (type.kind === "f64") {
             this.declare(`declare double @scr_coro_direct_f64(ptr)`);
             B.line(`${t} = call double @scr_coro_direct_f64(ptr ${record})`);
         }
@@ -2627,6 +2922,15 @@ export class LlEmitter {
         else if (this.currentCoro !== null) {
             terminator = `br label %${this.currentCoro.finalLabel}`;
         }
+        else if (this.currentUnwindAbi && this.inLandingPad) {
+            // The unwind that entered the landing pad goes on to the caller's.
+            terminator = UNWIND_RESUME;
+        }
+        else if (this.currentUnwindAbi) {
+            // The caller's invoke takes the exception at its landing pad.
+            this.declare(UNWIND_RAISE_DECLARATION);
+            terminator = UNWIND_RAISE;
+        }
         else {
             const t = this.llType(this.currentReturnType);
             terminator =
@@ -2656,6 +2960,7 @@ export class LlEmitter {
             if (passThrough !== undefined &&
                 !target &&
                 this.currentCoro === null &&
+                !this.currentUnwindAbi &&
                 !this.currentInt32Result) {
                 const t = this.llType(this.currentReturnType);
                 if (t === "double" || t === "i1")
@@ -2836,6 +3141,42 @@ export class LlEmitter {
         this.emitUnwind();
         B.startBlock(B.newLabel("exc.dead"));
     }
+    /** Emits `line`, a `call` of `callee`, and reports whether the call
+     * needs no pending check after it: a call of an unwinding-ABI function
+     * becomes an `invoke` whose landing pad runs this frame's unwind (the
+     * same releases and handler as emitPendingCheck's unwind path), and the
+     * normal path continues in a fresh block. Coroutine bodies keep plain
+     * calls (unwindAbiFunctions excludes their callees). */
+    emitDirectCall(line, callee) {
+        const B = this.B;
+        if (!this.unwindAbi.has(callee) || this.currentCoro !== null || B.isTerminated()) {
+            B.line(line);
+            return false;
+        }
+        const ok = B.newLabel("inv.ok");
+        const pad = B.newLabel("inv.lp");
+        this.usesLandingPads = true;
+        // A trailing comment goes after the invoke's labels.
+        const comment = line.indexOf(" ; ");
+        const text = comment < 0 ? line : line.slice(0, comment);
+        B.terminate(`${text.replace(/^((?:%[\w.]+ = )?)call /, "$1invoke ")} to label %${ok} unwind label %${pad}`);
+        B.startBlock(pad);
+        const exception = B.tmp();
+        B.line(`${exception} = landingpad { ptr, i32 } cleanup`);
+        if (this.currentUnwindAbi)
+            B.line(`store { ptr, i32 } ${exception}, ptr ${UNWIND_SLOT}`);
+        this.inLandingPad = true;
+        this.emitUnwind();
+        this.inLandingPad = false;
+        B.startBlock(ok);
+        return true;
+    }
+    /** Whether emitUnwind runs for a landing pad, whose unwind goes on with
+     * `resume` instead of raising again. */
+    inLandingPad = false;
+    /** Whether the function being emitted has a landing pad (its definition
+     * needs the personality). */
+    usesLandingPads = false;
     /** A recursive function's entry check: below the running stack's guard,
      * throw Node's RangeError and unwind like any other pending exception. */
     emitStackCheck() {
@@ -2925,12 +3266,14 @@ export class LlEmitter {
                 return false;
         }
     }
+    /** Whether `loop` needs a cancellation poll: in a worker program, unless
+     * the loop already runs a pending test on every iteration, which observes
+     * a terminate request itself (a separate poll would only add work). */
+    workerLoopPolls(loop) {
+        return (this.mod.workers === true && !loopTestsEveryIteration(loop, (e) => this.callTestsPending(e)));
+    }
     workerLoopBudget(loop) {
-        if (!this.mod.workers)
-            return null;
-        // A loop that runs a pending test on every iteration observes a
-        // terminate request through it; a separate poll would only add work.
-        if (loopTestsEveryIteration(loop, (e) => this.callTestsPending(e)))
+        if (!this.workerLoopPolls(loop))
             return null;
         const slot = this.B.slot();
         this.B.entryAllocas.push(`${slot} = alloca i32 ; worker cancellation budget`);
@@ -3002,17 +3345,12 @@ export class LlEmitter {
                     set.add(fn.name);
                     writers.set(localId, set);
                 };
-                everyStmtList(fn.body, {
-                    stmt: (s) => {
-                        if (s.kind === "assign" || s.kind === "varDecl")
-                            note(s.localId);
-                        return true;
-                    },
-                    expr: (e) => {
-                        if (e.kind === "assignExpr" || e.kind === "incDec")
-                            note(e.localId);
-                        return true;
-                    },
+                eachNodeOfKinds(fn, this.nodeIndex.scan(GLOBAL_WRITE_KINDS), (node) => {
+                    if (node.kind === "assign" ||
+                        node.kind === "varDecl" ||
+                        node.kind === "assignExpr" ||
+                        node.kind === "incDec")
+                        note(node.localId);
                 });
             }
             this.globalWriters = writers;
@@ -3040,13 +3378,17 @@ export class LlEmitter {
         B.condBr(due, poll, next);
         B.startBlock(poll);
         B.line(`store i32 63, ptr ${slot}`);
-        // Calls and throwing operations retain their own checks. Amortize the
-        // extra cancellation poll across bounded loop batches so scalar work
-        // does not cross the runtime ABI on every iteration.
-        this.emitPendingCheck(undefined, true);
+        this.emitWorkerPoll();
         B.br(next);
         B.startBlock(next);
     }
+    /** The cancellation poll itself. Calls and throwing operations retain
+     * their own checks. Loops amortize this extra poll across bounded
+     * batches of iterations so scalar work does not cross the runtime ABI on
+     * every iteration. */
+    emitWorkerPoll() {
+        this.emitPendingCheck(undefined, true);
+    }
     /** Moves an already-evaluated value into the runtime's exception cell —
      * the `throw` statement's kind dispatch (stmts.ts's), shared with
      * every synthetic thrower. Ownership of a refcounted payload must have
@@ -3236,8 +3578,12 @@ export class LlEmitter {
     // ── union plumbing ──────────────────────────────────────────────────────
     /** A union value's tag (i32): the box's tag word, or the NULL test of a
      * nullable union's pointer. */
+    /** A box's tag and payload words are written only where the box is
+     * built (runtime constructors and untagged stack stores), so their loads
+     * carry their own alias tags: element and field stores cannot change
+     * them, and loops keep them hoisted. */
     unionTag(uName, unionId) {
-        return emitUnionTag(this.B, this.nullableUnions.get(unionId), uName);
+        return emitUnionTag(this.B, this.nullableUnions.get(unionId), uName, this.memoryAliasAttachment("union.tag"));
     }
     /** Guard for helpers that build or read a union's `%ScrUnion` box
      * directly (library-shaped unions with fixed arms): a nullable union
@@ -3249,7 +3595,7 @@ export class LlEmitter {
     /** The BORROWED payload pointer of a ref arm (scr_union_peek inlined —
      * the runtime's is a static inline); a nullable union is its payload. */
     unionPeek(uName, unionId) {
-        return emitUnionPeek(this.B, this.nullableUnions.get(unionId), uName);
+        return emitUnionPeek(this.B, this.nullableUnions.get(unionId), uName, this.memoryAliasAttachment("union.payload"));
     }
     /** Emits `switch` over a union's tag with one block per arm; each arm
      * body must TERMINATE its block (the callers branch to a join). The
@@ -3360,6 +3706,14 @@ export class LlEmitter {
         return t;
     }
     // ── class plumbing ──────────────────────────────────────────────────────
+    /** `name` and every class whose membership interval falls in its own. */
+    classSubtypes(name) {
+        const target = this.classMetaOf(name);
+        const intervals = classMembershipIntervals(this.classMeta, target.def.name);
+        return [...this.classMeta.values()]
+            .filter((meta) => intervals.some((range) => range.pre <= meta.pre && meta.pre <= range.post))
+            .map((meta) => meta.def.name);
+    }
     classMetaOf(className) {
         const meta = this.classMeta.get(className);
         if (!meta)
@@ -3444,7 +3798,12 @@ export class LlEmitter {
     classObjSym(className) {
         if (!this.classObjs.has(className)) {
             const meta = this.classMetaOf(className);
-            this.classObjs.set(className, { nameSym: this.internLiteral(classJsName(meta.def)) });
+            // util.inspect prints `[class C extends B]` with the base's name.
+            const baseName = meta.base ? classJsName(meta.base.def) : "";
+            this.classObjs.set(className, {
+                nameSym: this.internLiteral(classJsName(meta.def)),
+                baseNameSym: baseName ? this.internLiteral(baseName) : null,
+            });
         }
         this.journal?.entry("classObj", className, this.classObjs.get(className));
         return mangleClassObj(className);
@@ -3656,8 +4015,17 @@ export class LlEmitter {
             workers: this.mod.workers === true,
             owned: this.bytesOwned,
             scalarStrings: new Set(this.stringSlices.keys()),
+            unitSafeFunctions: this.unitSafeFunctions(),
         }, paired);
     }
+    unitSafe;
+    /** The functions a loop over a string's UTF-16 units may call, or null
+     * where emitted reads do not use the runtime's scan receiver. */
+    unitSafeFunctions() {
+        if (this.unitSafe === undefined)
+            this.unitSafe = stringScanInline(this) ? unitSafeFunctions(this.mod) : null;
+        return this.unitSafe;
+    }
     /** The alias tag of a whole-slot load of a binding (module globals only). */
     slotAlias(id) {
         const binding = this.binding(id);
@@ -4096,6 +4464,7 @@ export class LlEmitter {
     chunkResult() {
         return {
             entries: this.journal.entries,
+            debugRequests: this.debug?.recorded() ?? [],
             flags: {
                 usesCoroutines: this.usesCoroutines,
                 usesTimers: this.usesTimers,
@@ -4130,6 +4499,8 @@ export class LlEmitter {
         // emission left them (a helper set guards the declarations its first
         // request adds).
         replayJournal(this, chunk.entries, names, tags);
+        const debugIds = new Map();
+        this.debug?.replay(chunk.debugRequests, (name) => this.functionNamed(name), debugIds);
         const flags = chunk.flags;
         this.usesCoroutines ||= flags.usesCoroutines;
         this.usesTimers ||= flags.usesTimers;
@@ -4160,7 +4531,13 @@ export class LlEmitter {
             this.directAwaitCallees.add(name);
         for (const name of chunk.inlineAwaitCallees)
             this.inlineAwaitCallees.add(name);
-        return renameChunk(names, tags);
+        return renameChunk(names, tags, debugIds);
+    }
+    functionNamed(name) {
+        const fn = this.fnByName.get(name);
+        if (fn === undefined)
+            throw new InternalCompilerError(`llvm emitter bug: no function ${name}`);
+        return fn;
     }
     /** Adds a content-named constant a chunk interned. Its symbol is this
      * emitter's own: a digest collision retries in first-use order, which a
@@ -4253,7 +4630,9 @@ export class LlEmitter {
     /** An emitter for a range of this module's functions: the same facts,
      * empty module-level tables, and a journal of what the range adds. */
     chunkEmitter() {
-        return new LlEmitter(this.mod, { ...this.targetOptions, functionChunks: 1 }, new EmitJournal());
+        const chunk = new LlEmitter(this.mod, { ...this.targetOptions, functionChunks: 1 }, new EmitJournal());
+        chunk.debug?.record();
+        return chunk;
     }
     /** The double-ABI entry of a function with an i32 entry: every caller
      * that is not a direct call (closures, dispatch, adapters, the runtime)
@@ -4416,7 +4795,23 @@ export class LlEmitter {
             params.unshift("ptr %sc_env");
         const ret = this.llType(fn.returnType);
         const call = `call ${ret} @${mangleBorrowedFunction(fn.name)}(${params.join(", ")})`;
-        B.line(ret === "void" ? call : `%result = ${call}`);
+        // The owned entry of an unwinding-ABI body releases the borrowed
+        // parameters on the unwind too, then resumes it to its caller.
+        const unwinding = this.unwindAbi.has(fn.name) && kept.size === 0;
+        if (unwinding) {
+            const ok = B.newLabel("adapt.ok");
+            const pad = B.newLabel("adapt.lp");
+            B.terminate(`${ret === "void" ? "" : "%result = "}invoke ${call.slice(5)} to label %${ok} unwind label %${pad}`);
+            B.startBlock(pad);
+            const exception = B.tmp();
+            B.line(`${exception} = landingpad { ptr, i32 } cleanup`);
+            for (const index of borrowed)
+                this.releaseFrame(`%p${index}`, fn.params[index].type);
+            B.terminate(`resume { ptr, i32 } ${exception}`);
+            B.startBlock(ok);
+        }
+        else
+            B.line(ret === "void" ? call : `%result = ${call}`);
         // A borrowed result is reachable from the parameters: own it first. A
         // throwing body returns a null dummy, which needs no owner (and string
         // or array retains do not accept NULL).
@@ -4442,7 +4837,10 @@ export class LlEmitter {
                 this.releaseFrame(`%p${index}`, fn.params[index].type);
         B.terminate(ret === "void" ? "ret void" : `ret ${ret} ${result}`);
         const symbol = kept.size > 0 ? `${mangleBorrowedFunction(fn.name)}.virtual` : mangleFunction(fn.name);
-        return `define internal ${ret} @${symbol}(${params.join(", ")}) ${FN_ATTRS} {\n${B.render()}\n}`;
+        if (unwinding)
+            this.declare(`declare i32 @__gcc_personality_v0(...)`);
+        const personality = unwinding ? ` ${UNWIND_PERSONALITY}` : "";
+        return `define internal ${ret} @${symbol}(${params.join(", ")}) ${FN_ATTRS}${personality} {\n${B.render()}\n}`;
     }
     canBorrowCallArgument(value) {
         return canBorrowCallArgument(this, value);
@@ -4617,12 +5015,20 @@ export class LlEmitter {
     readsLoopMetrics() {
         if (this.loopMetricsRead !== undefined)
             return this.loopMetricsRead;
-        const visitor = {
-            expr: (e) => !(e.kind === "libCall" && (e.fn === "perf.loopStart" || e.fn === "perf.idleTime")),
-            stmt: () => true,
-        };
-        this.loopMetricsRead = !this.mod.functions.every((fn) => everyStmtList(fn.body, visitor));
-        return this.loopMetricsRead;
+        let found = false;
+        const scan = this.nodeIndex.scan(LIB_CALL_KINDS);
+        for (const fn of this.mod.functions) {
+            eachNodeOfKinds(fn, scan, (node) => {
+                if (node.kind === "libCall" &&
+                    (node.fn === "perf.loopStart" || node.fn === "perf.idleTime"))
+                    found = true;
+                return !found;
+            });
+            if (found)
+                break;
+        }
+        this.loopMetricsRead = found;
+        return found;
     }
     lastUseMovesAllowed() {
         return (this.debug === null &&
@@ -4806,6 +5212,9 @@ export class LlEmitter {
                 if (cursor)
                     cursor.index = i;
                 this.stmtCursor = cursor;
+                const reserve = reservedArrayLength(stmts, i);
+                if (reserve !== null)
+                    this.reservedArrayLengths.set(reserve.literal, reserve.hint);
                 const deferred = this.deferredPromiseAllLocal(stmts, i);
                 if (deferred !== null) {
                     this.deferredPromiseAllEntries.set(stmts[i].localId, deferred);
@@ -4838,6 +5247,24 @@ export class LlEmitter {
         if (resume && !resume.placed && !this.B.isTerminated()) {
             // The fast copy's failed checks re-execute this statement here.
             resume.placed = true;
+            // A write with hoisted leaves resumed from its own failed check
+            // re-executes its arithmetic over the leaves' stored doubles: each
+            // leaf ran once, in the fast copy. Reached from an earlier statement
+            // of this copy (a failed check before it), its leaves never ran, so
+            // it runs as written.
+            const rewritten = this.speculationResumeValues?.get(s);
+            if (rewritten !== undefined && rewritten.leaves.length > 0) {
+                const join = this.B.newLabel("spec.resume.join");
+                this.emitStmt(s);
+                if (!this.B.isTerminated())
+                    this.B.br(join);
+                this.B.startBlock(resume.label);
+                this.emitStmt(withSpeculativeValue(s, rewritten.value));
+                if (!this.B.isTerminated())
+                    this.B.br(join);
+                this.B.startBlock(join);
+                return;
+            }
             this.B.br(resume.label);
             this.B.startBlock(resume.label);
         }
@@ -4966,6 +5393,10 @@ export class LlEmitter {
             });
             return;
         }
+        if (e.kind === "dynInvoke" && e.calleeNameValue === undefined) {
+            emitDynamicExpr(this, e, true);
+            return;
+        }
         this.emitExpr(e);
     }
     emitControlExpr(e) {
@@ -5955,8 +6386,8 @@ export class LlEmitter {
     emitBytesLength(elem, receiver, bytes) {
         return emitBytesLength(this, elem, receiver, bytes);
     }
-    emitBytesGet(elem, receiver, index, expr, inBounds = false, invalidNaN = false, owned = false) {
-        return emitBytesGet(this, elem, receiver, index, expr, inBounds, invalidNaN, owned);
+    emitBytesGet(elem, receiver, index, expr, inBounds = false, invalidNaN = false, owned = false, scope = "") {
+        return emitBytesGet(this, elem, receiver, index, expr, inBounds, invalidNaN, owned, scope);
     }
     emitToUint32(value, expr, uint32) {
         return emitToUint32(this, value, expr, uint32);
@@ -6011,8 +6442,8 @@ export class LlEmitter {
         const integer = this.emitToUint32(value.name, expr, value.uint32);
         this.B.line(`store i32 ${integer}, ptr ${slot}`);
     }
-    emitBytesSet(elem, receiver, index, value, indexExpr, expr, inBounds = false, owned = false) {
-        return emitBytesSet(this, elem, receiver, index, value, indexExpr, expr, inBounds, owned);
+    emitBytesSet(elem, receiver, index, value, indexExpr, expr, inBounds = false, owned = false, scope = "") {
+        return emitBytesSet(this, elem, receiver, index, value, indexExpr, expr, inBounds, owned, scope);
     }
     emitBytesIntrinsic(e) {
         return emitBytesIntrinsic(this, e);