@scriptc/compiler 0.3.6 → 0.4.1

package/dist/backend/llvm/emit-function.js

@@ -0,0 +1,1193 @@
+/* LlEmitter's function emission: function bodies, generator and async scaffolding, async awaits, WASI fulfillment and trampolines. */
+import { SUSPENDS, UNOPTIMIZED_BOX_SLOTS, boxSlotCount, } from "./emitter.js";
+import { isRefCounted, typeKey } from "../../ir/ir.js";
+import { BlockBuilder } from "./blocks.js";
+import { findPrivateSplitLocals } from "./split-loops.js";
+import { conditionalLoopWrites, initializerLocalCandidates, withInitializerBindings, withInitializerLocals, } from "../../ir/initializer-bindings.js";
+import { findUnaliasedLocalArrays } from "../../ir/array-length-stability.js";
+import { constructorClass } from "../../ir/final-fields.js";
+import { findInitializedByteLoopBindings } from "./byte-windows.js";
+import { findLoopArrayBorrows } from "./loop-array-borrows.js";
+import { findCallArrayReads, findLocalArrayReads } from "./local-array-reads.js";
+import { findScalarStringSlices, findScratchStringSlices } from "./string-slices.js";
+import { findMapReadLifetimes, findReadNumberMapWrites } from "./map-read-lifetimes.js";
+import { findMapEntryFusions } from "./map-entry-fusion.js";
+import { findLocalStackUnions } from "./stack-unions.js";
+import { findLocalStackObjects, findReassignedStackObjects } from "./stack-objects.js";
+import { findLocalUnionStorage } from "./local-union-storage.js";
+import { INT32_RANGE, analyzeIntegerRangeFacts } from "../../ir/integer-ranges.js";
+import { findBytesBounds } from "./bytes-bounds.js";
+import { findInt32Locals } from "./int32-locals.js";
+import { findIntegerViews } from "./integer-views.js";
+import { mangleArgPack, mangleAsyncAdoptCall, mangleAsyncAwaitCall, mangleAsyncSpawn, mangleBorrowedFunction, mangleClassStruct, mangleFunction, mangleGenDrop, mangleGenSourceClose, mangleGenSpawn, mangleGlobal, mangleLocal, mangleTrampoline, } from "../mangle.js";
+import { InternalCompilerError } from "../../errors.js";
+import { classEnvironmentIndex } from "./classes.js";
+import { FN_ATTRS, boxNewCall, releaseSym, traceArg, vAdapters } from "./shapes.js";
+import { INT32_ABI_SUFFIX, SCALAR_UNION_RESULT } from "./int32-abi.js";
+import { f64Lit } from "./common.js";
+import { RESULT_ENTRY_PREFIX } from "./record-results.js";
+import { emitIntegerClone, emitIntegerDispatcher } from "./integer-clones.js";
+export function emitFunction(emitter, fn) {
+    const B = new BlockBuilder();
+    emitter.B = B;
+    emitter.currentFunctionName = fn.name;
+    emitter.currentTier = fn.tier;
+    // Fiber switches restore the active exception cell before control
+    // returns to a synchronous frame (emitPendingCheck).
+    B.excCellInvariant = fn.async !== true && fn.generator === undefined;
+    emitter.debugScope = emitter.debug?.function(fn) ?? null;
+    B.debugLocation = emitter.debug?.location(fn.loc, emitter.debugScope) ?? null;
+    emitter.frames = [];
+    emitter.scopes = [];
+    emitter.unwindCleanups.clear();
+    emitter.jumpTargets = [];
+    emitter.currentLocals = new Map(fn.locals.map((l) => [l.id, l]));
+    emitter.lazyCaptureBoxes.clear();
+    emitter.lazyStackBoxes.clear();
+    // An unchanged captured parameter keeps its value in a plain slot; the
+    // binding is boxed only for closures (captureBox).
+    const lazyParameters = emitter.lazyCaptures.parameters(fn);
+    for (const id of lazyParameters) {
+        const { boxed: _boxed, ...plain } = emitter.currentLocals.get(id);
+        emitter.currentLocals.set(id, plain);
+        emitter.lazyCaptureBoxes.set(id, "");
+    }
+    emitter.currentConstantCallbacks =
+        emitter.debug === null ? (emitter.constantCallbacks.get(fn.name) ?? new Map()) : new Map();
+    // Preserve concrete source bindings for debugger inspection. Suspended
+    // functions keep the established array lifetime across continuations.
+    emitter.streamingSplitsEnabled = emitter.debug === null && !fn.async && !fn.generator;
+    emitter.stackCallbacks.reset(emitter.streamingSplitsEnabled);
+    emitter.stackCaptures.reset(fn, emitter.streamingSplitsEnabled, lazyParameters);
+    emitter.privateSplitLocals = emitter.streamingSplitsEnabled
+        ? findPrivateSplitLocals(fn)
+        : new Map();
+    emitter.storedSplits.clear();
+    emitter.splitSpans.clear();
+    emitter.stringSlices.clear();
+    const initializerBindings = emitter.initializerBindings.get(fn.name) ?? [];
+    const numericFn = withInitializerBindings(fn, initializerBindings);
+    emitter.numericLocals = new Map(numericFn.locals.map((l) => [l.id, l]));
+    emitter.unaliasedArrays = findUnaliasedLocalArrays(fn);
+    emitter.borrowedParameters.clear();
+    emitter.referenceWrites?.enter(fn);
+    emitter.reboundOwnerSlots.clear();
+    emitter.projectedParameters.clear();
+    const projectedParameterIndexes = emitter.callLifetimes.parameters.get(fn.name);
+    if (projectedParameterIndexes)
+        for (const index of projectedParameterIndexes)
+            emitter.projectedParameters.add(fn.params[index].localId);
+    emitter.stableCallBindings = emitter.callLifetimes.bindings.get(fn.name) ?? new Set();
+    const borrowedParameterIndexes = emitter.callLifetimes.borrowed.get(fn.name);
+    if (borrowedParameterIndexes) {
+        // Rebound walk parameters borrow too, but only unchanged ones may
+        // serve as stable owners for call arguments and aliases.
+        for (const index of borrowedParameterIndexes)
+            if (!emitter.walkBorrowsByFunction.get(fn.name)?.has(fn.params[index].localId) &&
+                !emitter.reboundByFunction.get(fn.name)?.has(fn.params[index].localId))
+                emitter.borrowedParameters.add(fn.params[index].localId);
+    }
+    // A generator frame owns its unchanged parameters across suspensions.
+    const generatorParameters = emitter.callLifetimes.generatorParameters.get(fn.name);
+    if (generatorParameters !== undefined)
+        for (const id of generatorParameters)
+            emitter.borrowedParameters.add(id);
+    emitter.captureIds = new Set([...(fn.captures ?? []), ...(fn.classCaptures ?? [])].map((c) => c.localId));
+    emitter.stableCaptureIds =
+        fn.captures !== undefined ? emitter.lazyCaptures.stableCaptures(fn.name) : new Set();
+    emitter.integerLoopBindings.clear();
+    emitter.countedLoopsEnabled = emitter.debug === null && !fn.async && !fn.generator;
+    emitter.speculativeLoopsEnabled = emitter.countedLoopsEnabled;
+    emitter.speculativeFunction = fn.name;
+    emitter.speculation = null;
+    emitter.speculationResume = null;
+    emitter.finalFieldBorrows =
+        !fn.async &&
+            !fn.generator &&
+            constructorClass(fn.name) === null &&
+            emitter.finalFields.size > 0;
+    emitter.byteWindowsEnabled = emitter.countedLoopsEnabled;
+    emitter.bytesLoopVersionsEnabled = emitter.countedLoopsEnabled;
+    emitter.bytesLoopEntries = null;
+    emitter.bytesLoopFunction = fn;
+    emitter.denseLoopsEnabled = emitter.countedLoopsEnabled;
+    emitter.denseLoop = null;
+    emitter.denseLoopEntries =
+        emitter.denseLoopsEnabled && numericFn.locals.some((l) => l.type.kind === "array")
+            ? findInitializedByteLoopBindings(fn)
+            : new Map();
+    emitter.byteWindowEntries =
+        emitter.byteWindowsEnabled && numericFn.locals.some((l) => l.type.kind === "bytes")
+            ? findInitializedByteLoopBindings(fn)
+            : new Map();
+    emitter.integerViews.clear();
+    emitter.fieldPointerTags.clear();
+    emitter.integerArrayBindings.clear();
+    emitter.loopArrayBorrows =
+        emitter.debug === null
+            ? findLoopArrayBorrows(fn, emitter.callLifetimes, emitter.referenceEffects, emitter.optionalArrayReads, emitter.referenceWrites, emitter.fnByName)
+            : new Set();
+    emitter.localArrayReads = findLocalArrayReads(fn, emitter.fnByName, emitter.unionsById, emitter.referenceEffects.functions, emitter.callLifetimes, emitter.optionalArrayReads, emitter.loopArrayBorrows, emitter.debug === null ? emitter.referenceEffects : undefined);
+    emitter.callArrayReads = findCallArrayReads(fn, emitter.optionalArrayReads, emitter.referenceEffects.functions, emitter.callLifetimes, emitter.debug === null ? emitter.referenceEffects : undefined);
+    emitter.scalarStringSlices =
+        emitter.debug === null ? findScalarStringSlices(fn, emitter.callLifetimes) : new Map();
+    emitter.scratchStringSlices =
+        emitter.debug === null
+            ? findScratchStringSlices(fn, emitter.callLifetimes, emitter.scalarStringSlices, emitter.lentStringParameters)
+            : new Set();
+    emitter.mapReadLifetimes = findMapReadLifetimes(fn, emitter.boxedUnionsById, emitter.callLifetimes);
+    emitter.readNumberMapWrites = findReadNumberMapWrites(fn);
+    emitter.mapEntryFusion = emitter.debug === null ? findMapEntryFusions(fn) : null;
+    emitter.localStackUnions = findLocalStackUnions(fn, emitter.callLifetimes, emitter.boxedUnionsById);
+    emitter.localStackObjects = emitter.stackObjects
+        ? findLocalStackObjects(emitter, fn, emitter.callLifetimes)
+        : new Map();
+    emitter.reassignedStackObjects = emitter.stackObjects
+        ? findReassignedStackObjects(emitter, fn, emitter.callLifetimes)
+        : new Set();
+    emitter.stackObjectSlots = new Map();
+    emitter.stackObjectFieldPtrs = new Map();
+    emitter.localUnionStorageProofs = findLocalUnionStorage(fn, emitter.callLifetimes, emitter.unionSources);
+    emitter.localUnionStorage = new Map();
+    emitter.walkBorrows = emitter.walkBorrowsByFunction.get(fn.name) ?? new Set();
+    const slotFacts = emitter.int32Slots.facts(fn.name);
+    const rangeFacts = analyzeIntegerRangeFacts(numericFn, slotFacts);
+    emitter.integerRanges = rangeFacts.ranges;
+    emitter.integerLoopRanges = rangeFacts.loopRanges;
+    emitter.int32Elements = slotFacts.elements ?? null;
+    emitter.bytesBounds = findBytesBounds(numericFn, emitter.integerRanges);
+    // Debug builds keep every number local a double for the debugger's
+    // variable view. Lazily boxed captured parameters keep their double,
+    // which the box is created from, and so do plain copies of a capture or
+    // such a parameter (copyStableCaptures): the source already holds the
+    // double, which an i32 copy would convert and widen again on every call.
+    // Initializer bindings proven int32 move into an i32 local of their
+    // initializer: no other code observes the global, so the loop-carried
+    // value never round-trips through the double slot.
+    const initializerLocals = initializerLocalCandidates(initializerBindings);
+    const int32Locals = emitter.debug === null
+        ? findInt32Locals(withInitializerLocals(fn, initializerLocals), emitter.integerRanges, emitter.int32Slots.facts(fn.name), new Set([...emitter.lazyCaptureBoxes.keys(), ...emitter.doubleCopies(fn)]))
+        : null;
+    emitter.int32Locals = int32Locals?.locals ?? new Map();
+    emitter.int32LocalWrites = int32Locals?.values ?? new Set();
+    const localizedGlobals = new Set();
+    for (const global of initializerLocals) {
+        if (!emitter.int32Locals.has(global.id))
+            continue;
+        localizedGlobals.add(global.id);
+        emitter.currentLocals.set(global.id, {
+            id: global.id,
+            name: global.name,
+            type: global.type,
+            mutable: global.mutable,
+        });
+    }
+    emitter.chainSlots.clear();
+    emitter.finallyStack = [];
+    emitter.tryStack = [];
+    emitter.loopDepth = 0;
+    emitter.currentReturnType = fn.returnType;
+    const int32Abi = emitter.int32Abis.get(fn.name);
+    emitter.currentInt32Result = int32Abi?.result === true;
+    emitter.currentUnionResult = int32Abi?.union === true;
+    emitter.currentGenerator = fn.generator ?? null;
+    emitter.currentAsyncGenerator = fn.async === true && fn.generator !== undefined;
+    emitter.currentBorrowedReturn = emitter.borrowedReturns.has(fn.name);
+    emitter.lastUseMoves.clear();
+    emitter.varRefCounts = null;
+    emitter.currentFunctionBody = fn.body;
+    emitter.inConstructor = emitter.constructorFunctions.has(fn.name);
+    emitter.currentCoro = null;
+    emitter.logArgSlots = 0;
+    if (emitter.coroLowered(fn)) {
+        emitter.usesCoroutines = true;
+        emitter.declare(`declare token @llvm.coro.id(i32, ptr, ptr, ptr)`);
+        emitter.declare(`declare ${emitter.sizeType} @llvm.coro.size.${emitter.sizeType}()`);
+        emitter.declare(`declare ptr @llvm.coro.begin(token, ptr)`);
+        emitter.declare(`declare token @llvm.coro.save(ptr)`);
+        emitter.declare(`declare i8 @llvm.coro.suspend(token, i1)`);
+        emitter.declare(`declare ptr @llvm.coro.free(token, ptr)`);
+        emitter.declare(`declare void @llvm.coro.end(ptr, i1, token)`);
+        // Native frames come from the runtime's small-block allocator.
+        const frameAlloc = emitter.wasi ? "malloc" : "scr_rt_calloc";
+        emitter.declare(`declare ptr @${frameAlloc}(${emitter.sizeType})`);
+        emitter.declare(`declare void @${emitter.wasi ? "free" : "scr_rt_free"}(ptr)`);
+        const started = emitter.wasi ? "scr_wasi_coro_started" : "scr_coro_started";
+        emitter.declare(`declare void @${started}(ptr)`);
+        emitter.declare(`declare ptr @scr_fiber_self()`);
+        const id = B.tmp();
+        const size = B.tmp();
+        const mem = B.tmp();
+        const handle = B.tmp();
+        const self = B.tmp();
+        B.line(`${id} = call token @llvm.coro.id(i32 0, ptr null, ptr null, ptr null)`);
+        B.line(`${size} = call ${emitter.sizeType} @llvm.coro.size.${emitter.sizeType}()`);
+        B.line(`${mem} = call ptr @${frameAlloc}(${emitter.sizeType} ${size})`);
+        B.line(`${handle} = call ptr @llvm.coro.begin(token ${id}, ptr ${mem})`);
+        B.line(`call void @${started}(ptr ${handle})`);
+        B.line(`${self} = call ptr @scr_fiber_self()`);
+        // A native synchronous body owns its generator for the whole run (the
+        // ScrGen owns the frame), so its OUT slot address is fixed.
+        let out = null;
+        const yieldKind = fn.generator?.yieldT.kind;
+        if (!emitter.wasi &&
+            fn.async !== true &&
+            (yieldKind === "f64" || yieldKind === "date" || yieldKind === "bool")) {
+            emitter.declare(`declare ptr @scr_gen_of_fiber(ptr)`);
+            const gen = B.tmp();
+            B.line(`${gen} = call ptr @scr_gen_of_fiber(ptr ${self})`);
+            out = emitter.genOutSlot(gen);
+        }
+        emitter.currentCoro = {
+            kind: fn.generator !== undefined ? "generator" : "async",
+            id,
+            handle,
+            self,
+            finalLabel: B.newLabel("coro.final"),
+            cleanupLabel: B.newLabel("coro.cleanup"),
+            suspendLabel: B.newLabel("coro.suspend"),
+            out,
+        };
+    }
+    const paramIds = new Set(fn.params.map((p) => p.localId));
+    if (emitter.debug === null) {
+        for (const id of findIntegerViews(numericFn)) {
+            // An i32 slot is its own view.
+            if (emitter.int32Locals.has(id))
+                continue;
+            const slot = B.slot();
+            B.entryAllocas.push(`${slot} = alloca i32 ; integer view ${emitter.numericLocals.get(id).name}`);
+            emitter.integerViews.set(id, slot);
+        }
+    }
+    for (const global of initializerBindings) {
+        if (!emitter.integerViews.has(global.id))
+            continue;
+        const value = B.tmp();
+        B.line(`${value} = load double, ptr @${mangleGlobal(global.id)}`);
+        emitter.storeIntegerView(global.id, { name: value, type: global.type });
+    }
+    // A number global only its guarded, synchronous initializer reads or
+    // writes (findInitializerBindings) is invisible to every other function,
+    // and reentry returns before reaching it, so a local slot can stand in
+    // for it: LLVM keeps it in a register across calls and loops that would
+    // otherwise force a store and reload per iteration. The global's
+    // storage is never read again, so nothing writes the slot back. Bindings
+    // proven int32 already live in an i32 local (localizedGlobals above);
+    // this covers the remaining doubles.
+    emitter.privateGlobalSlots.clear();
+    if (emitter.debug === null && emitter.currentCoro === null && !fn.async && !fn.generator) {
+        const conditional = initializerLocals.length ? conditionalLoopWrites(fn.body) : null;
+        for (const global of initializerLocals) {
+            if (localizedGlobals.has(global.id))
+                continue;
+            // A conditional update in a loop stays a memory update (see
+            // conditionalLoopWrites).
+            if (conditional?.has(global.id))
+                continue;
+            const slot = B.slot();
+            const value = B.tmp();
+            B.entryAllocas.push(`${slot} = alloca double ; module binding ${global.name}`);
+            B.line(`${value} = load double, ptr @${mangleGlobal(global.id)}`);
+            B.line(`store double ${value}, ptr ${slot}`);
+            emitter.privateGlobalSlots.set(global.id, slot);
+        }
+    }
+    for (const local of emitter.currentLocals.values()) {
+        // Boxed locals' slots hold their capture BOX (a ptr); captured
+        // (env-borrowed) locals bind the incoming box below. A caught-typed
+        // local is a catch binding: its slot holds the ScrCaught snapshot
+        // box the catch prologue takes (scr_exc_take).
+        const slotTy = local.boxed || emitter.captureIds.has(local.id) || local.type.kind === "caught"
+            ? "ptr"
+            : emitter.int32Locals.has(local.id)
+                ? "i32"
+                : emitter.llType(local.type);
+        B.entryAllocas.push(`%${mangleLocal(local.id)} = alloca ${slotTy} ; ${local.name}`);
+        emitter.emitDebugLocal(local, `%${mangleLocal(local.id)}`, fn.params.findIndex((p) => p.localId === local.id) + 1, emitter.captureIds.has(local.id));
+        // A localized initializer binding starts at its global's zero value.
+        if (localizedGlobals.has(local.id))
+            B.line(`store i32 0, ptr %${mangleLocal(local.id)}`);
+        // Refcounted/boxed locals start NULL (the C prologue's `= NULL`):
+        // scope-exit releases run whether or not an assign ever did.
+        if (paramIds.has(local.id) || emitter.captureIds.has(local.id))
+            continue;
+        if (local.boxed || isRefCounted(local.type)) {
+            B.line(`store ptr null, ptr %${mangleLocal(local.id)}`);
+        }
+    }
+    // A body holding thousands of capture boxes compiles unoptimized, its
+    // box pointers kept in their slots (see boxSlotCount).
+    const unoptimized = emitter.currentCoro === null && boxSlotCount(fn) >= UNOPTIMIZED_BOX_SLOTS;
+    if (unoptimized) {
+        const slots = new Set();
+        for (const local of emitter.currentLocals.values())
+            if (local.boxed || emitter.captureIds.has(local.id))
+                slots.add(`%${mangleLocal(local.id)}`);
+        B.volatileSlots = slots;
+        // Without optimization a versioned copy only adds code.
+        emitter.bytesLoopVersionsEnabled = false;
+    }
+    // Captured bindings come in through the environment — borrowed for the
+    // whole call (the closure owns them): bound here, never released here.
+    (fn.captures ?? []).forEach((c, i) => {
+        const caps = B.tmp();
+        const p = B.tmp();
+        const box = B.tmp();
+        B.line(`${caps} = getelementptr inbounds %ScrClosure, ptr %sc_env, i64 1 ; caps`);
+        B.line(`${p} = getelementptr inbounds ptr, ptr ${caps}, ${emitter.sizeType} ${i} ; caps[${i}]`);
+        B.line(`${box} = load ptr, ptr ${p}`);
+        B.line(`store ptr ${box}, ptr %${mangleLocal(c.localId)} ; captured ${c.name}`);
+    });
+    (fn.classCaptures ?? []).forEach((c) => {
+        const self = fn.params[0];
+        if (self.type.kind !== "object")
+            throw new InternalCompilerError("class captures require an instance receiver");
+        const classSlot = B.tmp();
+        const classValue = B.tmp();
+        const caps = B.tmp();
+        const slot = B.tmp();
+        const box = B.tmp();
+        B.line(`${classSlot} = getelementptr inbounds %${mangleClassStruct(self.type.className)}, ptr %p_${mangleLocal(self.localId)}, i64 0, i32 ${classEnvironmentIndex(emitter.classMeta.get(self.type.className))}`);
+        B.line(`${classValue} = load ptr, ptr ${classSlot}`);
+        B.line(`${caps} = getelementptr inbounds %ScrClassObj, ptr ${classValue}, i64 1`);
+        B.line(`${slot} = getelementptr inbounds ptr, ptr ${caps}, ${emitter.sizeType} ${c.slot}`);
+        B.line(`${box} = load ptr, ptr ${slot}`);
+        B.line(`store ptr ${box}, ptr %${mangleLocal(c.localId)} ; class capture ${c.name}`);
+    });
+    // Params spill into their slots; the function scope owns refcounted
+    // params (callees own their params — callers passed +1). Boxed params
+    // allocate the shared binding and move the raw value in.
+    const fnScope = [];
+    for (const id of emitter.lazyCaptureBoxes.keys()) {
+        const cache = B.slot();
+        B.entryAllocas.push(`${cache} = alloca ptr ; lazy box ${emitter.currentLocals.get(id).name}`);
+        B.line(`store ptr null, ptr ${cache}`);
+        fnScope.push({
+            slot: cache,
+            type: emitter.currentLocals.get(id).type,
+            boxed: true,
+            lazy: true,
+        });
+        emitter.lazyCaptureBoxes.set(id, cache);
+        // Environments that finish during the call can share an immortal
+        // frame box. Its payload owner is released with the function scope.
+        const stack = emitter.stackCaptures.emit(emitter, id, emitter.currentLocals.get(id).type);
+        if (stack) {
+            emitter.lazyStackBoxes.set(id, stack.box);
+            if (stack.owner)
+                fnScope.push(stack.owner);
+        }
+    }
+    const borrowed = emitter.callLifetimes.borrowed.get(fn.name);
+    for (const [index, p] of fn.params.entries()) {
+        const local = emitter.currentLocals.get(p.localId);
+        const slot = `%${mangleLocal(p.localId)}`;
+        if (local.boxed) {
+            const box = B.tmp();
+            B.line(`${box} = ${boxNewCall(emitter.shapeHost, p.type)} ; ${p.name} (boxed param)`);
+            emitter.boxSet(box, p.type, `%p_${mangleLocal(p.localId)}`);
+            B.line(`store ptr ${box}, ptr ${slot}`);
+            fnScope.push({ slot, type: p.type, boxed: true });
+            continue;
+        }
+        const incoming = { name: `%p_${mangleLocal(p.localId)}`, type: p.type };
+        if (int32Abi?.params[index]) {
+            const bits = `%p_${mangleLocal(p.localId)}${INT32_ABI_SUFFIX}`;
+            incoming.uint32 = bits;
+            incoming.integer = { name: bits, type: "i32", signed: true, range: INT32_RANGE };
+        }
+        if (emitter.storeInt32Local(p.localId, incoming))
+            continue;
+        B.line(`store ${emitter.llType(p.type)} %p_${mangleLocal(p.localId)}, ptr ${slot}`);
+        emitter.storeIntegerView(p.localId, incoming);
+        if (isRefCounted(p.type) && !borrowed?.has(index)) {
+            fnScope.push({ slot, type: p.type });
+            emitter.noteParameterMove(fn, p.localId, slot);
+        }
+        if (emitter.reboundByFunction.get(fn.name)?.has(p.localId) && borrowed?.has(index)) {
+            // The caller's borrow stays in the parameter slot; values the body
+            // assigns are owned here until rebinding or exit releases them.
+            const owner = `${slot}.owner`;
+            B.entryAllocas.push(`${owner} = alloca ptr`);
+            B.line(`store ptr null, ptr ${owner}`);
+            fnScope.push({ slot: owner, type: p.type });
+            emitter.reboundOwnerSlots.set(p.localId, owner);
+        }
+    }
+    emitter.scopes.push(fnScope);
+    // Stackful native fibers and native coroutine frames retain these frames
+    // while suspended. The active exception context owns the chain, so
+    // concurrent fibers isolate it.
+    if (emitter.stackTraces &&
+        !emitter.wasi &&
+        fn.sourceName !== undefined &&
+        !emitter.unframedFunctions.has(fn.name)) {
+        B.entryAllocas.push(`%source_frame = alloca { ptr, ptr }`);
+        const frame = emitter.cstr(`    at ${fn.sourceName} (${fn.loc.file})`);
+        if (B.excCellInvariant &&
+            emitter.mod.lib === undefined &&
+            emitter.mod.workers !== true &&
+            emitter.sizeType === "i64") {
+            // Executables link the active-cell pointer as a plain global, and a
+            // synchronous body sees one cell from entry to every return: push
+            // and pop the frame inline (ScrExcCell.stack is at byte 56 on
+            // 64-bit targets) instead of calling the runtime twice per call.
+            emitter.declare(`@scr_exc_active = external global ptr`);
+            B.line(`%source_frame.cell = load ptr, ptr @scr_exc_active`);
+            B.line(`%source_frame.chain = getelementptr inbounds i8, ptr %source_frame.cell, i64 56`);
+            B.line(`%source_frame.previous = load ptr, ptr %source_frame.chain`);
+            B.line(`store ptr %source_frame.previous, ptr %source_frame`);
+            B.line(`%source_frame.text = getelementptr inbounds i8, ptr %source_frame, i64 8`);
+            B.line(`store ptr ${frame}, ptr %source_frame.text`);
+            B.line(`store ptr %source_frame, ptr %source_frame.chain`);
+            B.returnEpilogue = `store ptr %source_frame.previous, ptr %source_frame.chain`;
+        }
+        else {
+            emitter.declare(`declare void @scr_stack_enter(ptr, ptr)`);
+            emitter.declare(`declare void @scr_stack_leave(ptr)`);
+            B.line(`call void @scr_stack_enter(ptr %source_frame, ptr ${frame})`);
+            B.returnEpilogue = `call void @scr_stack_leave(ptr %source_frame)`;
+        }
+    }
+    if (emitter.stackChecks?.has(fn.name))
+        emitter.emitStackCheck();
+    if (emitter.workerEntryPolls?.has(fn.name))
+        emitter.emitPendingCheck(undefined, true);
+    emitter.emitStmts(fn.body, true);
+    // Implicit exit of a void function: release the function scope unless
+    // the body already terminated its final block (return, or a throw
+    // whose unwind released everything down to depth 0).
+    if (fn.returnType.kind === "void" && !B.isTerminated()) {
+        emitter.releaseScope(emitter.scopes[0]);
+        // Completing by falling off the end has nothing to await.
+        if (emitter.currentAsyncGenerator)
+            emitter.emitAsyncGenSettled();
+        if (emitter.currentCoro !== null) {
+            emitter.emitWasiFulfill(null);
+            B.terminate(`br label %${emitter.currentCoro.finalLabel}`);
+        }
+        else {
+            B.terminate("ret void");
+        }
+    }
+    emitter.scopes.pop();
+    const coro = emitter.currentCoro;
+    if (coro !== null) {
+        B.startBlock(coro.finalLabel);
+        // Every exit of a coroutine body joins here: its source frame (part
+        // of the coroutine frame) leaves the generator's own chain.
+        const epilogue = B.returnEpilogue;
+        B.returnEpilogue = null;
+        if (epilogue !== null)
+            B.line(epilogue);
+        if (fn.captures !== undefined) {
+            emitter.declare(`declare void @scr_closure_release(ptr)`);
+            B.line(`call void @scr_closure_release(ptr %sc_env)`);
+        }
+        if (coro.kind === "generator") {
+            const finish = emitter.wasi ? "scr_wasi_gen_finish" : "scr_gen_coro_finish";
+            emitter.declare(`declare void @${finish}(ptr)`);
+            B.line(`call void @${finish}(ptr ${coro.self})`);
+        }
+        else {
+            const finish = emitter.wasi ? "scr_wasi_async_finish" : "scr_coro_async_finish";
+            emitter.declare(`declare void @${finish}(ptr)`);
+            B.line(`call void @${finish}(ptr ${coro.self})`);
+        }
+        const finalState = B.tmp();
+        B.line(`${finalState} = call i8 @llvm.coro.suspend(token none, i1 true)`);
+        B.terminate(`switch i8 ${finalState}, label %${coro.suspendLabel} [ i8 1, label %${coro.cleanupLabel} ]`);
+        B.startBlock(coro.cleanupLabel);
+        const frame = B.tmp();
+        B.line(`${frame} = call ptr @llvm.coro.free(token ${coro.id}, ptr ${coro.handle})`);
+        B.line(`call void @${emitter.wasi ? "free" : "scr_rt_free"}(ptr ${frame})`);
+        B.br(coro.suspendLabel);
+        B.startBlock(coro.suspendLabel);
+        B.line(`call void @llvm.coro.end(ptr ${coro.handle}, i1 false, token none)`);
+        const ret = emitter.llType(fn.returnType);
+        if (ret === "void")
+            B.terminate(`ret void`);
+        else if (ret === "double")
+            B.terminate(`ret double ${f64Lit(0)}`);
+        else if (ret === "i1")
+            B.terminate(`ret i1 false`);
+        else
+            B.terminate(`ret ptr null`);
+    }
+    // Shared blocks have multiple source locations. Keep the location on
+    // each incoming exception check instead of assigning one to the cleanup.
+    B.debugLocation = null;
+    for (const cleanup of emitter.unwindCleanups.values()) {
+        B.startBlock(cleanup.label);
+        emitter.releaseScope(cleanup.entries);
+        B.terminate(cleanup.terminator);
+    }
+    if (emitter.logArgSlots > 0) {
+        B.entryAllocas.push(`%logargs = alloca [${emitter.logArgSlots} x %ScrLogArg]`);
+    }
+    const params = fn.params.map((p, index) => int32Abi?.params[index]
+        ? `i32 %p_${mangleLocal(p.localId)}${INT32_ABI_SUFFIX}`
+        : `${emitter.llType(p.type)} %p_${mangleLocal(p.localId)}`);
+    // The i32 entry widens its i32 parameters once; the proof makes the
+    // double exact, so readers of the incoming number see the same value.
+    if (int32Abi)
+        fn.params.forEach((p, index) => {
+            if (!int32Abi.params[index])
+                return;
+            const name = `%p_${mangleLocal(p.localId)}`;
+            B.entryAllocas.push(`${name} = sitofp i32 ${name}${INT32_ABI_SUFFIX} to double`);
+        });
+    // Lifted functions receive their closure first (the callValue ABI).
+    if (fn.captures !== undefined)
+        params.unshift("ptr %sc_env");
+    const ret = int32Abi?.result
+        ? "i32"
+        : int32Abi?.union
+            ? SCALAR_UNION_RESULT
+            : emitter.llType(fn.returnType);
+    let attrs = coro !== null ? "#1" : FN_ATTRS;
+    // The shared indexed-read helper (lowerSafeIndexRead) is a dense bounds
+    // test, a state test and a load behind cold runtime fallbacks, called
+    // from every `a[i]` of one element type. Without the hint LLVM keeps the
+    // hot instances out of line once they have many callers: every read pays
+    // a call, and an integer index converts to a double and back across it.
+    if (fn.name.startsWith("%arr.idxOr."))
+        attrs = `inlinehint ${attrs}`;
+    // Interface dispatch is a few key tests and direct calls ahead of the
+    // checked dynamic call. Keeping that fallback out of line leaves the
+    // tests small enough to inline at each call site.
+    if (fn.name.startsWith("%dyn.class.protocol.fallback."))
+        attrs = `noinline ${attrs}`;
+    else if (fn.name.startsWith("%dyn.class.protocol."))
+        attrs = `inlinehint ${attrs}`;
+    if (fn.name.startsWith(RESULT_ENTRY_PREFIX))
+        attrs = `inlinehint ${attrs}`;
+    if (unoptimized && attrs === FN_ATTRS)
+        attrs = `noinline optnone ${attrs}`;
+    const debug = emitter.debugScope === null ? "" : ` !dbg ${emitter.debugScope}`;
+    const symbol = borrowed ? mangleBorrowedFunction(fn.name) : mangleFunction(fn.name);
+    emitter.privateGlobalSlots.clear();
+    let body = `define internal ${ret} @${symbol}${int32Abi ? INT32_ABI_SUFFIX : ""}(${params.join(", ")}) ${attrs}${debug} { ; ${fn.name}\n${B.render()}\n}`;
+    if (int32Abi)
+        body += "\n" + emitter.emitInt32AbiWrapper(fn, symbol, int32Abi);
+    if (emitter.integerClones.members.has(fn.name)) {
+        body += "\n" + emitIntegerClone(fn, emitter, emitter.stackChecks?.has(fn.name) === true);
+        if (emitter.integerClones.dispatched.has(fn.name))
+            body +=
+                "\n" +
+                    emitIntegerDispatcher(fn, emitter, `${symbol}${int32Abi ? INT32_ABI_SUFFIX : ""}`, int32Abi?.params ?? [], int32Abi?.result === true);
+    }
+    if (!borrowed)
+        return body;
+    const slot = emitter.implSlotBorrowed(fn.name);
+    const virtual = slot.size > 0 && slot.size !== borrowed.size
+        ? "\n" + emitter.emitOwnedCallAdapter(fn, borrowed, slot)
+        : "";
+    return body + "\n" + emitter.emitOwnedCallAdapter(fn, borrowed) + virtual;
+}
+/** Per-generator-function machinery — the async scaffolding's lazy
+ * sibling (async.ts's emitGenScaffolding): the same argument pack,
+ * a fiber trampoline whose epilogue stores the COMPLETION value (or
+ * consumes the GENRET sentinel, promoting the parked .return value), a
+ * spawn wrapper that only ALLOCATES the suspended fiber, and the
+ * never-started teardown that drops the packed (+1) arguments. */
+export function emitGenScaffolding(emitter) {
+    const out = [];
+    for (const fn of emitter.mod.functions) {
+        if (fn.generator === undefined)
+            continue;
+        const { definitions, pack, lifted, fieldTys, ret, tr, spawnParams, argPackLines } = emitter.emitArgPackAndTrampolinePrologue(fn);
+        for (const line of definitions)
+            out.push(line);
+        emitter.declare(`declare zeroext i1 @scr_exc_genret_pending()`);
+        emitter.declare(`declare void @scr_exc_clear()`);
+        emitter.declare(`declare ptr @scr_gen_of_fiber(ptr)`);
+        emitter.declare(`declare void @scr_gen_ret_to_out(ptr)`);
+        emitter.declare(`declare ptr @scr_gen_new(ptr, ptr, ptr)`);
+        emitter.needOom();
+        const coroBody = emitter.coroLowered(fn);
+        if (lifted && !coroBody) {
+            emitter.declare(`declare void @scr_closure_release(ptr)`);
+        }
+        // Normal completion stores the (typed) return value; void completes
+        // with the NONE slot — JS's undefined done-value. A GENRET unwind
+        // consumes the sentinel and promotes the parked .return value; a
+        // real exception stays pending (the consumer-side resume moves it).
+        if (coroBody) {
+            tr.push(`  ret void`, `}`, ``);
+        }
+        else {
+            tr.push(`  %g = call ptr @scr_gen_of_fiber(ptr %self)`, ...emitter.pendingTestLines("%pend"), `  br i1 %pend, label %thrown, label %clean`, `clean:`);
+            switch (ret.kind) {
+                case "void":
+                    tr.push(`  br label %done ; void body: the done value is undefined (NONE)`);
+                    break;
+                case "f64":
+                case "date":
+                    emitter.declare(`declare void @scr_gen_out_f64(ptr, double)`);
+                    tr.push(`  call void @scr_gen_out_f64(ptr %g, double %r)`, `  br label %done`);
+                    break;
+                case "bool":
+                    emitter.declare(`declare void @scr_gen_out_bool(ptr, i1 zeroext)`);
+                    tr.push(`  call void @scr_gen_out_bool(ptr %g, i1 %r)`, `  br label %done`);
+                    break;
+                default: {
+                    const v = vAdapters(emitter.shapeHost, ret);
+                    emitter.declare(`declare void @scr_gen_out_ref(ptr, ptr, ptr)`);
+                    tr.push(`  call void @scr_gen_out_ref(ptr %g, ptr %r, ptr ${v.release})`, `  br label %done`);
+                }
+            }
+            tr.push(`thrown:`, `  %genret = call zeroext i1 @scr_exc_genret_pending()`, `  br i1 %genret, label %promote, label %dropdummy`, `promote:`, `  call void @scr_exc_clear()`, `  call void @scr_gen_ret_to_out(ptr %g)`, `  br label %dropdummy`, `dropdummy:`);
+            if (ret.kind !== "void" && isRefCounted(ret)) {
+                tr.push(`  call void ${releaseSym(emitter.shapeHost, ret)}(ptr %r) ; unwound: the never-read dummy`);
+            }
+            tr.push(`  br label %done`, `done:`, `  ret void`, `}`, ``);
+        }
+        for (const line of tr)
+            out.push(line);
+        let settleAsync = null;
+        if (fn.async) {
+            emitter.declare(`declare ptr @scr_async_gen_new(ptr, ptr, ptr, ptr)`);
+            const genT = {
+                kind: "generator",
+                async: true,
+                yieldT: fn.generator.yieldT,
+                retT: ret,
+                nextT: fn.generator.nextT,
+            };
+            const resultT = fn.generator.resultType;
+            const build = emitter.genResultThunkFor(genT, resultT);
+            settleAsync = `${build}_async`;
+            const settleKey = `ags:${typeKey(genT)}`;
+            if (!emitter.resolveThunks.has(settleKey)) {
+                emitter.resolveThunks.set(settleKey, settleAsync);
+                const adapters = vAdapters(emitter.shapeHost, resultT);
+                emitter.resolveThunkDefs.push(`define internal void @${settleAsync}(ptr %g, ptr %p) ${FN_ATTRS} {`, `entry:`, `  %r = call ptr @${build}(ptr %g)`, `  call void @scr_promise_fulfill_ref(ptr %p, ptr %r, ptr ${adapters.retain}, ptr ${adapters.release}, ptr ${traceArg(emitter.shapeHost, resultT)})`, `  ret void`, `}`, ``);
+            }
+            emitter.declare(`declare void @scr_promise_fulfill_ref(ptr, ptr, ptr, ptr, ptr)`);
+        }
+        // The never-started teardown: drop the packed (+1) arguments.
+        const dr = [
+            `define internal void @${mangleGenDrop(fn.name)}(ptr %ap) ${FN_ATTRS} {`,
+            `entry:`,
+        ];
+        fieldTys.forEach((ty, i) => {
+            const pType = lifted && i === 0 ? null : fn.params[lifted ? i - 1 : i].type;
+            const refcounted = pType === null || isRefCounted(pType);
+            if (!refcounted)
+                return;
+            dr.push(`  %dp${i} = getelementptr inbounds %${pack}, ptr %ap, i64 0, i32 ${i}`, `  %dv${i} = load ptr, ptr %dp${i}`);
+            if (pType === null) {
+                emitter.declare(`declare void @scr_closure_release(ptr)`);
+                dr.push(`  call void @scr_closure_release(ptr %dv${i})`);
+            }
+            else {
+                dr.push(`  call void ${releaseSym(emitter.shapeHost, pType)}(ptr %dv${i})`);
+            }
+        });
+        dr.push(`  call void @free(ptr %ap)`, `  ret void`, `}`, ``);
+        for (const line of dr)
+            out.push(line);
+        // An iterator helper's return() before its first next() closes the
+        // source generator in its first argument slot.
+        const closesSource = fn.generator.closesSource === true;
+        if (fn.generator.tag !== undefined)
+            emitter.declare(`declare void @scr_gen_set_tag(ptr, ptr)`);
+        if (fn.generator.inheritsTag === true)
+            emitter.declare(`declare void @scr_gen_copy_tag(ptr, ptr)`);
+        if (closesSource) {
+            const field = lifted ? 1 : 0;
+            emitter.declare(`declare void @scr_gen_close(ptr)`);
+            emitter.declare(`declare void @scr_gen_set_unstarted_return(ptr, ptr)`);
+            out.push(`define internal void @${mangleGenSourceClose(fn.name)}(ptr %ap) ${FN_ATTRS} {`, `entry:`, `  %sp = getelementptr inbounds %${pack}, ptr %ap, i64 0, i32 ${field}`, `  %sv = load ptr, ptr %sp`, `  call void @scr_gen_close(ptr %sv)`, `  ret void`, `}`, ``);
+        }
+        // Spawn wrapper: pack the args (+1 moves in), allocate the
+        // SUSPENDED fiber — nothing runs until the first .next().
+        if (lifted) {
+            emitter.declare(`declare ptr @scr_closure_retain_v(ptr)`);
+        }
+        const sp = [
+            `define internal ptr @${mangleGenSpawn(fn.name)}(${spawnParams.join(", ")}) ${FN_ATTRS} { ; gen spawn ${fn.name}`,
+            `entry:`,
+            ...argPackLines,
+        ];
+        const genNew = coroBody && !emitter.wasi ? "scr_gen_new_stackless" : "scr_gen_new";
+        if (genNew !== "scr_gen_new")
+            emitter.declare(`declare ptr @${genNew}(ptr, ptr, ptr)`);
+        sp.push(settleAsync === null
+            ? `  %gg = call ptr @${genNew}(ptr @${mangleTrampoline(fn.name)}, ptr %ap, ptr @${mangleGenDrop(fn.name)})`
+            : `  %gg = call ptr @scr_async_gen_new(ptr @${mangleTrampoline(fn.name)}, ptr %ap, ptr @${mangleGenDrop(fn.name)}, ptr @${settleAsync})`, ...(fn.generator.tag !== undefined
+            ? [`  call void @scr_gen_set_tag(ptr %gg, ptr ${emitter.cstr(fn.generator.tag)})`]
+            : []), ...(fn.generator.inheritsTag === true
+            ? [`  call void @scr_gen_copy_tag(ptr %gg, ptr %a${lifted ? 1 : 0})`]
+            : []), ...(closesSource
+            ? [
+                `  call void @scr_gen_set_unstarted_return(ptr %gg, ptr @${mangleGenSourceClose(fn.name)})`,
+            ]
+            : []), `  ret ptr %gg`, `}`, ``);
+        for (const line of sp)
+            out.push(line);
+    }
+    return out;
+}
+/** Per-async-function machinery — async.ts's scaffolding, .ll
+ * flavored: an argument-pack struct type, a fiber trampoline (unpacks,
+ * frees the pack, runs the ordinary compiled body, settles the
+ * promise — fulfilling on clean return, leaving a pending exception
+ * for the runtime to reject with), and a spawn wrapper call sites and
+ * closures enter through (packs the args +1, scr_async_spawn runs the
+ * fiber eagerly to its first suspension and returns the promise). */
+export function emitAsyncScaffolding(emitter) {
+    const out = [];
+    if (emitter.wasi) {
+        emitter.declare(`declare void @llvm.coro.resume(ptr)`);
+        emitter.declare(`declare void @llvm.coro.destroy(ptr)`);
+        out.push(`define void @scr_wasi_coro_resume(ptr %handle) ${FN_ATTRS} {`, `entry:`, `  call void @llvm.coro.resume(ptr %handle)`, `  ret void`, `}`, `define void @scr_wasi_coro_destroy(ptr %handle) ${FN_ATTRS} {`, `entry:`, `  call void @llvm.coro.destroy(ptr %handle)`, `  ret void`, `}`, ``);
+    }
+    const fiberless = emitter.fiberlessAsyncFunctions();
+    for (const fn of emitter.mod.functions) {
+        if (fn.async !== true || fn.generator !== undefined)
+            continue;
+        if (fiberless.has(fn.name) && emitter.awaitCallees.has(fn.name)) {
+            for (const line of emitter.emitAsyncAwaitCall(fn, null))
+                out.push(line);
+        }
+        if (fiberless.has(fn.name) && emitter.adoptCallees.has(fn.name)) {
+            for (const line of emitter.emitAsyncAwaitCall(fn, null, true))
+                out.push(line);
+        }
+        if (emitter.inlineAwaitCallees.has(fn.name)) {
+            for (const line of emitter.emitAsyncInlineAwaitCall(fn))
+                out.push(line);
+        }
+        if (fiberless.has(fn.name)) {
+            const fieldTys = [
+                ...(fn.captures !== undefined ? ["ptr"] : []),
+                ...fn.params.map((p) => emitter.llType(p.type)),
+            ];
+            const params = fieldTys.map((ty, i) => `${ty} %a${i}`);
+            for (const line of emitter.emitFiberlessAsyncSpawn(fn, params))
+                out.push(line);
+            continue;
+        }
+        const { definitions, ret, tr, spawnParams, argPackLines } = emitter.emitArgPackAndTrampolinePrologue(fn);
+        for (const line of definitions)
+            out.push(line);
+        if (emitter.awaitCallees.has(fn.name)) {
+            for (const line of emitter.emitAsyncAwaitCall(fn, argPackLines))
+                out.push(line);
+        }
+        if (emitter.adoptCallees.has(fn.name)) {
+            for (const line of emitter.emitAsyncAwaitCall(fn, argPackLines, true))
+                out.push(line);
+        }
+        const stackless = emitter.stacklessAsync.has(fn.name);
+        const spawn = stackless ? "scr_async_spawn_stackless" : "scr_async_spawn";
+        emitter.declare(`declare ptr @scr_fiber_promise(ptr)`);
+        emitter.declare(`declare ptr @${spawn}(ptr, ptr)`);
+        emitter.needOom();
+        if (emitter.wasi || stackless) {
+            // The coroutine body settles its own promise and owns the retained
+            // lifted environment until final suspension. Its initial call
+            // returns here at the first suspend (or final suspend).
+            tr.push(`  ret void`, `}`, ``);
+        }
+        else {
+            if (fn.captures !== undefined) {
+                emitter.declare(`declare void @scr_closure_release(ptr)`);
+            }
+            tr.push(...emitter.pendingTestLines("%pend"), `  br i1 %pend, label %thrown, label %clean`, `clean:`, `  %pr = call ptr @scr_fiber_promise(ptr %self)`);
+            for (const line of emitter.asyncFulfillLines(ret, "%pr"))
+                tr.push(line);
+            tr.push(`  ret void`, `thrown:`);
+            if (ret.kind !== "void" && isRefCounted(ret)) {
+                // An escaping throw means %r is the never-read dummy (NULL).
+                tr.push(`  call void ${releaseSym(emitter.shapeHost, ret)}(ptr %r)`);
+            }
+            tr.push(`  ret void`, `}`, ``);
+        }
+        for (const line of tr)
+            out.push(line);
+        // Spawn wrapper: pack the args (+1 moves in), spawn the fiber.
+        const cache = fn.asyncCacheGlobal !== undefined ? mangleGlobal(fn.asyncCacheGlobal) : null;
+        const cycleCache = fn.asyncCycleCacheGlobal !== undefined ? mangleGlobal(fn.asyncCycleCacheGlobal) : null;
+        if (cache !== null || cycleCache !== null) {
+            emitter.declare(`declare ptr @scr_promise_retain_v(ptr)`);
+            emitter.declare(`declare void @scr_promise_release(ptr)`);
+        }
+        if (cache !== null) {
+            emitter.declare(`declare void @scr_promise_mark_handled(ptr)`);
+        }
+        if (fn.captures !== undefined) {
+            emitter.declare(`declare ptr @scr_closure_retain_v(ptr)`);
+        }
+        const sp = [
+            `define internal ptr @${mangleAsyncSpawn(fn.name)}(${spawnParams.join(", ")}) ${FN_ATTRS} { ; spawn ${fn.name}`,
+            `entry:`,
+            ...(cache !== null
+                ? [
+                    `  %cached = load ptr, ptr @${cache}`,
+                    `  %cache_hit = icmp ne ptr %cached, null`,
+                    `  br i1 %cache_hit, label %cached_return, label %cache_miss`,
+                    `cached_return:`,
+                    `  %cached_owned = call ptr @scr_promise_retain_v(ptr %cached)`,
+                    `  ret ptr %cached_owned`,
+                    `cache_miss:`,
+                ]
+                : []),
+            ...argPackLines,
+        ];
+        sp.push(`  %p = call ptr @${spawn}(ptr @${mangleTrampoline(fn.name)}, ptr %ap)`, ...(cache !== null
+            ? [
+                // The module loader owns this evaluation promise
+                // immediately. A later sibling can throw before the
+                // aggregate dependency wait is built, but this rejection
+                // must never become an unrelated unhandled rejection.
+                `  call void @scr_promise_mark_handled(ptr %p)`,
+            ]
+            : []), ...(cache !== null
+            ? [
+                `  %cache_owned = call ptr @scr_promise_retain_v(ptr %p)`,
+                // The eager spawn may have re-entered this guarded module
+                // through an admitted async cycle and installed a temporary
+                // cache entry. Drop that owned slot before replacing it
+                // with the outer evaluation promise.
+                `  %replaced_cache = load ptr, ptr @${cache}`,
+                `  call void @scr_promise_release(ptr %replaced_cache)`,
+                `  store ptr %cache_owned, ptr @${cache}`,
+            ]
+            : []), ...(cycleCache !== null
+            ? [
+                // Eager recursive spawns publish from the inside out. The
+                // runtime-requested outermost member writes last and is the
+                // SCC's actual evaluation root.
+                `  %cycle_cache_owned = call ptr @scr_promise_retain_v(ptr %p)`,
+                `  %replaced_cycle_cache = load ptr, ptr @${cycleCache}`,
+                `  call void @scr_promise_release(ptr %replaced_cycle_cache)`,
+                `  store ptr %cycle_cache_owned, ptr @${cycleCache}`,
+            ]
+            : []), `  ret ptr %p`, `}`, ``);
+        for (const line of sp)
+            out.push(line);
+    }
+    for (const line of emitter.emitGenScaffolding())
+        out.push(line);
+    return out;
+}
+/** The await-call entry of an async function (`sc_aw_*`): one direct
+ * `await f(...)` as a call returning the awaited value, so the call site
+ * needs no promise of its own. A fiberless body runs on the caller's
+ * stack as in its spawn wrapper; a clean result then takes the await's
+ * one microtask turn (scr_await_hop) and is returned without ever
+ * materializing the promise, which nothing else could observe. A throw
+ * builds the rejected promise exactly as the spawn wrapper would and
+ * awaits it, so the rejection reaches the awaiter on the same turn. Other
+ * bodies pack their arguments as the spawn wrapper does and enter
+ * scr_async_spawn_await_*, which parks the caller on the promise as soon
+ * as the body first suspends. The adopting entry (`sc_ad_*`, for
+ * `return f(...)` in an async function) adds the turn of the then job
+ * that adopts the returned promise, before the await proper. */
+export function emitAsyncAwaitCall(emitter, fn, argPackLines, adopt) {
+    const ret = fn.returnType;
+    const retTy = emitter.llType(ret);
+    const params = fn.params.map((p, i) => `${emitter.llType(p.type)} %a${i}`);
+    const awaitLine = (dest, promise) => {
+        switch (ret.kind) {
+            case "void":
+                emitter.declare(`declare void @scr_await_void(ptr)`);
+                return `  call void @scr_await_void(ptr ${promise})`;
+            case "f64":
+            case "date":
+                emitter.declare(`declare double @scr_await_f64(ptr)`);
+                return `  ${dest} = call double @scr_await_f64(ptr ${promise})`;
+            case "bool":
+                emitter.declare(`declare zeroext i1 @scr_await_bool(ptr)`);
+                return `  ${dest} = call zeroext i1 @scr_await_bool(ptr ${promise})`;
+            case "string":
+                emitter.declare(`declare ptr @scr_await_str(ptr)`);
+                return `  ${dest} = call ptr @scr_await_str(ptr ${promise})`;
+            default:
+                emitter.declare(`declare ptr @scr_await_ref(ptr)`);
+                return `  ${dest} = call ptr @scr_await_ref(ptr ${promise})`;
+        }
+    };
+    const retLine = (v) => (retTy === "void" ? `  ret void` : `  ret ${retTy} ${v}`);
+    const head = adopt
+        ? `define internal ${retTy} @${mangleAsyncAdoptCall(fn.name)}(${params.join(", ")}) ${FN_ATTRS} { ; adopt ${fn.name}`
+        : `define internal ${retTy} @${mangleAsyncAwaitCall(fn.name)}(${params.join(", ")}) ${FN_ATTRS} { ; await ${fn.name}`;
+    const args = params.join(", ");
+    if (adopt && argPackLines !== null) {
+        // A body that may suspend settles later than the adoption's then job
+        // runs: materialize its promise, take that turn, then await it.
+        emitter.declare(`declare void @scr_await_hop()`);
+        emitter.declare(`declare void @scr_promise_release(ptr)`);
+        return [
+            head,
+            `entry:`,
+            `  %p = call ptr @${mangleAsyncSpawn(fn.name)}(${args})`,
+            `  call void @scr_await_hop()`,
+            awaitLine("%v", "%p"),
+            `  call void @scr_promise_release(ptr %p)`,
+            retLine("%v"),
+            `}`,
+            ``,
+        ];
+    }
+    if (argPackLines !== null && emitter.stacklessAsync.has(fn.name)) {
+        // A stackless body's eager prefix already runs on the awaiter's
+        // stack, which then awaits the promise like the two separate steps.
+        emitter.declare(`declare void @scr_promise_release(ptr)`);
+        return [
+            head,
+            `entry:`,
+            `  %p = call ptr @${mangleAsyncSpawn(fn.name)}(${args})`,
+            awaitLine("%v", "%p"),
+            `  call void @scr_promise_release(ptr %p)`,
+            retLine("%v"),
+            `}`,
+            ``,
+        ];
+    }
+    if (argPackLines !== null) {
+        const kind = ret.kind === "void"
+            ? "void"
+            : ret.kind === "f64" || ret.kind === "date"
+                ? "f64"
+                : ret.kind === "bool"
+                    ? "bool"
+                    : ret.kind === "string"
+                        ? "str"
+                        : "ref";
+        const cret = kind === "bool" ? "zeroext i1" : retTy;
+        emitter.declare(`declare ${cret} @scr_async_spawn_await_${kind}(ptr, ptr)`);
+        emitter.needOom();
+        const call = `call ${cret} @scr_async_spawn_await_${kind}(ptr @${mangleTrampoline(fn.name)}, ptr %ap)`;
+        return [
+            head,
+            `entry:`,
+            ...argPackLines,
+            kind === "void" ? `  ${call}` : `  %v = ${call}`,
+            retLine("%v"),
+            `}`,
+            ``,
+        ];
+    }
+    emitter.declare(`declare void @scr_async_inline_enter(ptr)`);
+    emitter.declare(`declare void @scr_async_inline_exit(ptr)`);
+    emitter.declare(`declare void @scr_await_hop()`);
+    emitter.declare(`declare ptr @scr_promise_new()`);
+    emitter.declare(`declare void @scr_promise_reject_pending(ptr)`);
+    emitter.declare(`declare void @scr_promise_release(ptr)`);
+    const bodyCall = `call ${retTy} @${mangleFunction(fn.name)}(${args})`;
+    const out = [
+        head,
+        `entry:`,
+        `  %frame = alloca { ptr, ptr }`,
+        `  call void @scr_async_inline_enter(ptr %frame)`,
+        retTy === "void" ? `  ${bodyCall}` : `  %r = ${bodyCall}`,
+        ...emitter.pendingTestLines("%pend"),
+        `  call void @scr_async_inline_exit(ptr %frame)`,
+        // Adopting: the then job's turn comes before the settled promise's
+        // own reaction, whichever way the body completed.
+        ...(adopt ? [`  call void @scr_await_hop()`] : []),
+        `  br i1 %pend, label %thrown, label %clean`,
+        `clean:`,
+        `  call void @scr_await_hop()`,
+        retLine("%r"),
+        `thrown:`,
+    ];
+    if (ret.kind !== "void" && isRefCounted(ret)) {
+        // An escaping throw means %r is the never-read dummy (NULL).
+        out.push(`  call void ${releaseSym(emitter.shapeHost, ret)}(ptr %r)`);
+    }
+    out.push(`  %p = call ptr @scr_promise_new()`, `  call void @scr_promise_reject_pending(ptr %p)`, awaitLine("%v", "%p"), `  call void @scr_promise_release(ptr %p)`, retLine("%v"), `}`, ``);
+    return out;
+}
+/** Move a clean async return value into the current fiber's promise. */
+export function emitWasiFulfill(emitter, v) {
+    const coro = emitter.currentCoro;
+    if (coro === null)
+        throw new InternalCompilerError("llvm emitter bug: wasm fulfillment outside async body");
+    const ret = emitter.currentReturnType;
+    if (coro.kind === "generator") {
+        emitter.declare(`declare ptr @scr_gen_of_fiber(ptr)`);
+        const gen = emitter.B.tmp();
+        emitter.B.line(`${gen} = call ptr @scr_gen_of_fiber(ptr ${coro.self})`);
+        switch (ret.kind) {
+            case "void":
+                return;
+            case "f64":
+            case "date":
+                emitter.declare(`declare void @scr_gen_out_f64(ptr, double)`);
+                emitter.B.line(`call void @scr_gen_out_f64(ptr ${gen}, double ${v.name})`);
+                return;
+            case "bool":
+                emitter.declare(`declare void @scr_gen_out_bool(ptr, i1 zeroext)`);
+                emitter.B.line(`call void @scr_gen_out_bool(ptr ${gen}, i1 ${v.name})`);
+                return;
+            default:
+                emitter.declare(`declare void @scr_gen_out_ref(ptr, ptr, ptr)`);
+                emitter.B.line(`call void @scr_gen_out_ref(ptr ${gen}, ptr ${v.name}, ptr ${vAdapters(emitter.shapeHost, ret).release})`);
+                return;
+        }
+    }
+    emitter.declare(`declare ptr @scr_fiber_promise(ptr)`);
+    const pr = emitter.B.tmp();
+    emitter.B.line(`${pr} = call ptr @scr_fiber_promise(ptr ${coro.self})`);
+    switch (ret.kind) {
+        case "void":
+            emitter.declare(`declare void @scr_promise_fulfill_void(ptr)`);
+            emitter.B.line(`call void @scr_promise_fulfill_void(ptr ${pr})`);
+            break;
+        case "f64":
+        case "date":
+            emitter.declare(`declare void @scr_promise_fulfill_f64(ptr, double)`);
+            emitter.B.line(`call void @scr_promise_fulfill_f64(ptr ${pr}, double ${v.name})`);
+            break;
+        case "bool":
+            emitter.declare(`declare void @scr_promise_fulfill_bool(ptr, i1 zeroext)`);
+            emitter.B.line(`call void @scr_promise_fulfill_bool(ptr ${pr}, i1 ${v.name})`);
+            break;
+        case "string":
+            emitter.declare(`declare void @scr_promise_fulfill_str(ptr, ptr)`);
+            emitter.B.line(`call void @scr_promise_fulfill_str(ptr ${pr}, ptr ${v.name}) ; moves in`);
+            break;
+        case "dyn":
+            emitter.declare(`declare void @scr_promise_resolve_dyn(ptr, ptr)`);
+            emitter.B.line(`call void @scr_promise_resolve_dyn(ptr ${pr}, ptr ${v.name}) ; moves in`);
+            break;
+        default: {
+            const rc = vAdapters(emitter.shapeHost, ret);
+            emitter.declare(`declare void @scr_promise_fulfill_ref(ptr, ptr, ptr, ptr, ptr)`);
+            emitter.B.line(`call void @scr_promise_fulfill_ref(ptr ${pr}, ptr ${v.name}, ptr ${rc.retain}, ptr ${rc.release}, ptr ${traceArg(emitter.shapeHost, ret)})`);
+        }
+    }
+}
+/** The argument-pack ABI and trampoline prefix shared by async functions
+ * and generators. Their promise/generator completion and spawn tails stay
+ * with the callers below. */
+export function emitArgPackAndTrampolinePrologue(emitter, fn) {
+    const pack = mangleArgPack(fn.name);
+    const lifted = fn.captures !== undefined;
+    const fieldTys = [...(lifted ? ["ptr"] : []), ...fn.params.map((p) => emitter.llType(p.type))];
+    const definitions = [`%${pack} = type { ${fieldTys.join(", ") || "i8"} } ; ${fn.name} args`];
+    const sizeOf = `ptrtoint (ptr getelementptr (%${pack}, ptr null, i32 1) to ${emitter.sizeType})`;
+    // Async argument packs live from the spawn to the trampoline's first
+    // instructions, so they take the runtime's small-block allocator.
+    // Generator packs keep malloc: the runtime frees never-started ones.
+    const smallPack = fn.generator === undefined;
+    const packAlloc = smallPack ? "scr_rt_calloc" : "malloc";
+    const packFree = smallPack ? "scr_rt_free" : "free";
+    emitter.declare(`declare void @${packFree}(ptr)`);
+    emitter.declare(`declare ptr @${packAlloc}(${emitter.sizeType})`);
+    const tr = [
+        `define internal void @${mangleTrampoline(fn.name)}(ptr %self, ptr %ap) ${FN_ATTRS} {`,
+        `entry:`,
+    ];
+    const loads = [];
+    fieldTys.forEach((ty, i) => {
+        tr.push(`  %fp${i} = getelementptr inbounds %${pack}, ptr %ap, i64 0, i32 ${i}`, `  %a${i} = load ${ty}, ptr %fp${i}`);
+        loads.push(`${ty} %a${i}`);
+    });
+    tr.push(`  call void @${packFree}(ptr %ap)`);
+    const ret = fn.returnType;
+    const retTy = emitter.llType(ret);
+    const bodyCall = `call ${retTy} @${mangleFunction(fn.name)}(${loads.join(", ")})`;
+    tr.push(retTy === "void" ? `  ${bodyCall}` : `  %r = ${bodyCall}`);
+    if (lifted && !emitter.coroLowered(fn)) {
+        tr.push(`  call void @scr_closure_release(ptr %a0)`);
+    }
+    const spawnParams = fieldTys.map((ty, i) => `${ty} %a${i}`);
+    const argPackLines = [
+        `  %ap = call ptr @${packAlloc}(${emitter.sizeType} ${sizeOf})`,
+        `  %isnull = icmp eq ptr %ap, null`,
+        `  br i1 %isnull, label %oom, label %ok`,
+        `oom:`,
+        `  call void @sc_oom()`,
+        `  unreachable`,
+        `ok:`,
+    ];
+    if (lifted) {
+        // scr_closure_retain is a header static inline — the `_v` twin is
+        // the exported symbol.
+        argPackLines.push(`  %env = call ptr @scr_closure_retain_v(ptr %a0)`);
+    }
+    fieldTys.forEach((ty, i) => {
+        const src = lifted && i === 0 ? "%env" : `%a${i}`;
+        argPackLines.push(`  %sp${i} = getelementptr inbounds %${pack}, ptr %ap, i64 0, i32 ${i}`, `  store ${ty} ${src}, ptr %sp${i}`);
+    });
+    return { definitions, pack, lifted, fieldTys, ret, tr, spawnParams, argPackLines };
+}
+/** The module functions whose bodies may park their execution context:
+ * their emitted text mentions a suspension entry point (SUSPENDS), or
+ * they mention such a function (transitively). Null where async bodies
+ * must keep fibers regardless: wasm32 (its coroutine lowering owns async
+ * bodies), worker programs, and programs that use node:test subtests,
+ * whose `t.test()` awaits on the CURRENT fiber even from synchronous
+ * code. */
+export function parkingFunctions(emitter) {
+    if (emitter.wasi || emitter.mod.workers === true)
+        return null;
+    for (const decl of emitter.decls)
+        if (/@scr_test_sub\b/.test(decl))
+            return null;
+    for (const text of emitter.fnDefText.values())
+        if (/@scr_test_sub\b/.test(text))
+            return null;
+    // Reverse mention edges, then propagate "may park" to every mentioner.
+    const mentionedBy = new Map();
+    const parks = new Set();
+    const work = [];
+    for (const [name, text] of emitter.fnDefText) {
+        if (SUSPENDS.test(text)) {
+            parks.add(name);
+            work.push(name);
+        }
+        for (const callee of emitter.mentionedFunctions(text)) {
+            if (callee === name)
+                continue;
+            let callers = mentionedBy.get(callee);
+            if (callers === undefined)
+                mentionedBy.set(callee, (callers = new Set()));
+            callers.add(name);
+        }
+    }
+    while (work.length > 0) {
+        const callers = mentionedBy.get(work.pop());
+        if (callers === undefined)
+            continue;
+        for (const caller of callers) {
+            if (parks.has(caller))
+                continue;
+            parks.add(caller);
+            work.push(caller);
+        }
+    }
+    return parks;
+}
+/** Async functions whose calls can skip the fiber (scr_async_inline_*).
+ * A body qualifies only when nothing it runs can park its execution
+ * context: its emitted text, and the text of every module function it
+ * mentions (transitively), contains no suspension entry point. Such a
+ * body always completes before the call returns, which is exactly the
+ * behavior of a fiber that never suspends. Excluded conservatively:
+ * wasm32 (its coroutine lowering owns async bodies), module-evaluation
+ * bodies (their promise caches),
+ * `dyn` results (resolution may adopt a thenable), and every program
+ * that uses node:test subtests — `t.test()` runs a subtest inline and
+ * awaits it on the CURRENT fiber even from synchronous code. Worker
+ * programs keep fibers too: their fiber epilogue owns context
+ * termination (a stopping worker rejects and observes the promise), which
+ * the fiberless wrapper does not reproduce. A missed suspension still
+ * cannot reorder execution undetected: the runtime traps when a context parks while a
+ * fiberless body is on its stack. */
+export function fiberlessAsyncFunctions(emitter) {
+    const parks = emitter.parkingFunctions();
+    const eligible = parks === null ? new Set() : emitter.neverParkingAsync(parks);
+    for (const name of emitter.inlineAwaitCallees)
+        if (!eligible.has(name))
+            throw new InternalCompilerError(`llvm emitter bug: inline await of suspending ${name}`);
+    return eligible;
+}
+//# sourceMappingURL=emit-function.js.map