@scriptc/compiler 0.3.1 → 0.3.2

package/dist/backend/llvm/emitter.js

@@ -1,7 +1,7 @@
 import { emitLlvmLayouts } from "./layouts.js";
 import { compileRegexMatcher } from "./regex-matchers.js";
-import { FIRST_EMITTED_CYC_KIND, inlineAllocSupported } from "./alloc.js";
-import { llvmBytes as llBytes } from "../literals.js";
+import { CYC_BUF, CYC_HDR, FIRST_EMITTED_CYC_KIND, inlineAllocSupported } from "./alloc.js";
+import { hasLoneSurrogate, llvmBytes as llBytes, wtf8Bytes } from "../literals.js";
 import { InternalCompilerError } from "../../errors.js";
 import { createHash } from "node:crypto";
 import { endsWithJump, matchStringSelfConcat, streamTypedRefEligible, undefinedArmTag, } from "../../ir/analysis.js";
@@ -16,6 +16,7 @@ import { findBytesBounds } from "./bytes-bounds.js";
 import { emitByteWindowGuard, findInitializedByteLoopBindings, matchByteWindow, } from "./byte-windows.js";
 import { canDeferSplitPiece, emitSplitSpanNext, materializeSplitPiece, emitSplitCursor, emitSplitNext, emitSplitScratch, emitSplitSnapshot, findPrivateSplitLocals, loadSplitSnapshot, storeSplitSnapshot, stringSplit, } from "./split-loops.js";
 import { scalarizeNumericRecords } from "../../ir/scalar-records.js";
+import { addRecordResultEntries } from "./record-results.js";
 import { specializeNumericCalls } from "../../ir/numeric-call-specialization.js";
 import { everyExpr, everyExprChild, everyStmtChild, everyStmtList } from "../../ir/traverse.js";
 import { analyzeIntegerRanges, INT32_RANGE } from "../../ir/integer-ranges.js";
@@ -24,7 +25,7 @@ import { analyzeInt32Slots } from "../../ir/int32-slots.js";
 import { analyzeFinalFields, constructorClass } from "../../ir/final-fields.js";
 import { foldConstantGlobals } from "../../ir/constant-globals.js";
 import { findIntegerViews } from "./integer-views.js";
-import { findInt32Abis, INT32_ABI_SUFFIX } from "./int32-abi.js";
+import { findInt32Abis, INT32_ABI_SUFFIX, SCALAR_UNION_DUMMY, SCALAR_UNION_RESULT, } from "./int32-abi.js";
 import { findInt32Locals } from "./int32-locals.js";
 import { exactInteger, narrowInteger } from "./integer-values.js";
 import { findInitializerBindings, withInitializerBindings } from "../../ir/initializer-bindings.js";
@@ -35,7 +36,8 @@ import { computeMayThrow } from "../may-throw.js";
 import { mangleArgPack, mangleAsyncAdoptCall, mangleAsyncAwaitCall, mangleAsyncSpawn, mangleBorrowedFunction, mangleClassObj, mangleClassStruct, mangleFnClosure, mangleFunction, mangleGenDrop, mangleGenSpawn, mangleGlobal, mangleLocal, mangleRecordStruct, mangleTrampoline, mangleWrapper, mangleBorrowingWrapper, mangleVtInstance, } from "../mangle.js";
 import { analyzeCallLifetimes } from "./call-lifetimes.js";
 import { canStackUnion, emitStackUnion, findLocalStackUnions } from "./stack-unions.js";
-import { allocateLocalUnion, findLocalUnionStorage, storeLocalUnion, } from "./local-union-storage.js";
+import { emitStackObject, findLocalStackObjects, findReassignedStackObjects, stackObjectStruct, } from "./stack-objects.js";
+import { allocateLocalUnion, emitScalarOptional, findLocalUnionStorage, isScalarOptionalSource, storeLocalUnion, } from "./local-union-storage.js";
 import { BlockBuilder } from "./blocks.js";
 import { NullableRefFields } from "./nullable-fields.js";
 import { emitLocalArrayRead, emitProjectedArrayRead, emitBorrowedArrayRead, findLocalArrayReads, findCallArrayReads, OptionalArrayReads, } from "./local-array-reads.js";
@@ -51,6 +53,7 @@ import { ReferenceWrites } from "./reference-writes.js";
 import { LlvmDebugInfo } from "./debug-info.js";
 import { StackCallbacks } from "./stack-callbacks.js";
 import { findLoopArrayBorrows } from "./loop-array-borrows.js";
+import { findAppendedArrayLiterals } from "./appended-array-literals.js";
 import { AmbientReceiverReaders, findConstantCallbacks } from "./constant-callbacks.js";
 import { findScalarStringSlices, emitStringSliceSnapshot, } from "./string-slices.js";
 import { LazyCaptures } from "./lazy-captures.js";
@@ -77,7 +80,8 @@ import { emitChildProcessLibCall, emitAsyncContextLibCall, emitProcessLibCall, e
 import { emitStreamLibCall } from "./lib-stream.js";
 import { emitNetworkHttpLibCall } from "./lib-network.js";
 import { emitAssertInspectLibCall, emitIoLibCall, emitGenericLibCall, emitLibCall, } from "./lib-dispatch.js";
-import { buildClassGraph, classMembershipIntervals, emitClassMembershipHelper, classEnvironmentIndex, classFieldIndex, classStructSym, } from "./classes.js";
+import { buildClassGraph, classMembershipIntervals, emitClassMembershipHelper, classEnvironmentIndex, classFieldIndex, classStructSym, programLifetimeClass, } from "./classes.js";
+import { LifetimeRegions, REGION_TYPES } from "./regions.js";
 import { LlDyn } from "./dyn.js";
 import { VirtualBorrows } from "./virtual-borrows.js";
 import { LlvmUnsupportedError } from "./unsupported.js";
@@ -86,7 +90,7 @@ import { isObjectArm, NULLABLE_ABSENT, NULLABLE_NULL, NullableUnions } from "./n
 import { emitNullablePresent, emitUnionPeek, emitUnionTag } from "./union-repr.js";
 import { literalCapWord, literalCapWord32 } from "./string-key-hash.js";
 import { NUMBER_MAP_ENTRY_DECL, STRING_MAP_ENTRY_DECL } from "./map-number-lookup.js";
-import { STRING_READ_HELPERS } from "./string-reads.js";
+import { STRING_READ_HELPERS, stringScanDecl } from "./string-reads.js";
 import { emitConcatInputs, emitMixedConcat, numberPart, stringParts, } from "./string-construction.js";
 import { arrNewCall, boxAccess, boxNewCall, boxReleaseSym, computeTraced, elemAccess, emitInlineRcHelpers, emitNullableRetainWrappers, FN_ATTRS, inlineRcDecls, llFieldType, releaseSym, retainSym, traceArg, typedRefConstructor, vAdapters, } from "./shapes.js";
 import { INLINE_LITERAL_BYTES, literalEqualityDefinition } from "./string-literal-equality.js";
@@ -122,14 +126,14 @@ export function emitLlvmModule(input, options = {}) {
     const mod = unshareNodes(input);
     // Keep source storage intact for debugger inspection in dev builds.
     return new LlEmitter(options.debugSources === undefined
-        ? scalarizeNumericRecords(specializeNumericCalls(foldConstantGlobals(mod), mangleFunction))
+        ? addRecordResultEntries(scalarizeNumericRecords(specializeNumericCalls(foldConstantGlobals(mod), mangleFunction)))
         : mod, options).emit();
 }
 /** Keep large translation units below the host's single-string limit. */
 export function emitLlvmModuleSource(input, options = {}) {
     const mod = unshareNodes(input);
     const parts = new LlEmitter(options.debugSources === undefined
-        ? scalarizeNumericRecords(specializeNumericCalls(foldConstantGlobals(mod), mangleFunction))
+        ? addRecordResultEntries(scalarizeNumericRecords(specializeNumericCalls(foldConstantGlobals(mod), mangleFunction)))
         : mod, options).emitParts();
     const length = parts.reduce((total, part) => total + part.length, Math.max(0, parts.length - 1));
     if (length <= 256 * 1024 * 1024)
@@ -137,7 +141,7 @@ export function emitLlvmModuleSource(input, options = {}) {
     return parts.flatMap((part, index) => (index === 0 ? [part] : ["\n", part]));
 }
 function llStrBytes(text) {
-    return llBytes(Buffer.from(text, "utf8"));
+    return llBytes(wtf8Bytes(text));
 }
 const NO_BORROWED = new Set();
 /** Whether any node of `e` names the local (reads, writes, declarations,
@@ -181,6 +185,10 @@ export class LlEmitter {
     scalarStringSlices = new Map();
     stringSlices = new Map();
     localStackUnions = new Map();
+    localStackObjects = new Map();
+    reassignedStackObjects = new Set();
+    /** The binding slot of each reassigned stack-object local. */
+    stackObjectSlots = new Map();
     localUnionStorageProofs = new Map();
     localUnionStorage = new Map();
     integerArrayBindings = new Set();
@@ -203,6 +211,18 @@ export class LlEmitter {
         }
         this.fieldPointerTags.set(ptr, tag);
     }
+    /** The alias tag of a runtime word that emitted code reads and writes
+     * outside every field (the lifetime-region count, owner and cursors). */
+    memoryAliasAttachment(category) {
+        if (this.debug !== null)
+            return "";
+        let tag = this.fieldAliasTags.get(category);
+        if (tag === undefined) {
+            tag = 2 + this.fieldAliasTags.size * 2;
+            this.fieldAliasTags.set(category, tag);
+        }
+        return `, !tbaa !${tag}`;
+    }
     fieldAliasAttachment(ptr) {
         const tag = this.fieldPointerTags.get(ptr);
         return tag === undefined ? "" : `, !tbaa !${tag}`;
@@ -217,10 +237,12 @@ export class LlEmitter {
         return lines;
     }
     debug;
+    /** Projection-only scalar objects may live in frame storage (stack-objects.ts). */
+    stackObjects;
     debugScope = null;
     sizeType;
     ffiExtendNarrowIntegers;
-    cycleBufferedOffset;
+    cycleBufOffset;
     /** Emitted ScrCycType descriptors so far (their collector statistics
      * indices; see ShapeHost.nextCycKind). */
     cycKinds = 0;
@@ -307,6 +329,8 @@ export class LlEmitter {
      * when inline RC is off. */
     rcHelpers;
     fnByName = new Map();
+    /** Empty literals that construct with inline slots (appended-array-literals.ts). */
+    appendedArrayLiterals = new Set();
     /** Emitted LLVM text of every IR function body, by function name: the
      * fiberless-async analysis scans it for suspension entry points. */
     fnDefText = new Map();
@@ -318,6 +342,8 @@ export class LlEmitter {
      * whose bindings end with the list (emitBlock and function bodies). */
     stmtCursor = null;
     optionalArrayReads;
+    /** Union sources a binding or projection can hold in a stack box. */
+    unionSources;
     checkedNarrows;
     callArrayReads = new Map();
     mapReadLifetimes = { locals: new Map(), arguments: new Map() };
@@ -413,6 +439,8 @@ export class LlEmitter {
      * typeKey → thunk symbol. */
     resolveThunks = new Map();
     resolveThunkDefs = [];
+    /** Typed-ref recipe constants by type key (lazy receivers). */
+    typedRefRecipes = new Map();
     /** ReadableStream.from adapters keep typed arrays by reference and box
      * one current element per pull. */
     streamFromArrayAdapters = new Map();
@@ -530,6 +558,9 @@ export class LlEmitter {
     currentReturnType = VOID;
     /** The current body returns its number result as i32. */
     currentInt32Result = false;
+    /** The function's i32 entry returns its scalar union result as tag and
+     * payload bits (Int32Abi.union). */
+    currentUnionResult = false;
     /** Active only while emitting a body lowered with LLVM's
      * switched-coroutine intrinsics (wasm32 async bodies and generators,
      * native synchronous generators; see coroLowered). */
@@ -538,6 +569,8 @@ export class LlEmitter {
      * generator bodies): yieldExpr emission reads them, and emitTryCatch's
      * catch prologue emits the GENRET sentinel re-unwind exactly here. */
     currentGenerator = null;
+    /** The function being emitted is an async generator body. */
+    currentAsyncGenerator = false;
     /** Active optional-chain bind slots, by chain id (chainRecv reads). */
     chainSlots = new Map();
     logArgSlots = 0;
@@ -565,6 +598,7 @@ export class LlEmitter {
             options.wasi === true || ffiExtendsNarrowIntegers(options.targetTriple);
         this.wasi = options.wasi === true;
         this.stacklessGenerators = !this.wasi && mod.workers !== true && this.debug === null;
+        this.stackObjects = this.debug === null;
         // An empty triple targets the host, as for executableTls below: the
         // Node-hosted CLI passes none, the native CLI passes its toolchain's, and
         // both must emit the same module for the same target.
@@ -591,15 +625,14 @@ export class LlEmitter {
         this.runtimeAbiMarker = options.runtimeAbiMarker === true;
         this.detachedExit = options.detachedExit === true;
         this.rcHelpers = options.inlineRc === true ? new Set() : null;
-        // ScrCycHdr.buffered sits 6 bytes behind every object. The wasm32
-        // header pads before its descriptor so its payload remains
-        // double-aligned without changing this ABI offset.
-        this.cycleBufferedOffset = 6;
+        // ScrCycHdr.buf sits 4 bytes behind every object on every target.
+        this.cycleBufOffset = CYC_HDR - CYC_BUF;
         this.ffiCallbackAdapters = allocateFfiCallbackAdapters(mod.ffiImports ?? []);
         this.ffiHasRetainedCallback = hasRetainedFfiCallback(mod.ffiImports ?? []);
         this.ffiHasForeignCallback = hasForeignFfiCallback(mod.ffiImports ?? []);
         for (const fn of mod.functions)
             this.fnByName.set(fn.name, fn);
+        this.appendedArrayLiterals = findAppendedArrayLiterals(mod);
         for (const entry of mod.ffiImports ?? []) {
             this.ffiByName.set(entry.name, entry);
         }
@@ -633,6 +666,13 @@ export class LlEmitter {
         // then yields the (borrowed or owned) element pointer, not a stack box.
         // Arrays of such a union are read the same way.
         this.optionalArrayReads = new OptionalArrayReads(this.fnByName, this.unionsById, (id) => this.nullableUnions.has(id));
+        this.unionSources = {
+            unions: this.boxedUnionsById,
+            receiverUnions: this.unionsById,
+            nullableField: (e) => this.nullableFieldGet(e) !== null,
+            arrayRead: (e) => this.optionalArrayReads.get(e) !== null,
+            scalarCall: (e) => this.scalarResultCall(e),
+        };
         this.checkedNarrows = new CheckedNarrows(this.fnByName);
         this.referenceEffects = new ReferenceEffects(this.fnByName, (call) => this.optionalArrayReads.get(call) !== null, mod.classes ?? []);
         if (this.debug === null)
@@ -678,7 +718,7 @@ export class LlEmitter {
                 walkParameters.set(fn.name, indexes);
             }
         }
-        this.callLifetimes = analyzeCallLifetimes(this.fnByName, (className, field) => this.nullableFields.get(className, field) !== null, walkParameters);
+        this.callLifetimes = analyzeCallLifetimes(this.fnByName, (className, field) => this.nullableFields.get(className, field) !== null, walkParameters, new Set([...this.int32Abis].filter(([, abi]) => abi.union).map(([name]) => name)));
         this.constantCallbacks = findConstantCallbacks(mod, this.callLifetimes);
         this.stackCallbacks = new StackCallbacks(this.fnByName);
         this.receiverReaders = new AmbientReceiverReaders(this.fnByName);
@@ -691,7 +731,7 @@ export class LlEmitter {
         this.tracedUnions = traced.unions;
         this.shapeHost = {
             sizeType: this.sizeType,
-            cycleBufferedOffset: this.cycleBufferedOffset,
+            cycleBufOffset: this.cycleBufOffset,
             tracedShapes: this.tracedShapes,
             tracedUnions: this.tracedUnions,
             nullableUnions: this.nullableUnions,
@@ -733,6 +773,7 @@ export class LlEmitter {
             pendingTestLines: (dest) => this.pendingTestLines(dest),
             virtualEntry: (implFn) => this.virtualEntry(implFn),
             programLifetime: this.programLifetime,
+            regionAliasTag: (word) => this.memoryAliasAttachment(`region.${word}`),
         };
         this.walkers = new LlWalkers(this.shapeHost);
         this.dyn = new LlDyn(this.shapeHost);
@@ -770,6 +811,17 @@ export class LlEmitter {
         // whole-program preorder numbering for instanceof over runtime and
         // compiled error objects, and the per-hierarchy virtual slot lists.
         this.classMeta = buildClassGraph(mod, this.fnByName);
+        if (mod.lifetimeRegions === true) {
+            this.regions = new LifetimeRegions(this.shapeHost, mod, this.classMeta, (meta) => programLifetimeClass(this.shapeHost, meta), this.nullableFields);
+            const regions = this.regions;
+            // Member names differ from LifetimeRegions' own, which keeps the
+            // record from reading as that class's protocol.
+            this.shapeHost.regions = {
+                rcBody: (fnName, ret, comment) => regions.countBody(fnName, ret, comment),
+                newAlloc: (className, sizeOf) => regions.allocLines(className, sizeOf),
+                classDefinitions: (cls, sizeOf) => regions.classDefs(cls, sizeOf),
+            };
+        }
         for (const [name, rec] of RUNTIME_ERROR_CLASSES) {
             const meta = this.classMeta.get(name);
             if (!meta)
@@ -1402,6 +1454,7 @@ export class LlEmitter {
             ...emitClassMembershipHelper(this.classMeta, this.sizeType),
             ...(this.publishWalkersCache?.defs() ?? []),
             ...this.publishedClassNamesDefs(),
+            ...(this.regions?.moduleDefs() ?? []),
         ];
         if (this.constantNumericTables.size > 0)
             this.declare(`declare double @scr_arr_get_number(ptr, double)`);
@@ -1529,8 +1582,8 @@ export class LlEmitter {
             // half carries the precomputed Map key hash (SCR_STR_HASH_CACHE).
             // Bit 31 marks all-ASCII literals (SCR_STR_ASCII_BIT) on every target.
             const capWord = this.sizeType === "i64"
-                ? literalCapWord(Buffer.from(text, "utf8"))
-                : literalCapWord32(Buffer.from(text, "utf8"));
+                ? literalCapWord(wtf8Bytes(text))
+                : literalCapWord32(wtf8Bytes(text));
             out.push(`@${lit.sym} = internal global { ${this.sizeType}, ${this.sizeType}, ${this.sizeType}, [${lit.len + 1} x i8] } ` +
                 `{ ${this.sizeType} -1, ${this.sizeType} ${lit.len}, ${this.sizeType} ${capWord}, [${lit.len + 1} x i8] c"${llStrBytes(text)}" }`);
         }
@@ -1693,6 +1746,12 @@ export class LlEmitter {
         }
         if (this.streamIntervals.length > 0)
             out.push(``);
+        if ([...this.literals.keys(), ...this.cstrs.keys()].some(hasLoneSurrogate)) {
+            // The runtime skips surrogate work until a lone surrogate may exist
+            // (scr_wtf8_any); its own producers note one, and literals do here.
+            out.push(`declare void @scr_wtf8_note()`, ``);
+            stamps.push(`  call void @scr_wtf8_note()`);
+        }
         if (this.errorIntervals.length > 0 && this.tracedShapes.has("object:%Error")) {
             // The cycle fixpoint marked the Error hierarchy (a user subclass
             // holds cycle-capable fields — capability is hierarchy-uniform), so
@@ -2140,16 +2199,24 @@ export class LlEmitter {
             defs.push(`define internal zeroext i1 @sc_str_eq(ptr %a, ptr %b) ${FN_ATTRS} {`, `entry:`, `  %same = icmp eq ptr %a, %b`, `  br i1 %same, label %yes, label %lengths`, `lengths:`, `  %alen.p = getelementptr inbounds %ScrStr, ptr %a, i32 0, i32 1`, `  %blen.p = getelementptr inbounds %ScrStr, ptr %b, i32 0, i32 1`, `  %alen = load ${this.sizeType}, ptr %alen.p`, `  %blen = load ${this.sizeType}, ptr %blen.p`, `  %samelen = icmp eq ${this.sizeType} %alen, %blen`, `  br i1 %samelen, label %bytes, label %no`, `bytes:`, `  %eq = call zeroext i1 @scr_str_eq(ptr %a, ptr %b)`, `  ret i1 %eq`, `yes:`, `  ret i1 true`, `no:`, `  ret i1 false`, `}`, ``);
         }
         for (const [text, name] of this.literalEqualities) {
-            defs.push(...literalEqualityDefinition(name, Buffer.from(text, "utf8"), this.internLiteral(text), this.sizeType, FN_ATTRS), ``);
+            defs.push(...literalEqualityDefinition(name, wtf8Bytes(text), this.internLiteral(text), this.sizeType, FN_ATTRS), ``);
         }
         // String length and charCodeAt read proven-ASCII strings inline: bit 31
         // of the capacity word (SCR_STR_ASCII_BIT, the low half's sign bit on
         // little-endian targets) makes UTF-16 units bytes. Unproven strings and
         // anything but an in-range index take the runtime's UTF-16 mapping.
+        // A scan's receiver reads inline too where runtime state is ordinary
+        // global storage (ScrStrScan; SCR_TL makes it thread-local otherwise).
+        const scanInline = this.mod.workers !== true && this.mod.lib?.threadInstances !== true;
+        let scanDeclared = false;
         for (const helper of STRING_READ_HELPERS) {
             if (!this.decls.has(helper.decl))
                 continue;
-            defs.push(...helper.define(this.sizeType, FN_ATTRS), ``);
+            if (scanInline && !scanDeclared) {
+                defs.push(stringScanDecl(this.sizeType), ``);
+                scanDeclared = true;
+            }
+            defs.push(...helper.define(this.sizeType, FN_ATTRS, scanInline), ``);
         }
         // Ordering also decides inline when the first bytes differ: byte order
         // is the plain comparison's order, and the UTF-16 order whenever both
@@ -3197,11 +3264,13 @@ export class LlEmitter {
                     ? "ret void"
                     : this.currentInt32Result
                         ? "ret i32 0"
-                        : t === "double"
-                            ? `ret double ${f64Lit(0)}`
-                            : t === "i1"
-                                ? "ret i1 false"
-                                : "ret ptr null";
+                        : this.currentUnionResult
+                            ? `ret ${SCALAR_UNION_DUMMY}`
+                            : t === "double"
+                                ? `ret double ${f64Lit(0)}`
+                                : t === "i1"
+                                    ? "ret i1 false"
+                                    : "ret ptr null";
         }
         // Copy the entries now: later declarations extend these lexical scopes.
         // Release order and the handler are part of the shared block's identity.
@@ -4397,7 +4466,14 @@ export class LlEmitter {
             this.debug === null ? findScalarStringSlices(fn, this.callLifetimes) : new Map();
         this.mapReadLifetimes = findMapReadLifetimes(fn, this.boxedUnionsById, this.callLifetimes);
         this.localStackUnions = findLocalStackUnions(fn, this.callLifetimes, this.boxedUnionsById);
-        this.localUnionStorageProofs = findLocalUnionStorage(fn, this.callLifetimes, this.boxedUnionsById, (e) => this.nullableFieldGet(e) !== null);
+        this.localStackObjects = this.stackObjects
+            ? findLocalStackObjects(this, fn, this.callLifetimes)
+            : new Map();
+        this.reassignedStackObjects = this.stackObjects
+            ? findReassignedStackObjects(this, fn, this.callLifetimes)
+            : new Set();
+        this.stackObjectSlots = new Map();
+        this.localUnionStorageProofs = findLocalUnionStorage(fn, this.callLifetimes, this.unionSources);
         this.localUnionStorage = new Map();
         this.walkBorrows = this.walkBorrowsByFunction.get(fn.name) ?? new Set();
         this.integerRanges = analyzeIntegerRanges(numericFn, this.int32Slots.facts(fn.name));
@@ -4417,7 +4493,9 @@ export class LlEmitter {
         this.currentReturnType = fn.returnType;
         const int32Abi = this.int32Abis.get(fn.name);
         this.currentInt32Result = int32Abi?.result === true;
+        this.currentUnionResult = int32Abi?.union === true;
         this.currentGenerator = fn.generator ?? null;
+        this.currentAsyncGenerator = fn.async === true && fn.generator !== undefined;
         this.currentBorrowedReturn = this.borrowedReturns.has(fn.name);
         this.inConstructor = this.constructorFunctions.has(fn.name);
         this.currentCoro = null;
@@ -4604,6 +4682,9 @@ export class LlEmitter {
         // whose unwind released everything down to depth 0).
         if (fn.returnType.kind === "void" && !B.isTerminated()) {
             this.releaseScope(this.scopes[0]);
+            // Completing by falling off the end has nothing to await.
+            if (this.currentAsyncGenerator)
+                this.emitAsyncGenSettled();
             if (this.currentCoro !== null) {
                 this.emitWasiFulfill(null);
                 B.terminate(`br label %${this.currentCoro.finalLabel}`);
@@ -4681,7 +4762,11 @@ export class LlEmitter {
         // Lifted functions receive their closure first (the callValue ABI).
         if (fn.captures !== undefined)
             params.unshift("ptr %sc_env");
-        const ret = int32Abi?.result ? "i32" : this.llType(fn.returnType);
+        const ret = int32Abi?.result
+            ? "i32"
+            : int32Abi?.union
+                ? SCALAR_UNION_RESULT
+                : this.llType(fn.returnType);
         let attrs = coro !== null ? "#1" : FN_ATTRS;
         // The shared indexed-read helper (lowerSafeIndexRead) is a dense bounds
         // test, a state test and a load behind cold runtime fallbacks, called
@@ -4721,7 +4806,19 @@ export class LlEmitter {
                 args.push(`${type} %p${index}`);
         });
         const ret = this.llType(fn.returnType);
-        const call = `call ${abi.result ? "i32" : ret} @${symbol}${INT32_ABI_SUFFIX}(${args.join(", ")})`;
+        const entry = `@${symbol}${INT32_ABI_SUFFIX}(${args.join(", ")})`;
+        if (abi.union) {
+            // Box the register pair for every caller of the ordinary ABI.
+            const B = new BlockBuilder();
+            this.B = B;
+            for (const line of lines)
+                B.line(line.trim());
+            const pair = B.tmp();
+            B.line(`${pair} = call ${SCALAR_UNION_RESULT} ${entry}`);
+            B.terminate(`ret ptr ${this.boxScalarUnion(pair, fn.returnType)}`);
+            return `define internal ptr @${symbol}(${params.join(", ")}) ${FN_ATTRS} {\n${B.render()}\n}`;
+        }
+        const call = `call ${abi.result ? "i32" : ret} ${entry}`;
         if (ret === "void")
             lines.push(`  ${call}`, "  ret void");
         else if (abi.result)
@@ -4730,6 +4827,73 @@ export class LlEmitter {
             lines.push(`  %r = ${call}`, `  ret ${ret} %r`);
         return `define internal ${ret} @${symbol}(${params.join(", ")}) ${FN_ATTRS} {\nentry:\n${lines.join("\n")}\n}`;
     }
+    /** The SCALAR_UNION_RESULT pair of a `%ScrUnion` box of `type`. */
+    scalarUnionPair(box, type) {
+        const B = this.B;
+        const unionId = type.kind === "union" ? type.unionId : "";
+        const tag = this.unionTag(box, unionId);
+        const payload = B.tmp(), bits = B.tmp(), partial = B.tmp(), pair = B.tmp();
+        B.line(`${payload} = getelementptr inbounds %ScrUnion, ptr ${box}, i32 0, i32 5`);
+        B.line(`${bits} = load i64, ptr ${payload}`);
+        B.line(`${partial} = insertvalue ${SCALAR_UNION_RESULT} poison, i32 ${tag}, 0`);
+        B.line(`${pair} = insertvalue ${SCALAR_UNION_RESULT} ${partial}, i64 ${bits}, 1`);
+        return pair;
+    }
+    /** A private stack box holding a SCALAR_UNION_RESULT pair of `type`. */
+    stackScalarUnion(pair, type) {
+        const B = this.B;
+        const box = B.slot(), tagSlot = B.slot(), payloadSlot = B.slot();
+        B.entryAllocas.push(`${box} = alloca %ScrUnion`);
+        B.entryAllocas.push(`${tagSlot} = getelementptr inbounds %ScrUnion, ptr ${box}, i32 0, i32 1`);
+        B.entryAllocas.push(`${payloadSlot} = getelementptr inbounds %ScrUnion, ptr ${box}, i32 0, i32 5`);
+        const tag = B.tmp(), bits = B.tmp();
+        B.line(`${tag} = extractvalue ${SCALAR_UNION_RESULT} ${pair}, 0`);
+        B.line(`${bits} = extractvalue ${SCALAR_UNION_RESULT} ${pair}, 1`);
+        B.line(`store i32 ${tag}, ptr ${tagSlot}`);
+        B.line(`store i64 ${bits}, ptr ${payloadSlot}`);
+        return { name: box, type };
+    }
+    /** A direct call whose i32 entry returns its scalar union in registers. */
+    scalarResultCall(e) {
+        return e.kind === "call" && this.int32Abis.get(e.callee)?.union === true;
+    }
+    /** A fresh heap box (+1) for a SCALAR_UNION_RESULT pair of `type`; the
+     * exception dummy (tag -1) answers NULL. */
+    boxScalarUnion(pair, type) {
+        const B = this.B;
+        const unionId = type.kind === "union" ? type.unionId : "";
+        const arms = this.unionsById.get(unionId).arms;
+        const tag = B.tmp(), bits = B.tmp();
+        B.line(`${tag} = extractvalue ${SCALAR_UNION_RESULT} ${pair}, 0`);
+        B.line(`${bits} = extractvalue ${SCALAR_UNION_RESULT} ${pair}, 1`);
+        const join = B.newLabel("union.box.join"), none = B.newLabel("union.box.none");
+        const labels = arms.map(() => B.newLabel("union.box.arm"));
+        B.terminate(`switch i32 ${tag}, label %${none} [ ${labels.map((label, armTag) => `i32 ${armTag}, label %${label}`).join(" ")} ]`);
+        const incoming = [];
+        arms.forEach((arm, armTag) => {
+            B.startBlock(labels[armTag]);
+            let value;
+            if (isUnitType(arm))
+                value = this.unitInstanceRef(unionId, armTag);
+            else {
+                const payload = B.tmp();
+                if (arm.kind === "bool")
+                    B.line(`${payload} = trunc i64 ${bits} to i1`);
+                else
+                    B.line(`${payload} = bitcast i64 ${bits} to double`);
+                value = this.unionNewOwned(unionId, armTag, { name: payload, type: arm });
+            }
+            incoming.push(`[ ${value}, %${B.currentLabel()} ]`);
+            B.br(join);
+        });
+        B.startBlock(none);
+        incoming.push(`[ null, %${none} ]`);
+        B.br(join);
+        B.startBlock(join);
+        const box = B.tmp();
+        B.line(`${box} = phi ptr ${incoming.join(", ")}`);
+        return box;
+    }
     /** The i32 bits of a number proven an int32 (a direct call's i32
      * argument or an i32 result): its exact integer when the value carries
      * one, else the conversion the proof makes exact. */
@@ -5000,6 +5164,12 @@ export class LlEmitter {
         this.finallyStack = savedFinally;
         this.tryStack = savedTry;
     }
+    /** An async generator completing without a further turn: its request
+     * settles as the body returns (scr_async_gen_hop_done). */
+    emitAsyncGenSettled() {
+        this.declare(`declare void @scr_async_gen_hop_done()`);
+        this.B.line(`call void @scr_async_gen_hop_done()`);
+    }
     /** `return x` of a local whose slot is an owned scope entry of the same
      * type: the return can take over the local's reference. */
     movableReturn(value) {
@@ -5200,6 +5370,17 @@ export class LlEmitter {
                     this.scopes[this.scopes.length - 1].push({ slot: snapshot.sourceSlot, type: STRING }, { slot: snapshot.separatorSlot, type: STRING });
                     break;
                 }
+                const stackObject = this.localStackObjects.get(s.localId);
+                if (stackObject) {
+                    B.line(`store ptr ${emitStackObject(this, stackObject).name}, ptr ${b.slot}`);
+                    break;
+                }
+                if (s.init && this.reassignedStackObjects.has(s.localId)) {
+                    const slot = this.stackObjectSlot(s.localId, s.init.type);
+                    emitStackObject(this, s.init, slot);
+                    B.line(`store ptr ${slot}, ptr ${b.slot}`);
+                    break;
+                }
                 const localUnion = this.localStackUnions.get(s.localId);
                 if (localUnion) {
                     const union = emitStackUnion(this, localUnion);
@@ -5347,6 +5528,15 @@ export class LlEmitter {
                     B.line(`store ptr ${v.name}, ptr ${owner}`);
                     break;
                 }
+                if (this.reassignedStackObjects.has(s.localId)) {
+                    // The new value may read the old one: build it apart, then copy.
+                    const struct = stackObjectStruct(s.value.type);
+                    const value = emitStackObject(this, s.value);
+                    const bits = B.tmp();
+                    B.line(`${bits} = load ${struct}, ptr ${value.name}`);
+                    B.line(`store ${struct} ${bits}, ptr ${this.stackObjectSlot(s.localId, s.value.type)}`);
+                    break;
+                }
                 const localStorage = this.localUnionStorage.get(s.localId);
                 if (localStorage) {
                     storeLocalUnion(this, localStorage, s.value);
@@ -6199,8 +6389,16 @@ export class LlEmitter {
                     break;
                 }
                 let v = null;
-                const moved = this.finallyStack.length === 0 ? this.movableReturn(s.value) : null;
-                if (moved !== null) {
+                const moved = this.finallyStack.length === 0 && !this.currentUnionResult
+                    ? this.movableReturn(s.value)
+                    : null;
+                if (s.value !== null && this.currentUnionResult) {
+                    // The tag and payload bits leave in registers; a heap box the
+                    // value needed is released with its frame below.
+                    const box = this.emitUnionProjection(s.value);
+                    v = { name: this.scalarUnionPair(box.name, s.value.type), type: s.value.type };
+                }
+                else if (moved !== null) {
                     // Returning an owned local hands its reference to the caller: no
                     // retain now and no release of the local below.
                     const t = B.tmp();
@@ -6230,9 +6428,22 @@ export class LlEmitter {
                     v = this.emitExpr(s.value);
                     this.moveTemp(v);
                 }
+                // An async generator's `return v` awaits v before any crossed
+                // finally runs. Past finally regions that turn is taken here and
+                // the request settles at completion; otherwise the completion's
+                // own turn is the await (nothing runs in between).
+                const genSettles = this.currentAsyncGenerator && (s.awaited === true || this.finallyStack.length > 0);
+                if (this.currentAsyncGenerator && s.awaited !== true && this.finallyStack.length > 0) {
+                    if (this.wasi)
+                        this.emitWasiSuspend(null);
+                    else {
+                        this.declare(`declare void @scr_await_hop()`);
+                        B.line(`call void @scr_await_hop()`);
+                    }
+                }
                 if (this.finallyStack.length > 0) {
                     let pretSlot = null;
-                    if (v !== null && isRefCounted(v.type)) {
+                    if (v !== null && isRefCounted(v.type) && !this.currentUnionResult) {
                         pretSlot = B.slot();
                         B.entryAllocas.push(`${pretSlot} = alloca ptr ; pending return (through finally)`);
                         B.line(`store ptr ${v.name}, ptr ${pretSlot}`);
@@ -6265,6 +6476,8 @@ export class LlEmitter {
                     this.releaseForJump(0, 0);
                 }
                 if (!B.isTerminated()) {
+                    if (genSettles)
+                        this.emitAsyncGenSettled();
                     if (this.currentCoro !== null) {
                         this.emitWasiFulfill(v);
                         B.terminate(`br label %${this.currentCoro.finalLabel}`);
@@ -6273,6 +6486,8 @@ export class LlEmitter {
                         B.terminate("ret void");
                     else if (this.currentInt32Result)
                         B.terminate(`ret i32 ${this.int32Argument(v, s.value)}`);
+                    else if (this.currentUnionResult)
+                        B.terminate(`ret ${SCALAR_UNION_RESULT} ${v.name}`);
                     else
                         B.terminate(`ret ${this.llType(s.value.type)} ${v.name}`);
                 }
@@ -6787,8 +7002,45 @@ export class LlEmitter {
     emitIntrinsicExpr(e) {
         if (e.name === "threads.publish")
             return this.emitPublish(e);
+        if (e.name === "region.create" ||
+            e.name === "region.enter" ||
+            e.name === "region.leave" ||
+            e.name === "region.release")
+            return this.emitRegionIntrinsic(e);
         return emitIntrinsicExpr(this, e);
     }
+    /** Lifetime regions in modules with lifetimeRegions (llvm/regions.ts).
+     * Created with the class graph; null elsewhere. */
+    regions = null;
+    /** LifetimeRegion's runtime calls: create answers the region id, enter
+     * and leave bracket run(), release condemns. */
+    emitRegionIntrinsic(e) {
+        if (this.regions === null)
+            throw new InternalCompilerError(`llvm emitter bug: ${e.name} without lifetimeRegions`);
+        const args = e.args.map((a) => this.emitExpr(a));
+        if (this.B.isTerminated())
+            return { name: "", type: e.type };
+        switch (e.name) {
+            case "region.create": {
+                this.declare(`declare double @scr_region_create(ptr)`);
+                const id = this.B.tmp();
+                this.B.line(`${id} = call double @scr_region_create(ptr ${REGION_TYPES})`);
+                return { name: id, type: e.type };
+            }
+            case "region.enter":
+                this.declare(`declare void @scr_region_enter(double)`);
+                this.B.line(`call void @scr_region_enter(double ${args[0].name})`);
+                break;
+            case "region.leave":
+                this.declare(`declare void @scr_region_leave()`);
+                this.B.line(`call void @scr_region_leave()`);
+                break;
+            default:
+                this.declare(`declare void @scr_region_release(double)`);
+                this.B.line(`call void @scr_region_release(double ${args[0].name})`);
+        }
+        return { name: "", type: e.type };
+    }
     publishWalkersCache = null;
     /** Per-type publication walkers, created on the first publish. */
     get publishWalkers() {
@@ -7075,10 +7327,30 @@ export class LlEmitter {
             B.line(`${value} = load ptr, ptr ${slot}`);
             return { name: value, type: e.type };
         }
+        // Scalar optional values and register results answer a private stack
+        // box. A ternary of borrowable arms above keeps lending its arm.
+        if ((e.kind === "optChain" ||
+            e.kind === "nullish" ||
+            e.kind === "ternary" ||
+            e.kind === "arrIntrinsic") &&
+            isScalarOptionalSource(e, this.unionSources))
+            return emitScalarOptional(this, e);
+        if (e.kind === "call" && this.scalarResultCall(e))
+            return emitCallExpr(this, e, false, true);
         if (e.kind === "call" && this.canBorrowReceiver(e))
             return emitBorrowedResultCall(this, e);
         return this.emitExpr(e);
     }
+    /** The frame slot a reassigned stack-object local keeps for its value. */
+    stackObjectSlot(localId, type) {
+        let slot = this.stackObjectSlots.get(localId);
+        if (slot === undefined) {
+            slot = this.B.slot();
+            this.B.entryAllocas.push(`${slot} = alloca ${stackObjectStruct(type)}, align 8`);
+            this.stackObjectSlots.set(localId, slot);
+        }
+        return slot;
+    }
     /** A private stack union source with an independently owned payload. */
     canStackReceiver(e) {
         return this.isStackUnionSource(e);
@@ -7090,7 +7362,9 @@ export class LlEmitter {
             return false;
         return (canStackUnion(e, this.boxedUnionsById) ||
             matchMapRead(e, this.boxedUnionsById) !== null ||
-            this.optionalArrayReads.get(e) !== null);
+            this.optionalArrayReads.get(e) !== null ||
+            isScalarOptionalSource(e, this.unionSources) ||
+            this.scalarResultCall(e));
     }
     /** The nullable-pointer representation behind a union-typed class field
      * read, or null for every other expression. */