@scriptc/compiler 0.3.6 → 0.4.1

package/dist/backend/llvm/shapes.js

@@ -1,9 +1,73 @@
-import { isIdentityCollectionKey, isRefCounted, mapOf, POINTER_KINDS, runtimeRcStem, RUNTIME_EMITTER_CLASS, RUNTIME_EVENT_CLASS, RUNTIME_EVENT_TARGET_CLASS, RUNTIME_ERROR_CLASSES, RUNTIME_STREAM_CLASSES, STRING, } from "../../ir/ir.js";
+import { isIdentityCollectionKey, isRefCounted, mapOf, POINTER_KINDS, runtimeRcStem, RUNTIME_EMITTER_CLASS, RUNTIME_EVENT_CLASS, RUNTIME_EVENT_TARGET_CLASS, RUNTIME_ERROR_CLASSES, RUNTIME_STREAM_CLASSES, DYN, STRING, } from "../../ir/ir.js";
 import { mangleClassRelease, mangleClassRetain, mangleClassTrace, mangleRecordGcFree, mangleRecordClone, mangleRecordNew, mangleRecordRelease, mangleRecordRetain, mangleRecordStruct, mangleRecordTrace, } from "../mangle.js";
-import { cycTypeDescriptor, emitObjectAlloc, emitObjectFree } from "./alloc.js";
+import { CYC_HDR, cycTypeDescriptor, emitObjectAlloc, emitObjectFree, CYC_APPEND, cycleAppendDecls, cycleAppendHelper, inlineBufferingSupported, } from "./alloc.js";
 import { SCR_ARR_FIELD, llvmCommentText } from "./common.js";
+import { NULLABLE_NULL, SHAPE_WORD_INDEX, shapeReleaseFn, shapeTagFn, shapeTraceFn, shapeWordHeader, } from "./nullable-unions.js";
 import { INLINE_UNION_FIELD, loadInlineUnion, stackUnionBox, } from "./inline-unions.js";
+import { JS_BITS, JS_TAGS } from "./jsval-encoding.js";
+import { recordMayKeepSource } from "./record-sources.js";
 import { LlvmUnsupportedError } from "./unsupported.js";
+/** SCR_CYC_MAX_TYPES (scr_rc.h): the registry's size, and the length of
+ * the young-in nursery's per-id skip countdown. */
+const CYC_MAX_TYPES = 32768;
+/** The nursery entry of a new traced instance in `%o`, after its count is
+ * stored; nothing when the nursery is off. Where the allocator is inlined
+ * (64-bit, process-global state) the entry is inline too: a type that is
+ * skipping the nursery (its scr_yin_skip countdown is nonzero) counts down
+ * and stays out; otherwise, while the nursery has room (n < lim in ScrYin,
+ * scr_rc.h), list the object at index n and mark its header young
+ * (generation SCR_CYC_YOUNG, `buf` the buffered bit and the index), and
+ * call scr_yin_enter_slow when it has none (it collects the nursery
+ * first). Other targets call scr_yin_enter, which does all of it.
+ * Labels use the `ye.` prefix; the lines end in an open block. */
+export function youngEntryLines(host) {
+    if (host.youngNursery !== true)
+        return [];
+    if (host.inlineAlloc !== true || host.threadLocalAlloc === true) {
+        host.declare(`declare void @scr_yin_enter(ptr)`);
+        return [`  call void @scr_yin_enter(ptr %o) ; the young-in nursery`];
+    }
+    host.declare(`@scr_yin = external global { ptr, i64, i64, i64 }`);
+    host.declare(`@scr_yin_skip = external global [${CYC_MAX_TYPES} x i32]`);
+    host.declare(`declare void @scr_yin_enter_slow(ptr)`);
+    return [
+        `  %ye.idp = getelementptr inbounds i8, ptr %o, i64 -${CYC_HDR} ; the young-in nursery`,
+        `  %ye.id16 = load i16, ptr %ye.idp ; ScrCycHdr.id`,
+        `  %ye.id = zext i16 %ye.id16 to i64`,
+        `  %ye.sp = getelementptr inbounds [${CYC_MAX_TYPES} x i32], ptr @scr_yin_skip, i64 0, i64 %ye.id`,
+        `  %ye.s = load i32, ptr %ye.sp`,
+        `  %ye.skip = icmp ne i32 %ye.s, 0`,
+        `  br i1 %ye.skip, label %ye.skipped, label %ye.enter`,
+        `ye.skipped: ; a type that does not die young: counted down, stays out`,
+        `  %ye.s1 = sub i32 %ye.s, 1`,
+        `  store i32 %ye.s1, ptr %ye.sp`,
+        `  br label %ye.done`,
+        `ye.enter:`,
+        `  %ye.v = load ptr, ptr @scr_yin`,
+        `  %ye.np = getelementptr inbounds i8, ptr @scr_yin, i64 8`,
+        `  %ye.n = load i64, ptr %ye.np`,
+        `  %ye.lp = getelementptr inbounds i8, ptr @scr_yin, i64 16`,
+        `  %ye.l = load i64, ptr %ye.lp`,
+        `  %ye.room = icmp ult i64 %ye.n, %ye.l`,
+        `  br i1 %ye.room, label %ye.fast, label %ye.slow`,
+        `ye.fast:`,
+        `  %ye.slot = getelementptr inbounds ptr, ptr %ye.v, i64 %ye.n`,
+        `  store ptr %o, ptr %ye.slot`,
+        `  %ye.n1 = add i64 %ye.n, 1`,
+        `  store i64 %ye.n1, ptr %ye.np`,
+        `  %ye.genp = getelementptr inbounds i8, ptr %o, i64 -5`,
+        `  store i8 3, ptr %ye.genp ; ScrCycHdr.gen = SCR_CYC_YOUNG`,
+        `  %ye.bufp = getelementptr inbounds i8, ptr %o, i64 -4`,
+        `  %ye.i = trunc i64 %ye.n to i32`,
+        `  %ye.buf = or i32 %ye.i, -2147483648`,
+        `  store i32 %ye.buf, ptr %ye.bufp ; ScrCycHdr.buf = buffered bit | index`,
+        `  br label %ye.done`,
+        `ye.slow:`,
+        `  call void @scr_yin_enter_slow(ptr %o)`,
+        `  br label %ye.done`,
+        `ye.done:`,
+    ];
+}
 /** Every emitted function/helper carries #0 = { sanitize_address } — see
  * the emitter header. */
 export const FN_ATTRS = "#0";
@@ -18,6 +82,8 @@ export { computeTraced } from "../cycle-analysis.js";
  * `_v`-shaped. */
 export function vAdapters(host, t) {
     const nullable = host.nullableUnions.of(t);
+    if (nullable?.repr === "shapes")
+        return shapeUnionRc(nullable);
     if (nullable) {
         const arm = vAdapters(host, nullable.arm);
         return { retain: nullableRetainSym(host, nullable.arm), release: arm.release };
@@ -70,7 +136,11 @@ export function vAdapters(host, t) {
  * table above; the split exists so call sites read type-directedly.
  * Families with a mirrored fast path call the inline helper instead. */
 export function retainSym(host, t) {
+    if (t.kind === "jsval")
+        return jsvalRcSym(host, "retain");
     const nullable = host.nullableUnions.of(t);
+    if (nullable?.repr === "shapes")
+        return shapeUnionRc(nullable).retain;
     if (nullable)
         return nullableRetainSym(host, nullable.arm);
     const stem = runtimeRcStem(t);
@@ -83,6 +153,53 @@ export function retainSym(host, t) {
         return `@${mangleClassRetain(t.className)}`;
     return vAdapters(host, t).retain;
 }
+/** A shape union's RC entry points: every record retain is the same
+ * NULL- and immortal-skipping increment (the null sentinel is immortal),
+ * so the first arm's serves; release dispatches on the arm. */
+function shapeUnionRc(union) {
+    return {
+        retain: `@${mangleRecordRetain(union.arms[0].shapeId)}`,
+        release: shapeReleaseFn(union.unionId),
+    };
+}
+/** The tag, release and trace helpers of every shape union
+ * (nullable-unions.ts). The tag helper maps a value to its arm: NULL and
+ * the null sentinel are the unit arms, anything else a tagged record whose
+ * shape word names its arm (the switch folds to a lookup table). */
+export function emitShapeUnionHelpers(host) {
+    const out = [];
+    const S = host.sizeType;
+    for (const union of host.nullableUnions.allShapeUnions()) {
+        const header = shapeWordHeader(S);
+        const tagFn = shapeTagFn(union.unionId);
+        const cases = union.arms
+            .map((arm) => `i32 ${host.nullableUnions.shapeNumber(arm.shapeId)}, label %a${arm.tag}`)
+            .join(" ");
+        out.push(`define internal i32 ${tagFn}(ptr %u) alwaysinline ${FN_ATTRS} { ; shape union tag`, `entry:`, `  %isnull = icmp eq ptr %u, null`, `  br i1 %isnull, label %unit, label %ref`, `unit:`, `  ret i32 ${union.unitTag}`, `ref:`);
+        if (union.nullTag >= 0)
+            out.push(`  %isnullarm = icmp eq ptr %u, ${NULLABLE_NULL}`, `  br i1 %isnullarm, label %nullarm, label %rec`, `nullarm:`, `  ret i32 ${union.nullTag}`, `rec:`);
+        out.push(`  %wp = getelementptr inbounds ${header}, ptr %u, i32 0, i32 ${SHAPE_WORD_INDEX}`, `  %w = load i32, ptr %wp`, `  switch i32 %w, label %bad [ ${cases} ]`, ...union.arms.flatMap((arm) => [`a${arm.tag}:`, `  ret i32 ${arm.tag}`]), `bad:`, `  ret i32 -1`, `}`, ``);
+        // release: unit encodings are NULL or immortal; each arm releases
+        // through its own (NULL- and immortal-tolerant) helper.
+        out.push(`define internal void ${shapeReleaseFn(union.unionId)}(ptr %u) ${FN_ATTRS} { ; shape union release`, `entry:`, `  %t = call i32 ${tagFn}(ptr %u)`, `  switch i32 %t, label %done [ ${union.arms.map((arm) => `i32 ${arm.tag}, label %a${arm.tag}`).join(" ")} ]`, ...union.arms.flatMap((arm) => [
+            `a${arm.tag}:`,
+            `  call void @${mangleRecordRelease(arm.shapeId)}(ptr %u)`,
+            `  br label %done`,
+        ]), `done:`, `  ret void`, `}`, ``);
+        if (shapeUnionTraced(host, union))
+            out.push(`define internal void ${shapeTraceFn(union.unionId)}(ptr %u, ptr %visit, ptr %ctx) ${FN_ATTRS} { ; shape union trace`, `entry:`, `  %t = call i32 ${tagFn}(ptr %u)`, `  switch i32 %t, label %done [ ${union.arms.map((arm) => `i32 ${arm.tag}, label %a${arm.tag}`).join(" ")} ]`, ...union.arms.flatMap((arm) => [
+                `a${arm.tag}:`,
+                `  call void @${mangleRecordTrace(arm.shapeId)}(ptr %u, ptr %visit, ptr %ctx)`,
+                `  br label %done`,
+            ]), `done:`, `  ret void`, `}`, ``);
+    }
+    return out;
+}
+/** A shape union's arms carry collector headers (all or none: the emitter
+ * traces every arm of a union with one traced arm). */
+function shapeUnionTraced(host, union) {
+    return union.arms.some((arm) => host.tracedShapes.has(`record:${arm.shapeId}`));
+}
 /** The retain entry point of a nullable union's value: the arm's own when
  * it tolerates NULL (emitted class and record helpers), otherwise a
  * NULL-skipping wrapper around it (the null sentinel and literals are
@@ -107,7 +224,11 @@ export function emitNullableRetainWrappers(host) {
  * points serve where one exists; records use their emitted helper.
  * Families with a mirrored fast path call the inline helper instead. */
 export function releaseSym(host, t) {
+    if (t.kind === "jsval")
+        return jsvalRcSym(host, "release");
     const nullable = host.nullableUnions.of(t);
+    if (nullable?.repr === "shapes")
+        return shapeUnionRc(nullable).release;
     if (nullable)
         return releaseSym(host, nullable.arm);
     const stem = runtimeRcStem(t);
@@ -130,6 +251,67 @@ export function releaseSym(host, t) {
             throw new LlvmUnsupportedError(`rc:${t.kind}`);
     }
 }
+/** jsval retain/release (llvm/jsval.ts). On 32-bit targets a jsval is a
+ * dyn cell and uses dyn's entry points. Otherwise the inline helper skips
+ * immediates and forwards a cell to dyn's own entry point (requested here,
+ * so it is emitted with the other helpers) and a string or BigInt to the
+ * runtime; without inline RC every retain/release is a runtime call. */
+function jsvalRcSym(host, op) {
+    if (host.sizeType === "i32")
+        return op === "retain" ? retainSym(host, DYN) : releaseSym(host, DYN);
+    if (host.rcHelpers === null || mirroredFamily(host, "js", op) === null) {
+        const sym = op === "retain" ? "@scr_js_retain" : "@scr_js_release";
+        host.declare(op === "retain" ? `declare ptr ${sym}(ptr)` : `declare void ${sym}(ptr)`);
+        return sym;
+    }
+    if (op === "retain")
+        retainSym(host, DYN);
+    else
+        releaseSym(host, DYN);
+    return requestRcHelper(host, "js", op);
+}
+/** The inline retain/release helpers of jsval (64-bit targets). A cell
+ * goes through dyn's own retain/release entry point (inline or runtime);
+ * a string or BigInt through the runtime; immediates do nothing. */
+function jsvalRcHelper(host, op, attrs) {
+    const dynOp = op === "retain" ? retainSym(host, DYN) : releaseSym(host, DYN);
+    const runtime = op === "retain" ? "@scr_js_retain" : "@scr_js_release";
+    // Always inlined (the helper is a tag dispatch): LLVM would otherwise
+    // weigh it together with the dyn helper it calls and keep both out of
+    // line. dyn's release tolerates NULL; its retain does not.
+    const inline = host.rcHelpersNoinline === true ? attrs : `alwaysinline ${attrs}`;
+    return [
+        `define internal ${op === "retain" ? "ptr" : "void"} @sc_rc_${op}_js(ptr %o) ${inline} { ; jsval ${op}: cells, strings and BigInts`,
+        `entry:`,
+        `  %b = ptrtoint ptr %o to i64`,
+        `  %tag = lshr i64 %b, 48`,
+        `  %cell = icmp eq i64 %tag, ${JS_TAGS.cell}`,
+        `  br i1 %cell, label %c, label %other`,
+        `c:`,
+        `  %a = and i64 %b, ${JS_BITS.payloadMask}`,
+        `  %p = inttoptr i64 %a to ptr`,
+        ...(op === "retain"
+            ? [
+                `  %live = icmp ne ptr %p, null`,
+                `  br i1 %live, label %rc, label %done`,
+                `rc:`,
+                `  %r = call ptr ${dynOp}(ptr %p)`,
+            ]
+            : [`  call void ${dynOp}(ptr %p)`]),
+        `  br label %done`,
+        `other:`,
+        `  %h = sub i64 %tag, ${JS_TAGS.str}`,
+        `  %heap = icmp ult i64 %h, 2`,
+        `  br i1 %heap, label %slow, label %done`,
+        `slow:`,
+        op === "retain" ? `  %s = call ptr ${runtime}(ptr %o)` : `  call void ${runtime}(ptr %o)`,
+        `  br label %done`,
+        `done:`,
+        `  ${op === "retain" ? "ret ptr %o" : "ret void"}`,
+        `}`,
+        ``,
+    ];
+}
 /** Runtime RC families whose fast paths the backend mirrors in IR, keyed
  * by the family name without its `scr_` stem prefix. Each
  * row restates the C entry points in scr_runtime.h and the runtime TUs:
@@ -162,12 +344,24 @@ export const INLINE_RC_FAMILIES = {
     box: { cycle: "always", retainNull: false, release: "scr_box_release" },
     dyn: { cycle: "always", retainNull: false, release: "scr_dyn_release" },
 };
+/** The helper key (with the "family:op" keys) of the module's candidate
+ * append helper (alloc.ts cycleAppendHelper). */
+const CYC_APPEND_KEY = "cyc:append";
 /** The mirrored family for an RC operation on a runtime stem (`scr_str`),
  * or null when it stays a call. */
 export function inlineRcFamily(host, stem, op) {
     return mirroredFamily(host, stem.replace(/^scr_/, ""), op);
 }
-const RELEASE_RC_OPS = new Set(["str:retain", "dyn:retain", "dyn:release"]);
+const RELEASE_RC_OPS = new Set([
+    "str:retain",
+    "dyn:retain",
+    "dyn:release",
+    "js:retain",
+    "js:release",
+]);
+/** The jsval helpers' row: their bodies are custom (jsvalRcHelper), and
+ * they call the runtime themselves. */
+const JS_RC_ROW = { cycle: "none", retainNull: true, release: null };
 /** The emitter's inline RC option for a native optimization posture: none
  * at `dev`, the `release` subset by default. Sanitized builds compile the
  * whole program in one AddressSanitizer pass, whose time grows with every
@@ -181,6 +375,8 @@ export function inlineRcOption(optimization, sanitize = false) {
     };
 }
 function mirroredFamily(host, family, op) {
+    if (family === "js")
+        return host.rcMode !== "speed" && !RELEASE_RC_OPS.has(`js:${op}`) ? null : JS_RC_ROW;
     if (!Object.hasOwn(INLINE_RC_FAMILIES, family))
         return null;
     if (host.rcMode !== "speed" && !RELEASE_RC_OPS.has(`${family}:${op}`))
@@ -205,12 +401,19 @@ export function boxReleaseSym(host) {
 /** Record the helper and declare the runtime symbols it calls now, so they
  * land in the extern block. */
 function requestRcHelper(host, family, op) {
-    if (host.rcHelpers === null || mirroredFamily(host, family, op) === null) {
+    const row = host.rcHelpers === null ? null : mirroredFamily(host, family, op);
+    if (host.rcHelpers === null || row === null) {
         throw new LlvmUnsupportedError(`inline rc:${family}:${op}`);
     }
     host.rcHelpers.add(`${family}:${op}`);
     for (const decl of rcHelperDecls(family, op))
         host.declare(decl);
+    // Its body is written last (emitInlineRcHelpers), after the requests are
+    // collected: a release that can buffer a candidate requests the append
+    // helper now. The jsval row (JS_RC_ROW) has no collector header; its
+    // cells release through dyn's helper, which makes its own request.
+    if (op === "release" && row.cycle === "always")
+        requestCycleAppend(host);
     return `@${inlineRcName(family, op)}`;
 }
 function rcHelperAttrs(host) {
@@ -275,6 +478,10 @@ export function limitInlineDynReleases(parts) {
     });
 }
 function rcHelperDecls(family, op) {
+    if (family === "js")
+        return [
+            op === "retain" ? `declare ptr @scr_js_retain(ptr)` : `declare void @scr_js_release(ptr)`,
+        ];
     const row = INLINE_RC_FAMILIES[family];
     if (op === "retain" || row.release === null)
         return [];
@@ -298,6 +505,11 @@ function rcHelperKeys(host) {
 export function inlineRcDecls(host) {
     const decls = new Set();
     for (const key of rcHelperKeys(host)) {
+        if (key === CYC_APPEND_KEY) {
+            for (const decl of cycleAppendDecls())
+                decls.add(decl);
+            continue;
+        }
         const [family, op] = key.split(":");
         for (const decl of rcHelperDecls(family, op))
             decls.add(decl);
@@ -440,7 +652,15 @@ function releaseHelper(host, family, row) {
 export function emitInlineRcHelpers(host) {
     const out = [];
     for (const key of rcHelperKeys(host).sort()) {
+        if (key === CYC_APPEND_KEY) {
+            out.push(...cycleAppendHelper());
+            continue;
+        }
         const [family, op] = key.split(":");
+        if (family === "js") {
+            out.push(...jsvalRcHelper(host, op, rcHelperAttrs(host)));
+            continue;
+        }
         const row = INLINE_RC_FAMILIES[family];
         out.push(...(op === "retain" ? retainHelper(host, family, row) : releaseHelper(host, family, row)));
     }
@@ -469,6 +689,8 @@ export function traceAdapter(host, t) {
             return "@scr_promise_trace_v";
         case "union": {
             const nullable = host.nullableUnions.get(t.unionId);
+            if (nullable?.repr === "shapes")
+                return shapeUnionTraced(host, nullable) ? shapeTraceFn(nullable.unionId) : null;
             if (nullable)
                 return traceAdapter(host, nullable.arm);
             if (!host.tracedUnions.has(t.unionId))
@@ -588,6 +810,7 @@ small = false) {
         elem.kind === "promise" || // promise entries (Promise.all inputs): full REF story
         elem.kind === "child" || // spawned child handles: scr_child_* adapters, no trace
         elem.kind === "netServer" ||
+        elem.kind === "stats" || // fs.Stats snapshots: scr_stats_* adapters, no trace
         elem.kind === "dyn" || // native collection seeds/drains: traced REF
         elem.kind === "regex" || // RegExp values: scr_regex_* adapters, no trace (no refs inside)
         (elem.kind === "array" && traceAdapter(host, elem) !== null);
@@ -742,13 +965,15 @@ export function mapValKindNum(value) {
  * unions, plus the overflow map on index-signature shapes (one more
  * map-typed member). `index` is the member's field position in the
  * emitted struct type (rc header at 0). */
-export function rcMembers(shape, inline) {
+export function rcMembers(shape, inline, 
+/** The struct index of the first field (2 on tagged shapes). */
+base = 1) {
     return [
-        ...shape.fields.flatMap((f, i) => inline.record(f.type) ? [] : [{ index: i + 1, type: f.type, name: f.name }]),
+        ...shape.fields.flatMap((f, i) => inline.record(f.type) ? [] : [{ index: i + base, type: f.type, name: f.name }]),
         ...(shape.indexValue
             ? [
                 {
-                    index: shape.fields.length + 1,
+                    index: shape.fields.length + base,
                     type: mapOf(STRING, shape.indexValue),
                     name: "[key: string] overflow",
                 },
@@ -759,8 +984,8 @@ export function rcMembers(shape, inline) {
 /** The NULL- and immortal-skipping retain body shared by every shape. A
  * retain never touches the cycle header: a buffered candidate that is
  * retained again stays a candidate (scr_cycle.c, markRoots). */
-export function retainBody(host, fnName, comment = "", immortal = false) {
-    const S = host.sizeType;
+export function retainBody(host, fnName, comment = "", immortal = false, count = host.sizeType) {
+    const S = count;
     if (immortal)
         return immortalRcBody(host, fnName, "ptr", comment);
     return [
@@ -818,19 +1043,20 @@ export const DIRECT_DESTROY_CHAIN = 16;
  * the lines end in the open block `${tag}.next`. */
 export function directChildReleaseLines(host, tag, value, target, comment) {
     const S = host.sizeType;
+    const C = target.count ?? S;
     host.declare(`declare void @scr_rc_destroy(ptr, ptr)`);
     const lines = [
         `  %${tag}.nn = icmp ne ptr ${value}, null`,
         `  br i1 %${tag}.nn, label %${tag}.chk, label %${tag}.next`,
         `${tag}.chk:`,
-        `  %${tag}.rc = load ${S}, ptr ${value}`,
-        `  %${tag}.last = icmp eq ${S} %${tag}.rc, 1`,
+        `  %${tag}.rc = load ${C}, ptr ${value}`,
+        `  %${tag}.last = icmp eq ${C} %${tag}.rc, 1`,
         `  br i1 %${tag}.last, label %${tag}.dead, label %${tag}.shared`,
         `${tag}.shared:`,
         `  call void ${target.release}(ptr ${value}) ; ${comment}`,
         `  br label %${tag}.next`,
         `${tag}.dead: ; ${comment}: the last reference`,
-        `  store ${S} 0, ptr ${value}`,
+        `  store ${C} 0, ptr ${value}`,
     ];
     if (target.traced) {
         host.declare(`declare void @scr_cyc_on_dead(ptr)`);
@@ -847,7 +1073,13 @@ export function directDestroyTarget(host, type, targets) {
     return key === null ? null : (targets.get(key) ?? null);
 }
 function directDestroyKey(host, type) {
-    const t = host.nullableUnions.of(type)?.arm ?? type;
+    const nullable = host.nullableUnions.of(type);
+    let t = type;
+    if (nullable !== null) {
+        if (nullable.repr === "shapes")
+            return null;
+        t = nullable.arm;
+    }
     if (t.kind === "record")
         return `record:${t.shapeId}`;
     return t.kind === "object" ? `object:${t.className}` : null;
@@ -937,8 +1169,8 @@ export function releaseBody(host, fnName, traced, freeBody, comment = "", bounde
  * call `slow`, which
  * holds the complete release: inlining it everywhere would copy teardowns
  * (and their destroy calls) into every caller. */
-export function releaseFastPath(host, fnName, slow, traced, comment = "", immortal = false) {
-    const S = host.sizeType;
+export function releaseFastPath(host, fnName, slow, traced, comment = "", immortal = false, count = host.sizeType) {
+    const S = count;
     if (immortal)
         return immortalRcBody(host, fnName, "void", comment);
     return [
@@ -969,12 +1201,6 @@ export function releaseFastPath(host, fnName, slow, traced, comment = "", immort
         `}`,
     ];
 }
-/** The possible-cycle-root step of a release that left a headered object
- * alive: scr_cyc_on_release with its early return inlined. Test the sign
- * of `buf` (the i32 at obj-4; negative for a queued candidate or a tenured
- * object) and call the runtime only to enqueue a new candidate.
- * Every emitted release of a headered object shares this sequence
- * (scr_runtime.h asserts the offset); it ends by branching to `done`. */
 /** One traced field of an emitted trace function: load the slot and visit
  * it unless it is null. Every collector visitor ignores null children, and
  * most traced slots of a typical heap are empty (optional links, unset
@@ -992,16 +1218,40 @@ export function traceFieldVisitLines(struct, index, i, comment) {
         `tv${i}.next:`,
     ];
 }
+/** Whether candidate buffering goes through the module's append helper
+ * (alloc.ts cycleAppendHelper), requesting its definition if so: in
+ * optimized builds (those with inline RC helpers) that support it. */
+function requestCycleAppend(host) {
+    if (host.rcHelpers === null || !inlineBufferingSupported(host))
+        return false;
+    if (!host.rcHelpers.has(CYC_APPEND_KEY)) {
+        host.rcHelpers.add(CYC_APPEND_KEY);
+        for (const decl of cycleAppendDecls())
+            host.declare(decl);
+    }
+    return true;
+}
+/** The possible-cycle-root step of a release that left a headered object
+ * alive: scr_cyc_on_release with its early return inlined. Test the sign
+ * of `buf` (the i32 at obj-4; negative for a queued candidate or a tenured
+ * object), and buffer a new candidate: through the module's
+ * cycleAppendHelper in optimized builds that support it, through
+ * scr_cyc_on_release otherwise. Every emitted release of a headered object
+ * shares this sequence (scr_runtime.h asserts the offsets); it ends by
+ * branching to `done`. */
 export function cycleRootLines(host, done) {
     const S = host.sizeType;
     host.declare(`declare void @scr_cyc_on_release(ptr)`);
+    const append = requestCycleAppend(host);
     return [
         `  %bufp = getelementptr i8, ptr %o, ${S} -${host.cycleBufOffset}`,
         `  %buf = load i32, ptr %bufp`,
         `  %queued = icmp slt i32 %buf, 0`,
         `  br i1 %queued, label %${done}, label %enqueue`,
         `enqueue:`,
-        `  call void @scr_cyc_on_release(ptr %o) ; buffer the candidate; may collect`,
+        append
+            ? `  call preserve_mostcc void @${CYC_APPEND}(ptr %o) ; buffer the candidate; may collect`
+            : `  call void @scr_cyc_on_release(ptr %o) ; buffer the candidate; may collect`,
         `  br label %${done}`,
     ];
 }
@@ -1045,6 +1295,9 @@ export function emitRecordShapes(host, mod) {
         const fieldTys = shape.fields.map((f) => llFieldType(f.type, host.inlineUnions.record(f.type)));
         if (shape.indexValue)
             fieldTys.push("ptr"); // the overflow ScrMap *
+        // A tagged shape (an arm of a shape union) carries its shape number.
+        if (host.nullableUnions.shapeNumber(shape.id) !== 0)
+            fieldTys.unshift("i32");
         typeDefs.push(`%${struct} = type { ${host.sizeType}${fieldTys.length ? ", " + fieldTys.join(", ") : ""} } ` +
             `; record ${shape.id} { ${shape.fields.map((f) => llvmCommentText(f.name)).join("; ")}${shape.indexValue ? "; [key: string]" : ""} }`);
     }
@@ -1054,7 +1307,7 @@ export function emitRecordShapes(host, mod) {
     const candidates = [];
     const memberTypes = [];
     for (const shape of records) {
-        const members = rcMembers(shape, host.inlineUnions);
+        const members = rcMembers(shape, host.inlineUnions, host.nullableUnions.fieldBase(shape.id));
         for (const m of members)
             memberTypes.push(m.type);
         if (!members.some((m) => isRefCounted(m.type) && needsBoundedRelease(m.type)))
@@ -1073,7 +1326,9 @@ export function emitRecordShapes(host, mod) {
     for (const shape of records) {
         const struct = mangleRecordStruct(shape.id);
         const traced = host.tracedShapes.has(`record:${shape.id}`);
-        const members = rcMembers(shape, host.inlineUnions);
+        const base = host.nullableUnions.fieldBase(shape.id);
+        const shapeNumber = host.nullableUnions.shapeNumber(shape.id);
+        const members = rcMembers(shape, host.inlineUnions, base);
         const refMembers = members.filter((m) => isRefCounted(m.type));
         const sizeOf = `ptrtoint (ptr getelementptr (%${struct}, ptr null, i32 1) to ${host.sizeType})`;
         defs.push(...retainBody(host, mangleRecordRetain(shape.id)), ``);
@@ -1098,7 +1353,9 @@ export function emitRecordShapes(host, mod) {
             }
             t++;
         }
-        freeBody.push(...emitObjectFree(host, traced));
+        // Closed layouts can carry a kept record source (record-sources.ts).
+        const observers = recordMayKeepSource(shape) ? "weak-or-source" : "weak";
+        freeBody.push(...emitObjectFree(host, traced, observers));
         defs.push(...releaseBody(host, mangleRecordRelease(shape.id), traced, freeBody, "", refMembers.some((member) => needsBoundedRelease(member.type)), false, chained), ``);
         // new: zeroed allocation (+ the overflow map on index-signature
         // shapes), rc = 1. Traced shapes allocate with the collector header;
@@ -1112,6 +1369,8 @@ export function emitRecordShapes(host, mod) {
             defs.push(cycTypeDescriptor(cycType, `@${mangleRecordTrace(shape.id)}`, `@${mangleRecordGcFree(shape.id)}`, host.nextCycKind?.() ?? 0), ``);
         nw.push(...emitObjectAlloc(host, sizeOf, traced ? { type: cycType } : null));
         nw.push(`  store ${host.sizeType} 1, ptr %o`);
+        if (shapeNumber !== 0)
+            nw.push(`  %shp = getelementptr inbounds %${struct}, ptr %o, i64 0, i32 ${SHAPE_WORD_INDEX}`, `  store i32 ${shapeNumber}, ptr %shp ; shape number`);
         if (shape.indexValue) {
             // The overflow map (string-keyed): value handling is type-directed
             // exactly like shapes.ts's overflowNewC.
@@ -1120,8 +1379,10 @@ export function emitRecordShapes(host, mod) {
             const valKind = v.kind === "f64" ? 0 : v.kind === "bool" ? 1 : 2;
             const rc = valKind === 2 ? vAdapters(host, v) : { retain: "null", release: "null" };
             const trace = valKind === 2 ? traceArg(host, v) : "null";
-            nw.push(`  %ovf = call ptr @scr_map_new(i32 1, i32 ${valKind}, ptr ${host.mapOps(["null", "null", "null", rc.retain, rc.release, trace, "null"])})`, `  %ovfp = getelementptr inbounds %${struct}, ptr %o, i64 0, i32 ${shape.fields.length + 1}`, `  store ptr %ovf, ptr %ovfp`);
+            nw.push(`  %ovf = call ptr @scr_map_new(i32 1, i32 ${valKind}, ptr ${host.mapOps(["null", "null", "null", rc.retain, rc.release, trace, "null"])})`, `  %ovfp = getelementptr inbounds %${struct}, ptr %o, i64 0, i32 ${shape.fields.length + base}`, `  store ptr %ovf, ptr %ovfp`);
         }
+        if (traced)
+            nw.push(...youngEntryLines(host));
         nw.push(`  ret ptr %o`, `}`, ``);
         defs.push(...nw);
         if (host.recordCloneShapes.has(shape.id)) {
@@ -1133,7 +1394,7 @@ export function emitRecordShapes(host, mod) {
             ];
             let i = 0;
             for (const field of shape.fields) {
-                const index = i + 1;
+                const index = i + base;
                 const inline = host.inlineUnions.record(field.type);
                 const fieldTy = llFieldType(field.type, inline);
                 clone.push(`  %sp${i} = getelementptr inbounds %${struct}, ptr %src, i64 0, i32 ${index}`, `  %sv${i} = load ${fieldTy}, ptr %sp${i}`);
@@ -1170,7 +1431,7 @@ export function emitRecordShapes(host, mod) {
             untracedRefMembers.forEach((m, i) => {
                 gf.push(`  %f${i} = getelementptr inbounds %${struct}, ptr %o, i64 0, i32 ${m.index}`, `  %v${i} = load ptr, ptr %f${i}`, `  call void ${releaseSym(host, m.type)}(ptr %v${i}) ; ${llvmCommentText(m.name)} (acyclic)`);
             });
-            gf.push(...emitObjectFree(host, true), `  ret void`, `}`, ``);
+            gf.push(...emitObjectFree(host, true, observers), `  ret void`, `}`, ``);
             defs.push(...gf);
         }
     }