@scriptc/compiler 0.4.5 → 0.4.6

package/dist/backend/llvm/emit-module.js

@@ -1,4 +1,5 @@
 /* LlEmitter's module-level output: module parts, helper definitions, library declarations and FFI callback definitions. */
+import { loopUnrollMetadata } from "./periodic-guards.js";
 import { relieveHeapPressure } from "../../heap-pressure.js";
 import { emitWasmModules } from "./wasm-modules.js";
 import { ffiCallbackDummyLl, llStrBytes } from "./emitter.js";
@@ -19,7 +20,8 @@ import { RUNTIME_ABI_MARKER } from "../runtime-abi.js";
 import { threadSlotsBlockDefinitions } from "./thread-slots.js";
 import { ABSENT_FIELD_PAYLOAD, exactIndexLines, f64Lit, ffiNativeParamLl, ffiNativeReturnLl, ffiNativeTypeLl, } from "./common.js";
 import { literalCapWord, literalCapWord32 } from "./string-key-hash.js";
-import { hasLoneSurrogate, wtf8Bytes } from "../literals.js";
+import { hasLoneSurrogate, llvmBytes, wtf8Bytes } from "../literals.js";
+import { nodeIslandManifest } from "../../node-islands/modules.js";
 import { undefinedArmTag } from "../../ir/analysis.js";
 import { NULLABLE_ABSENT, NULLABLE_NULL } from "./nullable-unions.js";
 import { literalEqualityDefinition } from "./string-literal-equality.js";
@@ -27,6 +29,8 @@ import { STRING_READ_HELPERS, stringScanDecl, stringScanInline } from "./string-
 import { emitLibraryIdentityLines } from "../library-identity-markers.js";
 import { InternalCompilerError } from "../../errors.js";
 import { targetCpuAttribute } from "../target-cpu.js";
+import { CPU_LEVEL_INIT_CALL, cpuDispatchAttributes, cpuLevelDefs } from "./cpu-dispatch.js";
+import { layoutTierOf } from "./function-layout.js";
 /** The function bodies emitted in `chunks` contiguous ranges, each by its
  * own emitter (LlEmitter.chunkEmitter), then merged in function order: the
  * text one emitter gives. */
@@ -66,13 +70,33 @@ export function workerMinFunctions() {
     const value = Number(process.env["SCRIPTC_EMIT_WORKER_FUNCTIONS"] ?? "2000");
     return Number.isFinite(value) && value > 0 ? value : 2000;
 }
+/** A function's estimated emission cost beyond its IR nodes, in nodes:
+ * fitted to the worker emission times of a large program's ranges. */
+const FUNCTION_EMIT_COST = 22;
+/** Where each of about `count` contiguous ranges of `functions` starts,
+ * followed by the end: ranges of about equal estimated emission cost, a
+ * function's node count plus FUNCTION_EMIT_COST. No range is empty. */
+export function emitRangeStarts(functions, count, nodeCount) {
+    const costs = functions.map((fn) => (nodeCount(fn) ?? 0) + FUNCTION_EMIT_COST);
+    const total = costs.reduce((sum, cost) => sum + cost, 0);
+    const starts = [0];
+    let sum = 0;
+    for (let i = 0; i < functions.length - 1 && starts.length < count; i++) {
+        sum += costs[i];
+        if (sum >= (total * starts.length) / count)
+            starts.push(i + 1);
+    }
+    starts.push(functions.length);
+    return starts;
+}
 /** The function bodies emitted on forked worker processes, `workers` at a
- * time: the function list splits into contiguous ranges, each emitted in
- * its own process, which inherits this emitter as it is at the fork and
- * records a journal (emit-journal.ts). Ranges merge in function order, so
- * the text is the serial text. A range whose worker fails is emitted here
- * when its turn comes. Null without fork primitives (Node, Windows, inside
- * a worker) or when the module is small. */
+ * time: the function list splits into contiguous ranges of about equal
+ * estimated cost (emitRangeStarts), each emitted in its own process, which
+ * inherits this emitter as it is at the fork and records a journal
+ * (emit-journal.ts). A new range starts whenever a worker ends, and ranges
+ * merge in function order, so the text is the serial text. A range whose
+ * worker fails is emitted here when its turn comes. Null without fork
+ * primitives (Node, Windows, inside a worker) or when the module is small. */
 function emitFunctionsOnWorkers(emitter, workers) {
     const fork = emitter.targetOptions.forkPrimitives;
     const functions = emitter.mod.functions;
@@ -81,11 +105,15 @@ function emitFunctionsOnWorkers(emitter, workers) {
     if (functions.length < workerMinFunctions())
         return null;
     // More ranges than workers: a range's merge overlaps the later ranges'
-    // emission, and a slow range holds back less.
-    const count = Math.min(functions.length, workers * 3);
-    const size = Math.ceil(functions.length / count);
+    // emission, and a slow range holds back less. Ranges of equal cost finish
+    // at about the same time.
+    const starts = emitRangeStarts(functions, Math.min(functions.length, workers * 3), (fn) => emitter.nodeIndex.nodeCount(fn));
+    const count = starts.length - 1;
+    const rangeFunctions = (range) => functions.slice(starts[range], starts[range + 1]);
     const dir = mkdtempSync(join(tmpdir(), "scriptc-emit-"));
     const pids = [];
+    // The workers not yet reaped, by pid, with their ranges.
+    const running = new Map();
     const start = (range) => {
         const pid = fork.fork();
         if (pid === 0) {
@@ -94,7 +122,7 @@ function emitFunctionsOnWorkers(emitter, workers) {
                 emitter.journal = new EmitJournal();
                 emitter.debug?.record();
                 const texts = [];
-                for (const fn of functions.slice(range * size, (range + 1) * size)) {
+                for (const fn of rangeFunctions(range)) {
                     texts.push(emitter.emitFunction(fn));
                     relieveHeapPressure();
                 }
@@ -109,18 +137,42 @@ function emitFunctionsOnWorkers(emitter, workers) {
             fork.exit(code);
         }
         pids.push(pid);
+        if (pid > 0)
+            running.set(pid, range);
     };
     const fnDefs = [];
+    // Ended ranges by index, with what their workers wrote (null when a worker
+    // failed), waiting to merge in function order.
+    const ended = new Map();
+    // Reaps a worker and starts the next range in its place. A worker's journal
+    // records requests, not table contents, so its range's text doesn't depend
+    // on what merged before it forked.
+    const reap = (pid, range) => {
+        const status = pid > 0 ? fork.wait(pid) : -1;
+        running.delete(pid);
+        if (pids.length < count)
+            start(pids.length);
+        ended.set(range, status === 0 ? readRange(dir, range) : null);
+    };
     try {
         for (let range = 0; range < Math.min(workers, count); range++)
             start(range);
         for (let range = 0; range < count; range++) {
-            const pid = pids[range];
-            const status = pid > 0 ? fork.wait(pid) : -1;
-            // The next worker starts from this emitter as merged so far, which
-            // the journal allows: it records requests, not table contents.
-            const merged = status === 0 ? readRange(dir, range) : null;
-            const slice = functions.slice(range * size, (range + 1) * size);
+            // Until this range's worker ends, reap whichever worker ends first, so
+            // that `workers` keep running while ranges merge in order. A child
+            // that isn't a worker (or no `exited`) waits for this range's worker.
+            while (!ended.has(range)) {
+                const pid = pids[range];
+                const first = pid > 0 && fork.exited !== undefined ? fork.exited() : pid;
+                const other = running.get(first);
+                if (first !== pid && other !== undefined)
+                    reap(first, other);
+                else
+                    reap(pid, range);
+            }
+            const merged = ended.get(range);
+            ended.delete(range);
+            const slice = rangeFunctions(range);
             if (merged !== null && merged.texts.length === slice.length) {
                 const rename = emitter.mergeChunk(merged.result);
                 slice.forEach((fn, index) => {
@@ -136,8 +188,6 @@ function emitFunctionsOnWorkers(emitter, workers) {
                     fnDefs.push(def);
                     relieveHeapPressure();
                 }
-            if (pids.length < count)
-                start(pids.length);
         }
     }
     finally {
@@ -158,15 +208,22 @@ function readRange(dir, range) {
 }
 /** The attribute groups of the emitted functions: #0 for every function,
  * #1 for coroutine bodies and #2 for out-of-line record clones. Each carries
- * the build's `--target-cpu` level. */
+ * the build's `--target-cpu` level, `sanitize_address` in sanitized builds,
+ * and `uwtable` on native targets, so every function has an unwind table
+ * entry, including those LLVM infers `nounwind` (profilers and debuggers
+ * unwind through them). A group needs one attribute: without any, #0 holds a
+ * string attribute that LLVM ignores. Dispatching builds add the x86-64-v3
+ * copies' #3 and #4 (cpu-dispatch.ts). */
 function functionAttributes(emitter, hasNoInlineRecordClone) {
     const cpu = targetCpuAttribute(emitter.targetCpu);
+    const asan = emitter.sanitizeAttributes ? " sanitize_address" : "";
+    const common = `${asan}${emitter.wasi ? "" : " uwtable"}`;
+    const base = `${common}${cpu}` || ` "scriptc-function"`;
     return [
-        `attributes #0 = { sanitize_address${cpu} }`,
-        ...(emitter.usesCoroutines
-            ? [`attributes #1 = { sanitize_address presplitcoroutine${cpu} }`]
-            : []),
-        ...(hasNoInlineRecordClone ? [`attributes #2 = { noinline sanitize_address${cpu} }`] : []),
+        `attributes #0 = {${base} }`,
+        ...(emitter.usesCoroutines ? [`attributes #1 = {${common} presplitcoroutine${cpu} }`] : []),
+        ...(hasNoInlineRecordClone ? [`attributes #2 = { noinline${common}${cpu} }`] : []),
+        ...(emitter.cpuDispatch ? cpuDispatchAttributes(common) : []),
     ];
 }
 /** The group line written before each function's text. Functions from one
@@ -189,6 +246,78 @@ function partitionGroupLines(functions) {
 }
 /** Ends the group of the definitions that follow. */
 const PARTITION_GROUP_END = "; scriptc-group";
+/** Written before the entry module's top-level code (its `%init.<n>`). The
+ * LLVM helper gives that function and the chain that follows it a partition
+ * of their own, so an edit of the program's top level recompiles only that
+ * code (native/llvm-codegen/src/partition.cpp). */
+const PARTITION_ENTRY_LINE = "; scriptc-entry";
+/** Whether each function is the entry module's top-level code. */
+function partitionEntries(functions, sourceFile) {
+    return functions.map((fn) => fn.name.startsWith("%init.") && fn.loc.file === sourceFile);
+}
+/** Writes the generated helpers, then the function definitions with their
+ * group lines, and the entry line before the entry module's top-level code.
+ * With a layout, the definitions it lists, helpers included, move after the
+ * unlisted helpers and before the other functions, by tier, each in its
+ * tier's group (`layout-<tier>`). The helper compiles each
+ * partition's definitions in text order, so listed ones come first in their
+ * partitions, and it places a group's definitions in the same partitions.
+ * Everything else keeps its order and its source-file group. */
+function writeDefinitions(out, layout, helperTexts, fnDefs, groups, entries) {
+    const tierLine = (tier) => `${PARTITION_GROUP_END} layout-${tier}`;
+    const listed = new Map();
+    const listIn = (tier, lines) => {
+        const list = listed.get(tier) ?? [];
+        if (!listed.has(tier))
+            listed.set(tier, list);
+        for (const line of lines)
+            list.push(line);
+    };
+    if (layout === undefined || layout.size === 0) {
+        for (const texts of helperTexts)
+            for (const text of texts)
+                out.push(text);
+    }
+    else {
+        // A helper's text can span several entries (one per line), so the
+        // definitions are found in the joined text. A listed definition moves
+        // through its closing brace; what follows it stays in place.
+        const joined = [];
+        for (const texts of helperTexts)
+            for (const text of texts)
+                joined.push(text);
+        for (const part of joined.join("\n").split(/\n(?=define )/)) {
+            const tier = part.startsWith("define ") ? layoutTierOf(layout, part) : undefined;
+            const close = tier === undefined ? -1 : part.search(/\n}(\n|$)/);
+            if (tier === undefined || close < 0) {
+                out.push(part);
+                continue;
+            }
+            listIn(tier, [tierLine(tier), part.slice(0, close + 2)]);
+            const rest = part.slice(close + 3);
+            if (rest !== "")
+                out.push(rest);
+        }
+    }
+    const unlisted = [];
+    fnDefs.forEach((text, i) => {
+        const tier = layout === undefined || layout.size === 0 ? undefined : layoutTierOf(layout, text);
+        if (tier === undefined)
+            unlisted.push(i);
+        else
+            listIn(tier, ["", ...(entries[i] ? [PARTITION_ENTRY_LINE] : []), tierLine(tier), text]);
+    });
+    for (const tier of [...listed.keys()].sort((a, b) => a - b))
+        for (const line of listed.get(tier))
+            out.push(line);
+    for (const [k, i] of unlisted.entries()) {
+        if (k > 0 || listed.size > 0)
+            out.push("");
+        if (entries[i])
+            out.push(PARTITION_ENTRY_LINE);
+        out.push(groups[i] ?? PARTITION_GROUP_END, fnDefs[i]);
+    }
+}
 export function emitModuleParts(emitter) {
     // Function bodies first (the literal/unit/fn-value tables fill as they
     // emit), then the file assembles around them — the runtime ABI’s order.
@@ -280,6 +409,9 @@ export function emitModuleParts(emitter) {
     const usesEvents = runtimeFeatures.processEvents;
     const usesChildProcess = runtimeFeatures.childProcess;
     const usesFsWatch = runtimeFeatures.fsWatch;
+    // Programs that call into Node record the libnode pack and its Node
+    // version before %main (scr_node_island.c).
+    const usesNodeIsland = runtimeFeatures.nodeIsland;
     // Stream-surface programs fill the loop's stream hook (the deferred
     // next-tick emissions) and the emitter's post-registration flow kick
     // before %main — scr_stream.c links only when the line is emitted
@@ -338,6 +470,19 @@ export function emitModuleParts(emitter) {
         emitter.declare(`declare void @scr_worker_stdio_install(ptr, ptr, ptr)`);
     if (usesFsWatch)
         emitter.declare(`declare void @scr_watch_install()`);
+    if (usesNodeIsland)
+        emitter.declare(`declare void @scr_node_island_install(ptr, ptr)`);
+    // Interned now: the C-string constants flush before main assembles.
+    const nodeIslandConfig = usesNodeIsland
+        ? [
+            emitter.cstr(emitter.mod.nodeIsland?.libnode ?? ""),
+            emitter.cstr(emitter.mod.nodeIsland?.nodeVersion ?? ""),
+        ]
+        : null;
+    // The island module tree: its manifest, and its files unless they ship
+    // beside the executable (node-islands/modules.ts).
+    const nodeIslandModules = usesNodeIsland ? emitter.mod.nodeIsland?.modules : undefined;
+    const nodeIslandModulesCall = nodeIslandModules === undefined ? null : nodeIslandModulesInstall(emitter, nodeIslandModules);
     if (emitter.ffiHasForeignCallback)
         emitter.declare(`declare void @scr_ffi_install()`);
     if (usesStream)
@@ -409,13 +554,15 @@ export function emitModuleParts(emitter) {
     const globalReleases = exitReleaseLines("g");
     const asyncEntry = emitter.fnByName.get(emitter.mod.entry)?.async === true;
     const entryMayThrow = emitter.mayThrow.has(emitter.mod.entry);
-    // The event loop runs when timers appeared OR any async/generator
-    // function exists (the C main's hasAsync || hasGenerators ||
-    // usesTimers gate). Generator programs run
+    // The event loop runs when timers appeared, a promise is made (the loop's
+    // end reports an unhandled rejection, Node's exit 1), OR any
+    // async/generator function exists (the C main's hasAsync ||
+    // hasGenerators || usesTimers gate). Generator programs run
     // the loop too: its exit accounting notes still-suspended generator
     // fibers as abandoned, so the RC audit downgrades exactly like the
     // async loop-exhaustion story.
     const runsLoop = emitter.usesTimers ||
+        emitter.usesPromises ||
         emitter.ffiHasForeignCallback ||
         emitter.mod.functions.some((f) => f.async === true || f.generator !== undefined);
     const uncaughtReleases = entryMayThrow && !asyncEntry ? exitReleaseLines("gu") : [];
@@ -519,6 +666,7 @@ export function emitModuleParts(emitter) {
         ...(emitter.publishWalkersCache?.defs() ?? []),
         ...emitter.publishedClassNamesDefs(),
         ...(emitter.regions?.moduleDefs() ?? []),
+        ...(emitter.cpuDispatch ? cpuLevelDefs((line) => emitter.declare(line)) : []),
     ];
     if (emitter.constantNumericTables.size > 0)
         emitter.declare(`declare double @scr_arr_get_number(ptr, double)`);
@@ -532,6 +680,17 @@ export function emitModuleParts(emitter) {
         for (const decl of code.decls)
             emitter.declare(decl);
     }
+    // Replacers of global substitutions with a literal replacement; the call
+    // sites register only patterns inside the compiled subset.
+    const regexReplacers = [];
+    for (const rr of emitter.regexReplacers.values()) {
+        const code = compileRegexMatcher(rr.sym, rr.pattern, rr.flags, emitter.sizeType, !emitter.wasi, rr.replacement);
+        if (code === null)
+            throw new InternalCompilerError(`llvm emitter bug: replacer ${rr.sym} did not compile`);
+        regexReplacers.push(code);
+        for (const decl of code.decls)
+            emitter.declare(decl);
+    }
     const out = [
         `; Generated by scriptc (LLVM backend) from ${emitter.mod.sourceFile}. Do not edit.`,
         ``,
@@ -679,6 +838,8 @@ export function emitModuleParts(emitter) {
         for (let site = 0; site < emitter.jsIcSiteCount; site++)
             out.push(`@sc_js_ic_${site} = internal ${tl}global [${JS_IC_WORDS} x i64] zeroinitializer`);
     }
+    if (nodeIslandModulesCall !== null)
+        out.push(...nodeIslandModulesCall.globals);
     for (const table of emitter.constantNumericTables.values()) {
         const n = table.values.length;
         let tableTmp = 0;
@@ -774,6 +935,8 @@ export function emitModuleParts(emitter) {
         out.push(``);
     for (const code of regexMatchers.values())
         out.push(...code.lines, ``);
+    for (const code of regexReplacers)
+        out.push(...code.lines, ``);
     for (const [, inst] of emitter.templateStringsInstances) {
         // One immortal ScrArr per tagged-template site: a data global of
         // interned cooked-string literals, and the ScrArr header over it
@@ -817,32 +980,18 @@ export function emitModuleParts(emitter) {
     }
     if (globals.length > 0)
         out.push(``);
-    for (const line of helpers)
-        out.push(line);
-    for (const line of ffiCallbacks.defs)
-        out.push(line);
-    for (const line of shapes.defs)
-        out.push(line);
-    for (const line of classShapes.defs)
-        out.push(line);
-    for (const line of classObjDefs)
-        out.push(line);
-    for (const line of emitter.walkers.defs)
-        out.push(line);
-    for (const line of emitter.dyn.defs)
-        out.push(line);
-    for (const line of wrappers)
-        out.push(line);
-    for (const line of asyncDefs)
-        out.push(line);
-    for (const line of emitter.resolveThunkDefs)
-        out.push(line);
-    const groups = partitionGroupLines(emitter.mod.functions);
-    for (let i = 0; i < fnDefs.length; i++) {
-        if (i > 0)
-            out.push("");
-        out.push(groups[i] ?? PARTITION_GROUP_END, fnDefs[i]);
-    }
+    writeDefinitions(out, emitter.targetOptions.functionLayout, [
+        helpers,
+        ffiCallbacks.defs,
+        shapes.defs,
+        classShapes.defs,
+        classObjDefs,
+        emitter.walkers.defs,
+        emitter.dyn.defs,
+        wrappers,
+        asyncDefs,
+        emitter.resolveThunkDefs,
+    ], fnDefs, partitionGroupLines(emitter.mod.functions), partitionEntries(emitter.mod.functions, emitter.mod.sourceFile));
     if (fnDefs.length === 0)
         out.push("");
     out.push(PARTITION_GROUP_END, "");
@@ -911,11 +1060,12 @@ export function emitModuleParts(emitter) {
         out.push(...functionAttributes(emitter, hasNoInlineRecordClone));
         out.push(``);
         out.push(...emitter.fieldAliasMetadata());
+        out.push(...loopUnrollMetadata(out));
         if (emitter.debug !== null)
             out.push(emitter.debug.render());
         return out;
     }
-    out.push(`define i32 @${emitter.mod.workers ? "sc_context_entry" : emitter.wasi ? "__main_argc_argv" : "main"}(i32 %argc, ptr %argv) ${FN_ATTRS} {`, `entry:`, ...(emitter.runtimeAbiMarker ? [`  call void @${RUNTIME_ABI_MARKER}()`] : []), ...(emitter.mod.workers ? [`  call void @scr_runtime_workers_v8()`] : []), `  call void @scr_init()`, 
+    out.push(`define i32 @${emitter.mod.workers ? "sc_context_entry" : emitter.wasi ? "__main_argc_argv" : "main"}(i32 %argc, ptr %argv) ${FN_ATTRS} {`, `entry:`, ...(emitter.runtimeAbiMarker ? [`  call void @${RUNTIME_ABI_MARKER}()`] : []), ...(emitter.cpuDispatch && !emitter.mod.workers ? [CPU_LEVEL_INIT_CALL] : []), ...(emitter.mod.workers ? [`  call void @scr_runtime_workers_v8()`] : []), `  call void @scr_init()`, 
     // Compressed fields hold addresses in the program-lifetime region, which
     // is reserved below 32 GiB before compiled code runs.
     ...(emitter.compactRegion ? [`  call void @scr_plr_init()`] : []), 
@@ -936,7 +1086,11 @@ export function emitModuleParts(emitter) {
         : []), 
     // fs.watch programs fill the loop's watch hooks the same way —
     // scr_watch.c links only when this line is emitted.
-    ...(usesFsWatch ? [`  call void @scr_watch_install()`] : []), ...(emitter.ffiHasForeignCallback ? [`  call void @scr_ffi_install()`] : []), ...(snapshotsTlsCa ? [`  call void @scr_tls_ca_install()`] : []), ...(usesFetch ? [`  call void @scr_fetch_install()`] : []), ...(usesUrlIdna ? [`  call void @scr_url_idna_install()`] : []), ...(usesNet ? [`  call void @scr_net_install()`, `  call void @scr_net_dyn_install()`] : []), ...(usesNetLookup ? [`  call void @scr_net_lookup_install()`] : []), ...(usesHttp ? [`  call void @scr_http_dyn_install()`] : []), ...(usesDgram ? [`  call void @scr_dgram_install()`] : []), ...(usesHttp2 ? [`  call void @scr_http2_dyn_install()`] : []), ...(usesStream ? [`  call void @scr_stream_install()`] : []), `  call void @scr_lib_init(i32 %argc, ptr %argv)`, 
+    ...(usesFsWatch ? [`  call void @scr_watch_install()`] : []), ...(nodeIslandConfig
+        ? [
+            `  call void @scr_node_island_install(ptr ${nodeIslandConfig[0]}, ptr ${nodeIslandConfig[1]})`,
+        ]
+        : []), ...(nodeIslandModulesCall === null ? [] : [nodeIslandModulesCall.call]), ...(emitter.ffiHasForeignCallback ? [`  call void @scr_ffi_install()`] : []), ...(snapshotsTlsCa ? [`  call void @scr_tls_ca_install()`] : []), ...(usesFetch ? [`  call void @scr_fetch_install()`] : []), ...(usesUrlIdna ? [`  call void @scr_url_idna_install()`] : []), ...(usesNet ? [`  call void @scr_net_install()`, `  call void @scr_net_dyn_install()`] : []), ...(usesNetLookup ? [`  call void @scr_net_lookup_install()`] : []), ...(usesHttp ? [`  call void @scr_http_dyn_install()`] : []), ...(usesDgram ? [`  call void @scr_dgram_install()`] : []), ...(usesHttp2 ? [`  call void @scr_http2_dyn_install()`] : []), ...(usesStream ? [`  call void @scr_stream_install()`] : []), `  call void @scr_lib_init(i32 %argc, ptr %argv)`, 
     // node:test takes Node's --test-update-snapshots out of argv first.
     ...(usesNodeTest ? [`  call void @scr_test_init()`] : []), ...(asyncEntry
         ? [`  %top = call ptr @${mangleAsyncSpawn(emitter.mod.entry)}()`]
@@ -1031,21 +1185,157 @@ export function emitModuleParts(emitter) {
     for (const line of emitter.inlineRcTail(flushedDecls))
         out.push(line);
     out.push(
-    // sanitize_address is inert under the plain pipeline; the sanitized
-    // lane's -fsanitize=address link activates instrumentation over the
-    // emitted functions too (the runtime TUs get theirs from clang).
+    // The sanitized lane's -fsanitize=address link instruments the
+    // functions marked sanitize_address (the runtime TUs get theirs from
+    // clang).
     ...functionAttributes(emitter, hasNoInlineRecordClone), ``);
     if (emitter.mod.workers) {
-        out.push(`declare void @scr_context_cleanup()`, `declare i32 @scr_worker_argc()`, `declare ptr @scr_worker_argv()`, `define i32 @sc_worker_entry(i32 %root) ${FN_ATTRS} {`, `entry:`, `  %argc = call i32 @scr_worker_argc()`, `  %argv = call ptr @scr_worker_argv()`, `  %result = call i32 @sc_context_entry(i32 %argc, ptr %argv)`, `  ret i32 %result`, `}`, `define i32 @main(i32 %argc, ptr %argv) ${FN_ATTRS} {`, `entry:`, `  %result = call i32 @sc_context_entry(i32 %argc, ptr %argv)`, `  call void @scr_context_cleanup()`, `  ret i32 %result`, `}`);
+        out.push(`declare void @scr_context_cleanup()`, `declare i32 @scr_worker_argc()`, `declare ptr @scr_worker_argv()`, `define i32 @sc_worker_entry(i32 %root) ${FN_ATTRS} {`, `entry:`, `  %argc = call i32 @scr_worker_argc()`, `  %argv = call ptr @scr_worker_argv()`, `  %result = call i32 @sc_context_entry(i32 %argc, ptr %argv)`, `  ret i32 %result`, `}`, `define i32 @main(i32 %argc, ptr %argv) ${FN_ATTRS} {`, `entry:`, 
+        // Worker threads enter at sc_worker_entry and read the level main set.
+        ...(emitter.cpuDispatch ? [CPU_LEVEL_INIT_CALL] : []), `  %result = call i32 @sc_context_entry(i32 %argc, ptr %argv)`, `  call void @scr_context_cleanup()`, `  ret i32 %result`, `}`);
     }
     out.push(...emitter.fieldAliasMetadata());
+    out.push(...loopUnrollMetadata(out));
     if (emitter.debug !== null)
         out.push(emitter.debug.render());
     return out;
 }
+/** The out-of-line hashed probe of a number-key map (scr_map_find_f64):
+ * `keyLines` define the normalized key word %kb from the parameter, and
+ * `runtimeKey` is the double passed to the runtime lookup. A map with at
+ * most SCR_MAP_LINEAR_LIMIT entries keeps no bucket table: the absent-key
+ * filter is tested before its entries are scanned. A bucket table is
+ * probed without the filter: the filter keeps two of its 64 bits per key,
+ * so past a few dozen keys it rejects few absent keys, and the probe's
+ * first bucket read answers those. Compact tables tag each 24-bit entry
+ * index (+1) with the hash's top byte; wider tables take the runtime
+ * lookup. */
+function hashedNumberProbe(name, param, sz, keyLines, runtimeKey) {
+    return [
+        `define internal ptr @${name}(ptr %m, ${param}) noinline ${FN_ATTRS} {`,
+        `entry:`,
+        ...keyLines,
+        `  %x1s = lshr i64 %kb, 30`,
+        `  %x1 = xor i64 %kb, %x1s`,
+        `  %x2 = mul i64 %x1, -4658895280553007687`,
+        `  %x3s = lshr i64 %x2, 27`,
+        `  %x3 = xor i64 %x2, %x3s`,
+        `  %x4 = mul i64 %x3, -7723592293110705685`,
+        `  %x5s = lshr i64 %x4, 31`,
+        `  %x5 = xor i64 %x4, %x5s`,
+        `  %hz = icmp eq i64 %x5, 0`,
+        `  %h = select i1 %hz, i64 1, i64 %x5`,
+        `  %nbp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 13`,
+        ...(sz === "i64"
+            ? [`  %nb32 = load i32, ptr %nbp`, `  %nb = zext i32 %nb32 to i64`]
+            : [`  %nb = load i32, ptr %nbp`]),
+        `  %eps = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 11`,
+        `  %ents = load ptr, ptr %eps`,
+        `  %linear = icmp eq ${sz} %nb, 0`,
+        `  br i1 %linear, label %filter, label %buckets`,
+        `filter:`,
+        `  %f1s = lshr i64 %h, 44`,
+        `  %f1a = and i64 %f1s, 63`,
+        `  %f1 = shl i64 1, %f1a`,
+        `  %f2s = lshr i64 %h, 50`,
+        `  %f2a = and i64 %f2s, 63`,
+        `  %f2 = shl i64 1, %f2a`,
+        `  %fb = or i64 %f1, %f2`,
+        `  %fp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 10`,
+        `  %filter.bits = load i64, ptr %fp`,
+        `  %fm = and i64 %filter.bits, %fb`,
+        `  %fin = icmp eq i64 %fm, %fb`,
+        `  br i1 %fin, label %scan, label %miss`,
+        // A number-key entry has no cached hash; a dead entry's key matches no key.
+        `scan:`,
+        `  %nep = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 8`,
+        ...(sz === "i64"
+            ? [`  %ne32 = load i32, ptr %nep`, `  %ne = zext i32 %ne32 to i64`]
+            : [`  %ne = load i32, ptr %nep`]),
+        `  br label %scan.loop`,
+        `scan.loop:`,
+        `  %si = phi ${sz} [ 0, %scan ], [ %si.next, %scan.next ]`,
+        `  %sdone = icmp eq ${sz} %si, %ne`,
+        `  br i1 %sdone, label %miss, label %scan.entry`,
+        `scan.entry:`,
+        `  %se = getelementptr inbounds %ScrMapNumEntry, ptr %ents, ${sz} %si`,
+        `  %sk = load i64, ptr %se`,
+        `  %skq = icmp eq i64 %sk, %kb`,
+        `  br i1 %skq, label %scan.found, label %scan.next`,
+        `scan.found:`,
+        `  ret ptr %se`,
+        `scan.next:`,
+        `  %si.next = add ${sz} %si, 1`,
+        `  br label %scan.loop`,
+        `buckets:`,
+        `  %wide = icmp ugt ${sz} %nb, 16777216`,
+        `  br i1 %wide, label %runtime, label %probe`,
+        `probe:`,
+        `  %bp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 14`,
+        `  %bk = load ptr, ptr %bp`,
+        `  %mask = sub ${sz} %nb, 1`,
+        `  %ht = lshr i64 %h, 32`,
+        `  %ht32 = trunc i64 %ht to i32`,
+        `  %tag = and i32 %ht32, -16777216`,
+        sz === "i64" ? `  %hi = add i64 %h, 0` : `  %hi = trunc i64 %h to ${sz}`,
+        `  %i0 = and ${sz} %hi, %mask`,
+        `  br label %probe.loop`,
+        `probe.loop:`,
+        `  %pi = phi ${sz} [ %i0, %probe ], [ %pi.next, %probe.next ]`,
+        `  %bep = getelementptr inbounds i32, ptr %bk, ${sz} %pi`,
+        `  %b = load i32, ptr %bep`,
+        `  %bempty = icmp eq i32 %b, 0`,
+        `  br i1 %bempty, label %miss, label %probe.tag`,
+        `probe.tag:`,
+        `  %bt = and i32 %b, -16777216`,
+        `  %btq = icmp eq i32 %bt, %tag`,
+        `  br i1 %btq, label %probe.entry, label %probe.next`,
+        `probe.entry:`,
+        `  %bix = and i32 %b, 16777215`,
+        `  %bix1 = sub i32 %bix, 1`,
+        sz === "i32" ? `  %bi = add i32 %bix1, 0` : `  %bi = zext i32 %bix1 to ${sz}`,
+        `  %pe = getelementptr inbounds %ScrMapNumEntry, ptr %ents, ${sz} %bi`,
+        `  %pk = load i64, ptr %pe`,
+        `  %pkq = icmp eq i64 %pk, %kb`,
+        `  br i1 %pkq, label %probe.found, label %probe.next`,
+        `probe.found:`,
+        `  ret ptr %pe`,
+        `probe.next:`,
+        `  %pi.inc = add ${sz} %pi, 1`,
+        `  %pi.next = and ${sz} %pi.inc, %mask`,
+        `  br label %probe.loop`,
+        `miss:`,
+        `  ret ptr null`,
+        `runtime:`,
+        `  %r = call ptr @scr_map_entry_f64(ptr %m, double ${runtimeKey}) memory(read) nounwind willreturn`,
+        `  ret ptr %r`,
+        `}`,
+        ``,
+    ];
+}
 /** The shared abort helpers (emitted only when referenced): the OOM
  * abort of untraced shape allocation and the invalid-union-tag abort —
  * both print the runtime ABI’s exact message on fd 2 and abort. */
+/** The island module tree's install call before %main, and the constant
+ * that carries its files when the executable ships them itself
+ * (scr_node_island.c: scr_node_island_install_modules). */
+function nodeIslandModulesInstall(emitter, modules) {
+    emitter.declare(`declare void @scr_node_island_install_modules(ptr, ${emitter.sizeType}, ptr, ${emitter.sizeType}, ptr)`);
+    const manifest = nodeIslandManifest(modules);
+    const manifestPtr = emitter.cstr(manifest);
+    const location = emitter.cstr(modules.location ?? "");
+    const data = modules.location === undefined ? emitter.nodeIslandModuleData : null;
+    const globals = data === null || data.length === 0
+        ? []
+        : [
+            `@sc_node_island_modules = private unnamed_addr constant [${data.length} x i8] c"${llvmBytes(data, false)}", align 16`,
+        ];
+    const st = emitter.sizeType;
+    return {
+        call: `  call void @scr_node_island_install_modules(ptr ${manifestPtr}, ${st} ${Buffer.byteLength(manifest, "utf8")}, ptr ${globals.length > 0 ? "@sc_node_island_modules" : "null"}, ${st} ${globals.length > 0 ? data.length : 0}, ptr ${location})`,
+        globals,
+    };
+}
 export function helperDefs(emitter) {
     const defs = [];
     const msgHelper = (fnName, msgSym, msg) => {
@@ -1099,20 +1389,18 @@ export function helperDefs(emitter) {
         // the double `%k`'s exact value): the direct index tests the integer
         // itself, so the double is converted only by the hashed probe. A
         // negative key is not below the index's size as an unsigned value.
-        `define internal ptr @sc_map_entry_f64_int(ptr %m, i64 %ki, double %k) ${FN_ATTRS} {`, `entry:`, `  %ndp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 16`, `  %nd32 = load i32, ptr %ndp`, `  %nd = zext i32 %nd32 to i64`, `  %inside = icmp ult i64 %ki, %nd`, `  br i1 %inside, label %slot, label %outside`, `slot:`, sz === "i64" ? `  %i = add i64 %ki, 0` : `  %i = trunc i64 %ki to ${sz}`, `  %dp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 15`, `  %dense = load ptr, ptr %dp`, `  %sp = getelementptr inbounds i32, ptr %dense, ${sz} %i`, `  %s = load i32, ptr %sp`, `  %empty = icmp eq i32 %s, 0`, `  br i1 %empty, label %miss, label %hit`, `hit:`, `  %e32 = sub i32 %s, 1`, sz === "i32" ? `  %e = add i32 %e32, 0` : `  %e = zext i32 %e32 to ${sz}`, `  %ep = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 11`, `  %entries = load ptr, ptr %ep`, `  %found = getelementptr inbounds %ScrMapNumEntry, ptr %entries, ${sz} %e`, `  ret ptr %found`, `miss:`, `  ret ptr null`, `outside:`, `  %dense.any = icmp ne i64 %nd, 0`, `  br i1 %dense.any, label %miss, label %slow`, `slow:`, `  %hashed = call ptr @sc_map_entry_f64_hashed(ptr %m, double %k)`, `  ret ptr %hashed`, `}`, ``, 
-        // Hashing (scr_map_find_f64): the SameValueZero key word, its
-        // scr_key_mix hash (zero remapped to one) and the absent-key filter.
-        `define internal ptr @sc_map_entry_f64_hashed(ptr %m, double %k) noinline ${FN_ATTRS} {`, `entry:`, `  %nan = fcmp uno double %k, %k`, `  %zero = fcmp oeq double %k, 0.0`, `  %kz = select i1 %zero, double 0.0, double %k`, `  %kzb = bitcast double %kz to i64`, `  %kb = select i1 %nan, i64 9221120237041090560, i64 %kzb`, `  %x1s = lshr i64 %kb, 30`, `  %x1 = xor i64 %kb, %x1s`, `  %x2 = mul i64 %x1, -4658895280553007687`, `  %x3s = lshr i64 %x2, 27`, `  %x3 = xor i64 %x2, %x3s`, `  %x4 = mul i64 %x3, -7723592293110705685`, `  %x5s = lshr i64 %x4, 31`, `  %x5 = xor i64 %x4, %x5s`, `  %hz = icmp eq i64 %x5, 0`, `  %h = select i1 %hz, i64 1, i64 %x5`, `  %f1s = lshr i64 %h, 44`, `  %f1a = and i64 %f1s, 63`, `  %f1 = shl i64 1, %f1a`, `  %f2s = lshr i64 %h, 50`, `  %f2a = and i64 %f2s, 63`, `  %f2 = shl i64 1, %f2a`, `  %fb = or i64 %f1, %f2`, `  %fp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 10`, `  %filter = load i64, ptr %fp`, `  %fm = and i64 %filter, %fb`, `  %fin = icmp eq i64 %fm, %fb`, `  br i1 %fin, label %table, label %miss`, `table:`, `  %nbp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 13`, ...(sz === "i64"
-            ? [`  %nb32 = load i32, ptr %nbp`, `  %nb = zext i32 %nb32 to i64`]
-            : [`  %nb = load i32, ptr %nbp`]), `  %eps = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 11`, `  %ents = load ptr, ptr %eps`, `  %linear = icmp eq ${sz} %nb, 0`, `  br i1 %linear, label %scan, label %buckets`, 
-        // Small maps keep no bucket table: scan the entries' keys. A number-key
-        // entry has no cached hash; a dead entry's key matches no key.
-        `scan:`, `  %nep = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 8`, ...(sz === "i64"
-            ? [`  %ne32 = load i32, ptr %nep`, `  %ne = zext i32 %ne32 to i64`]
-            : [`  %ne = load i32, ptr %nep`]), `  br label %scan.loop`, `scan.loop:`, `  %si = phi ${sz} [ 0, %scan ], [ %si.next, %scan.next ]`, `  %sdone = icmp eq ${sz} %si, %ne`, `  br i1 %sdone, label %miss, label %scan.entry`, `scan.entry:`, `  %se = getelementptr inbounds %ScrMapNumEntry, ptr %ents, ${sz} %si`, `  %sk = load i64, ptr %se`, `  %skq = icmp eq i64 %sk, %kb`, `  br i1 %skq, label %scan.found, label %scan.next`, `scan.found:`, `  ret ptr %se`, `scan.next:`, `  %si.next = add ${sz} %si, 1`, `  br label %scan.loop`, 
-        // Compact tables tag each 24-bit entry index (+1) with the hash's
-        // top byte; wider tables take the runtime lookup.
-        `buckets:`, `  %wide = icmp ugt ${sz} %nb, 16777216`, `  br i1 %wide, label %runtime, label %probe`, `probe:`, `  %bp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 14`, `  %bk = load ptr, ptr %bp`, `  %mask = sub ${sz} %nb, 1`, `  %ht = lshr i64 %h, 32`, `  %ht32 = trunc i64 %ht to i32`, `  %tag = and i32 %ht32, -16777216`, sz === "i64" ? `  %hi = add i64 %h, 0` : `  %hi = trunc i64 %h to ${sz}`, `  %i0 = and ${sz} %hi, %mask`, `  br label %probe.loop`, `probe.loop:`, `  %pi = phi ${sz} [ %i0, %probe ], [ %pi.next, %probe.next ]`, `  %bep = getelementptr inbounds i32, ptr %bk, ${sz} %pi`, `  %b = load i32, ptr %bep`, `  %bempty = icmp eq i32 %b, 0`, `  br i1 %bempty, label %miss, label %probe.tag`, `probe.tag:`, `  %bt = and i32 %b, -16777216`, `  %btq = icmp eq i32 %bt, %tag`, `  br i1 %btq, label %probe.entry, label %probe.next`, `probe.entry:`, `  %bix = and i32 %b, 16777215`, `  %bix1 = sub i32 %bix, 1`, sz === "i32" ? `  %bi = add i32 %bix1, 0` : `  %bi = zext i32 %bix1 to ${sz}`, `  %pe = getelementptr inbounds %ScrMapNumEntry, ptr %ents, ${sz} %bi`, `  %pk = load i64, ptr %pe`, `  %pkq = icmp eq i64 %pk, %kb`, `  br i1 %pkq, label %probe.found, label %probe.next`, `probe.found:`, `  ret ptr %pe`, `probe.next:`, `  %pi.inc = add ${sz} %pi, 1`, `  %pi.next = and ${sz} %pi.inc, %mask`, `  br label %probe.loop`, `miss:`, `  ret ptr null`, `runtime:`, `  %r = call ptr @scr_map_entry_f64(ptr %m, double %k) memory(read) nounwind willreturn`, `  ret ptr %r`, `}`, ``);
+        `define internal ptr @sc_map_entry_f64_int(ptr %m, i64 %ki, double %k) ${FN_ATTRS} {`, `entry:`, `  %ndp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 16`, `  %nd32 = load i32, ptr %ndp`, `  %nd = zext i32 %nd32 to i64`, `  %inside = icmp ult i64 %ki, %nd`, `  br i1 %inside, label %slot, label %outside`, `slot:`, sz === "i64" ? `  %i = add i64 %ki, 0` : `  %i = trunc i64 %ki to ${sz}`, `  %dp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 15`, `  %dense = load ptr, ptr %dp`, `  %sp = getelementptr inbounds i32, ptr %dense, ${sz} %i`, `  %s = load i32, ptr %sp`, `  %empty = icmp eq i32 %s, 0`, `  br i1 %empty, label %miss, label %hit`, `hit:`, `  %e32 = sub i32 %s, 1`, sz === "i32" ? `  %e = add i32 %e32, 0` : `  %e = zext i32 %e32 to ${sz}`, `  %ep = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 11`, `  %entries = load ptr, ptr %ep`, `  %found = getelementptr inbounds %ScrMapNumEntry, ptr %entries, ${sz} %e`, `  ret ptr %found`, `miss:`, `  ret ptr null`, `outside:`, `  %dense.any = icmp ne i64 %nd, 0`, `  br i1 %dense.any, label %miss, label %slow`, `slow:`, `  %hashed = call ptr @sc_map_entry_f64_hashed_int(ptr %m, i64 %ki)`, `  ret ptr %hashed`, `}`, ``, 
+        // Hashing (scr_map_find_f64): the SameValueZero key word and its
+        // scr_key_mix hash. sc_map_entry_f64_int's probe converts the integer
+        // view to the key word, which is never NaN or -0, and so skips the
+        // normalization.
+        ...hashedNumberProbe("sc_map_entry_f64_hashed", "double %k", sz, [
+            `  %nan = fcmp uno double %k, %k`,
+            `  %zero = fcmp oeq double %k, 0.0`,
+            `  %kz = select i1 %zero, double 0.0, double %k`,
+            `  %kzb = bitcast double %kz to i64`,
+            `  %kb = select i1 %nan, i64 9221120237041090560, i64 %kzb`,
+        ], "%k"), ...hashedNumberProbe("sc_map_entry_f64_hashed_int", "i64 %ki", sz, [`  %kf = sitofp i64 %ki to double`, `  %kb = bitcast double %kf to i64`], "%kf"));
     }
     // Map/Set iteration steps read the entries array inline: an index is
     // live when it is below the entry count and its entry's cached hash is