@scriptc/compiler 0.4.6 → 0.4.8

package/dist/backend/llvm/emit-module.js

@@ -7,6 +7,7 @@ import { EmitJournal } from "./emit-journal.js";
 import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
+import { JS_GROUP_WORDS } from "./js-groups.js";
 import { JS_IC_WORDS } from "./js-ic.js";
 import { mangleAsyncSpawn, mangleClassObj, mangleFnClosure, mangleFunction, mangleGlobal, mangleVtInstance, } from "../mangle.js";
 import { RUNTIME_ERROR_CLASSES, RUNTIME_EVENT_CLASS, RUNTIME_EVENT_TARGET_CLASS, ffiCallbackType, isFfiContextParam, isRefCounted, typeKey, } from "../../ir/ir.js";
@@ -29,7 +30,7 @@ import { STRING_READ_HELPERS, stringScanDecl, stringScanInline } from "./string-
 import { emitLibraryIdentityLines } from "../library-identity-markers.js";
 import { InternalCompilerError } from "../../errors.js";
 import { targetCpuAttribute } from "../target-cpu.js";
-import { CPU_LEVEL_INIT_CALL, cpuDispatchAttributes, cpuLevelDefs } from "./cpu-dispatch.js";
+import { CPU_LEVEL_INIT_CALL, cpuCopyAttributes, cpuLevelDefs, targetLevelDefs, } from "./cpu-dispatch.js";
 import { layoutTierOf } from "./function-layout.js";
 /** The function bodies emitted in `chunks` contiguous ranges, each by its
  * own emitter (LlEmitter.chunkEmitter), then merged in function order: the
@@ -70,6 +71,14 @@ export function workerMinFunctions() {
     const value = Number(process.env["SCRIPTC_EMIT_WORKER_FUNCTIONS"] ?? "2000");
     return Number.isFinite(value) && value > 0 ? value : 2000;
 }
+/** Whether the worker for `range` exits failing on purpose, from
+ * SCRIPTC_EMIT_WORKER_FAIL (`all`, or one range's index). The self-hosting
+ * driver pins that a failed worker's range falls back to this process and
+ * still emits the serial text. */
+function workerFailureRequested(range) {
+    const requested = process.env["SCRIPTC_EMIT_WORKER_FAIL"];
+    return requested === "all" || requested === String(range);
+}
 /** A function's estimated emission cost beyond its IR nodes, in nodes:
  * fitted to the worker emission times of a large program's ranges. */
 const FUNCTION_EMIT_COST = 22;
@@ -119,6 +128,8 @@ function emitFunctionsOnWorkers(emitter, workers) {
         if (pid === 0) {
             let code = 1;
             try {
+                if (workerFailureRequested(range))
+                    throw new Error("requested worker failure");
                 emitter.journal = new EmitJournal();
                 emitter.debug?.record();
                 const texts = [];
@@ -170,7 +181,9 @@ function emitFunctionsOnWorkers(emitter, workers) {
                 else
                     reap(pid, range);
             }
-            const merged = ended.get(range);
+            // A failed range's worker, or one whose files are missing, left null:
+            // this process emits that range itself.
+            const merged = ended.get(range) ?? null;
             ended.delete(range);
             const slice = rangeFunctions(range);
             if (merged !== null && merged.texts.length === slice.length) {
@@ -212,8 +225,9 @@ function readRange(dir, range) {
  * and `uwtable` on native targets, so every function has an unwind table
  * entry, including those LLVM infers `nounwind` (profilers and debuggers
  * unwind through them). A group needs one attribute: without any, #0 holds a
- * string attribute that LLVM ignores. Dispatching builds add the x86-64-v3
- * copies' #3 and #4 (cpu-dispatch.ts). */
+ * string attribute that LLVM ignores. Builds with CPU copies add the x86-64-v3
+ * copies' #3 and #4, and for an x86-64-v3 target the baseline copies' #5 and
+ * #6 (cpu-dispatch.ts). */
 function functionAttributes(emitter, hasNoInlineRecordClone) {
     const cpu = targetCpuAttribute(emitter.targetCpu);
     const asan = emitter.sanitizeAttributes ? " sanitize_address" : "";
@@ -223,7 +237,7 @@ function functionAttributes(emitter, hasNoInlineRecordClone) {
         `attributes #0 = {${base} }`,
         ...(emitter.usesCoroutines ? [`attributes #1 = {${common} presplitcoroutine${cpu} }`] : []),
         ...(hasNoInlineRecordClone ? [`attributes #2 = { noinline${common}${cpu} }`] : []),
-        ...(emitter.cpuDispatch ? cpuDispatchAttributes(common) : []),
+        ...(emitter.cpuCopies === null ? [] : cpuCopyAttributes(common, emitter.cpuCopies)),
     ];
 }
 /** The group line written before each function's text. Functions from one
@@ -409,6 +423,10 @@ export function emitModuleParts(emitter) {
     const usesEvents = runtimeFeatures.processEvents;
     const usesChildProcess = runtimeFeatures.childProcess;
     const usesFsWatch = runtimeFeatures.fsWatch;
+    // fs.watchFile programs register the poller unit's exit cleanup — the
+    // same one-line gating (native/executable.ts links scr_watch_file.c on
+    // the same predicate).
+    const usesFsWatchFile = runtimeFeatures.fsWatchFile;
     // Programs that call into Node record the libnode pack and its Node
     // version before %main (scr_node_island.c).
     const usesNodeIsland = runtimeFeatures.nodeIsland;
@@ -431,6 +449,16 @@ export function emitModuleParts(emitter) {
     const usesFetch = runtimeFeatures.fetch;
     // URL programs give the host parser the IDNA unit's ToASCII.
     const usesUrlIdna = runtimeFeatures.urlIdna;
+    // Programs whose checked-dynamic reads answer Buffer.prototype methods by
+    // name fill scr_bytes_dyn.c's member hook; the hook's table links every
+    // scr_dyn_invoke_buffer entry, so only these programs install it.
+    const usesDynBufferMembers = runtimeFeatures.dynBufferMembers;
+    // Buffer.prototype as a value registers Symbol(nodejs.util.inspect.custom)
+    // through the symbol unit's Symbol.for, which only these programs install.
+    const usesBufferPrototypeObject = runtimeFeatures.bufferPrototypeObject;
+    // Programs that read Symbol.toPrimitive install ToNumber's
+    // Symbol.toPrimitive step; the others do not link it.
+    const usesToPrimitiveNumber = runtimeFeatures.toPrimitiveNumber;
     const snapshotsTlsCa = runtimeFeatures.tls || runtimeFeatures.tlsCa;
     // The process verdict has the same precedence as the C reference
     // emitter: node:test owns the final status when present; otherwise an
@@ -470,6 +498,8 @@ export function emitModuleParts(emitter) {
         emitter.declare(`declare void @scr_worker_stdio_install(ptr, ptr, ptr)`);
     if (usesFsWatch)
         emitter.declare(`declare void @scr_watch_install()`);
+    if (usesFsWatchFile)
+        emitter.declare(`declare void @scr_watch_file_install()`);
     if (usesNodeIsland)
         emitter.declare(`declare void @scr_node_island_install(ptr, ptr)`);
     // Interned now: the C-string constants flush before main assembles.
@@ -503,6 +533,12 @@ export function emitModuleParts(emitter) {
         emitter.declare(`declare void @scr_fetch_install()`);
     if (usesUrlIdna)
         emitter.declare(`declare void @scr_url_idna_install()`);
+    if (usesDynBufferMembers)
+        emitter.declare(`declare void @scr_bytes_buffer_members_install()`);
+    if (usesBufferPrototypeObject)
+        emitter.declare(`declare void @scr_sym_for_install()`);
+    if (usesToPrimitiveNumber)
+        emitter.declare(`declare void @scr_dyn_to_primitive_number_install()`);
     if (usesNodeTest) {
         emitter.declare(`declare i32 @scr_test_exit_code()`);
         emitter.declare(`declare void @scr_test_init()`);
@@ -666,7 +702,11 @@ export function emitModuleParts(emitter) {
         ...(emitter.publishWalkersCache?.defs() ?? []),
         ...emitter.publishedClassNamesDefs(),
         ...(emitter.regions?.moduleDefs() ?? []),
-        ...(emitter.cpuDispatch ? cpuLevelDefs((line) => emitter.declare(line)) : []),
+        ...(emitter.cpuDispatch
+            ? cpuLevelDefs((line) => emitter.declare(line))
+            : emitter.cpuCopies === "target"
+                ? targetLevelDefs()
+                : []),
     ];
     if (emitter.constantNumericTables.size > 0)
         emitter.declare(`declare double @scr_arr_get_number(ptr, double)`);
@@ -817,7 +857,10 @@ export function emitModuleParts(emitter) {
     const tl = emitter.mod.workers === true || emitter.mod.lib?.threadInstances === true
         ? "thread_local "
         : "";
-    out.push(``, `@scr_error_vts = external ${tl}global [${RUNTIME_ERROR_CLASSES.size} x %ScrVt]`, `declare void @scr_init()`, ...(emitter.compactRegion ? [`declare void @scr_plr_init()`] : []), `declare void @scr_lib_init(i32, ptr)`, ...(emitter.mod.workers ? [`declare void @scr_runtime_workers_v8()`] : []), ...(emitter.stackChecks?.size ? [`declare void @scr_stack_guard_init()`] : []), ...(emitter.runtimeAbiMarker && emitter.mod.lib === undefined
+    out.push(``, `@scr_error_vts = external ${tl}global [${RUNTIME_ERROR_CLASSES.size} x %ScrVt]`, `declare void @scr_init()`, ...(emitter.compactRegion ? [`declare void @scr_plr_init()`] : []), 
+    // A compact field holding an array, a map, a set or a heap instance
+    // (emitter.windowRefs) keeps address >> 3 in 32 bits.
+    ...(emitter.windowRefs ? [`declare void @scr_sa_require_window()`] : []), `declare void @scr_lib_init(i32, ptr)`, ...(emitter.mod.workers ? [`declare void @scr_runtime_workers_v8()`] : []), ...(emitter.stackChecks?.size ? [`declare void @scr_stack_guard_init()`] : []), ...(emitter.runtimeAbiMarker && emitter.mod.lib === undefined
         ? [`declare void @${RUNTIME_ABI_MARKER}()`]
         : []), 
     // Overrides the runtime's weak default (scr_lib.c).
@@ -837,6 +880,10 @@ export function emitModuleParts(emitter) {
             out.push(`@sc_dyn_read_ic_${site} = internal ${tl}global { ptr, ${emitter.sizeType}, ${emitter.sizeType}, ptr } zeroinitializer`);
         for (let site = 0; site < emitter.jsIcSiteCount; site++)
             out.push(`@sc_js_ic_${site} = internal ${tl}global [${JS_IC_WORDS} x i64] zeroinitializer`);
+        for (let group = 0; group < emitter.jsGroupCount; group++)
+            out.push(`@sc_js_group_${group} = internal ${tl}global [${JS_GROUP_WORDS} x i64] zeroinitializer`);
+        for (const [name, keys] of emitter.jsGroupKeysTable)
+            out.push(`${name} = internal constant [${keys.length} x ptr] [${keys.map((key) => `ptr ${key}`).join(", ")}]`);
     }
     if (nodeIslandModulesCall !== null)
         out.push(...nodeIslandModulesCall.globals);
@@ -949,7 +996,9 @@ export function emitModuleParts(emitter) {
         const wide = emitter.sizeType === "i64";
         const slot = wide ? "ptr" : "<{ ptr, i32 }>";
         const slotValue = (s) => wide ? `ptr ${s}` : `<{ ptr, i32 }> <{ ptr ${s}, i32 0 }>`;
-        out.push(`@${inst.sym}_data = internal constant [${n} x ${slot}] [ ${inst.slots.map(slotValue).join(", ")} ], align 8`, `@${inst.sym} = internal global %ScrArr { ${emitter.sizeType} -1, i32 ${n}, i32 ${n}, i8 2, i8 0, i16 0, i32 ${n}, ptr @${inst.sym}_data }`);
+        out.push(`@${inst.sym}_data = internal constant [${n} x ${slot}] [ ${inst.slots.map(slotValue).join(", ")} ], align 8`, `@${inst.sym} = internal global %ScrArr { ${emitter.sizeType} -1, i32 ${n}, i32 ${n}, i8 2, i8 0, i16 0, i32 ${n}, ptr @${inst.sym}_data }`, 
+        // The site's heap copy of the header (expr-primitives.ts).
+        ...(emitter.compactRegion ? [`@${inst.sym}_win = internal global ptr null, align 8`] : []));
     }
     if (emitter.templateStringsInstances.size > 0)
         out.push(``);
@@ -1057,10 +1106,15 @@ export function emitModuleParts(emitter) {
             out.push(line);
         for (const line of emitter.inlineRcTail(flushedDecls))
             out.push(line);
-        out.push(...functionAttributes(emitter, hasNoInlineRecordClone));
+        // A metadata table for a large program holds more lines than one call
+        // can take as arguments: append each table's lines one at a time.
+        for (const line of functionAttributes(emitter, hasNoInlineRecordClone))
+            out.push(line);
         out.push(``);
-        out.push(...emitter.fieldAliasMetadata());
-        out.push(...loopUnrollMetadata(out));
+        for (const line of emitter.fieldAliasMetadata())
+            out.push(line);
+        for (const line of loopUnrollMetadata(out))
+            out.push(line);
         if (emitter.debug !== null)
             out.push(emitter.debug.render());
         return out;
@@ -1069,6 +1123,9 @@ export function emitModuleParts(emitter) {
     // Compressed fields hold addresses in the program-lifetime region, which
     // is reserved below 32 GiB before compiled code runs.
     ...(emitter.compactRegion ? [`  call void @scr_plr_init()`] : []), 
+    // A reference block the window cannot serve ends the process here rather
+    // than reaching a compact field, which would truncate its address.
+    ...(emitter.windowRefs ? [`  call void @scr_sa_require_window()`] : []), 
     // Each context (the main thread, every worker) installs its stack
     // guard before running compiled code that checks it.
     ...(emitter.stackChecks?.size ? [`  call void @scr_stack_guard_init()`] : []), ...stamps, 
@@ -1086,11 +1143,14 @@ export function emitModuleParts(emitter) {
         : []), 
     // fs.watch programs fill the loop's watch hooks the same way —
     // scr_watch.c links only when this line is emitted.
-    ...(usesFsWatch ? [`  call void @scr_watch_install()`] : []), ...(nodeIslandConfig
+    ...(usesFsWatch ? [`  call void @scr_watch_install()`] : []), 
+    // fs.watchFile programs register the poller unit's atexit cleanup the
+    // same way — scr_watch_file.c links only when this line is emitted.
+    ...(usesFsWatchFile ? [`  call void @scr_watch_file_install()`] : []), ...(nodeIslandConfig
         ? [
             `  call void @scr_node_island_install(ptr ${nodeIslandConfig[0]}, ptr ${nodeIslandConfig[1]})`,
         ]
-        : []), ...(nodeIslandModulesCall === null ? [] : [nodeIslandModulesCall.call]), ...(emitter.ffiHasForeignCallback ? [`  call void @scr_ffi_install()`] : []), ...(snapshotsTlsCa ? [`  call void @scr_tls_ca_install()`] : []), ...(usesFetch ? [`  call void @scr_fetch_install()`] : []), ...(usesUrlIdna ? [`  call void @scr_url_idna_install()`] : []), ...(usesNet ? [`  call void @scr_net_install()`, `  call void @scr_net_dyn_install()`] : []), ...(usesNetLookup ? [`  call void @scr_net_lookup_install()`] : []), ...(usesHttp ? [`  call void @scr_http_dyn_install()`] : []), ...(usesDgram ? [`  call void @scr_dgram_install()`] : []), ...(usesHttp2 ? [`  call void @scr_http2_dyn_install()`] : []), ...(usesStream ? [`  call void @scr_stream_install()`] : []), `  call void @scr_lib_init(i32 %argc, ptr %argv)`, 
+        : []), ...(nodeIslandModulesCall === null ? [] : [nodeIslandModulesCall.call]), ...(emitter.ffiHasForeignCallback ? [`  call void @scr_ffi_install()`] : []), ...(snapshotsTlsCa ? [`  call void @scr_tls_ca_install()`] : []), ...(usesFetch ? [`  call void @scr_fetch_install()`] : []), ...(usesUrlIdna ? [`  call void @scr_url_idna_install()`] : []), ...(usesDynBufferMembers ? [`  call void @scr_bytes_buffer_members_install()`] : []), ...(usesBufferPrototypeObject ? [`  call void @scr_sym_for_install()`] : []), ...(usesToPrimitiveNumber ? [`  call void @scr_dyn_to_primitive_number_install()`] : []), ...(usesNet ? [`  call void @scr_net_install()`, `  call void @scr_net_dyn_install()`] : []), ...(usesNetLookup ? [`  call void @scr_net_lookup_install()`] : []), ...(usesHttp ? [`  call void @scr_http_dyn_install()`] : []), ...(usesDgram ? [`  call void @scr_dgram_install()`] : []), ...(usesHttp2 ? [`  call void @scr_http2_dyn_install()`] : []), ...(usesStream ? [`  call void @scr_stream_install()`] : []), `  call void @scr_lib_init(i32 %argc, ptr %argv)`, 
     // node:test takes Node's --test-update-snapshots out of argv first.
     ...(usesNodeTest ? [`  call void @scr_test_init()`] : []), ...(asyncEntry
         ? [`  %top = call ptr @${mangleAsyncSpawn(emitter.mod.entry)}()`]
@@ -1184,47 +1244,152 @@ export function emitModuleParts(emitter) {
         : [`  %native_exit = call i32 @scr_exit_code_hint_get()`, `  ret i32 %native_exit`]), `}`, ``);
     for (const line of emitter.inlineRcTail(flushedDecls))
         out.push(line);
-    out.push(
-    // The sanitized lane's -fsanitize=address link instruments the
-    // functions marked sanitize_address (the runtime TUs get theirs from
-    // clang).
-    ...functionAttributes(emitter, hasNoInlineRecordClone), ``);
+    // The sanitized lane's -fsanitize=address link instruments the functions
+    // marked sanitize_address (the runtime TUs get theirs from clang).
+    for (const line of functionAttributes(emitter, hasNoInlineRecordClone))
+        out.push(line);
+    out.push(``);
     if (emitter.mod.workers) {
         out.push(`declare void @scr_context_cleanup()`, `declare i32 @scr_worker_argc()`, `declare ptr @scr_worker_argv()`, `define i32 @sc_worker_entry(i32 %root) ${FN_ATTRS} {`, `entry:`, `  %argc = call i32 @scr_worker_argc()`, `  %argv = call ptr @scr_worker_argv()`, `  %result = call i32 @sc_context_entry(i32 %argc, ptr %argv)`, `  ret i32 %result`, `}`, `define i32 @main(i32 %argc, ptr %argv) ${FN_ATTRS} {`, `entry:`, 
         // Worker threads enter at sc_worker_entry and read the level main set.
         ...(emitter.cpuDispatch ? [CPU_LEVEL_INIT_CALL] : []), `  %result = call i32 @sc_context_entry(i32 %argc, ptr %argv)`, `  call void @scr_context_cleanup()`, `  ret i32 %result`, `}`);
     }
-    out.push(...emitter.fieldAliasMetadata());
-    out.push(...loopUnrollMetadata(out));
+    for (const line of emitter.fieldAliasMetadata())
+        out.push(line);
+    for (const line of loopUnrollMetadata(out))
+        out.push(line);
     if (emitter.debug !== null)
         out.push(emitter.debug.render());
     return out;
 }
-/** The out-of-line hashed probe of a number-key map (scr_map_find_f64):
- * `keyLines` define the normalized key word %kb from the parameter, and
- * `runtimeKey` is the double passed to the runtime lookup. A map with at
- * most SCR_MAP_LINEAR_LIMIT entries keeps no bucket table: the absent-key
- * filter is tested before its entries are scanned. A bucket table is
- * probed without the filter: the filter keeps two of its 64 bits per key,
- * so past a few dozen keys it rejects few absent keys, and the probe's
- * first bucket read answers those. Compact tables tag each 24-bit entry
- * index (+1) with the hash's top byte; wider tables take the runtime
- * lookup. */
-function hashedNumberProbe(name, param, sz, keyLines, runtimeKey) {
+/** The `.loop` lookup helpers only read memory and always return, as
+ * declared: LLVM then merges two lookups of one key (a has() followed by
+ * get()) before it inlines them. */
+const INLINED_LOOKUP_ATTRS = "alwaysinline memory(read) nounwind willreturn";
+/** The number-key entry lookups (sc_map_entry_f64 and sc_map_entry_f64_int)
+ * and their hashed probes. With `loop` they are the `.loop` flavor, which
+ * lookups inside a loop call (emitNumberMapEntry): each of its functions is
+ * inlined, so a lookup that finds its key in a bucket table makes no call,
+ * and the loop keeps no argument moves or spills around one. */
+function numberMapEntryHelpers(sz, loop) {
+    const flavor = loop ? ".loop" : "";
+    const attrs = loop ? `${INLINED_LOOKUP_ATTRS} ${FN_ATTRS}` : FN_ATTRS;
+    let mapTmp = 0;
+    const mapIndex = exactIndexLines("%k", "%nd", sz, () => `%ix${mapTmp++}`);
     return [
-        `define internal ptr @${name}(ptr %m, ${param}) noinline ${FN_ATTRS} {`,
+        `define internal ptr @sc_map_entry_f64${flavor}(ptr %m, double %k) ${attrs} {`,
+        `entry:`,
+        `  %ndp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 16`,
+        ...(sz === "i64"
+            ? [`  %nd32 = load i32, ptr %ndp`, `  %nd = zext i32 %nd32 to i64`]
+            : [`  %nd = load i32, ptr %ndp`]),
+        // A non-integral key inside the table is as absent as one outside it.
+        ...mapIndex.lines.map((line) => `  ${line}`),
+        `  br i1 ${mapIndex.ok}, label %slot, label %slow`,
+        `slot:`,
+        sz === "i64"
+            ? `  %i = add i64 ${mapIndex.wide}, 0`
+            : `  %i = trunc i64 ${mapIndex.wide} to ${sz}`,
+        `  %dp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 15`,
+        `  %dense = load ptr, ptr %dp`,
+        `  %sp = getelementptr inbounds i32, ptr %dense, ${sz} %i`,
+        `  %s = load i32, ptr %sp`,
+        `  %empty = icmp eq i32 %s, 0`,
+        `  br i1 %empty, label %miss, label %hit`,
+        `hit:`,
+        `  %e32 = sub i32 %s, 1`,
+        sz === "i32" ? `  %e = add i32 %e32, 0` : `  %e = zext i32 %e32 to ${sz}`,
+        `  %ep = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 11`,
+        `  %entries = load ptr, ptr %ep`,
+        `  %found = getelementptr inbounds %ScrMapNumEntry, ptr %entries, ${sz} %e`,
+        `  ret ptr %found`,
+        `miss:`,
+        `  ret ptr null`,
+        `slow:`,
+        `  %hashed = call ptr @sc_map_entry_f64_hashed${flavor}(ptr %m, double %k)`,
+        `  ret ptr %hashed`,
+        `}`,
+        ``,
+        // The same lookup for a key the emitter knows is an integer (`%ki`,
+        // the double `%k`'s exact value): the direct index tests the integer
+        // itself, so the double is converted only by the hashed probe. A
+        // negative key is not below the index's size as an unsigned value.
+        `define internal ptr @sc_map_entry_f64_int${flavor}(ptr %m, i64 %ki, double %k) ${attrs} {`,
+        `entry:`,
+        `  %ndp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 16`,
+        `  %nd32 = load i32, ptr %ndp`,
+        `  %nd = zext i32 %nd32 to i64`,
+        `  %inside = icmp ult i64 %ki, %nd`,
+        `  br i1 %inside, label %slot, label %slow`,
+        `slot:`,
+        sz === "i64" ? `  %i = add i64 %ki, 0` : `  %i = trunc i64 %ki to ${sz}`,
+        `  %dp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 15`,
+        `  %dense = load ptr, ptr %dp`,
+        `  %sp = getelementptr inbounds i32, ptr %dense, ${sz} %i`,
+        `  %s = load i32, ptr %sp`,
+        `  %empty = icmp eq i32 %s, 0`,
+        `  br i1 %empty, label %miss, label %hit`,
+        `hit:`,
+        `  %e32 = sub i32 %s, 1`,
+        sz === "i32" ? `  %e = add i32 %e32, 0` : `  %e = zext i32 %e32 to ${sz}`,
+        `  %ep = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 11`,
+        `  %entries = load ptr, ptr %ep`,
+        `  %found = getelementptr inbounds %ScrMapNumEntry, ptr %entries, ${sz} %e`,
+        `  ret ptr %found`,
+        `miss:`,
+        `  ret ptr null`,
+        `slow:`,
+        `  %hashed = call ptr @sc_map_entry_f64_hashed_int${flavor}(ptr %m, i64 %ki, double %k)`,
+        `  ret ptr %hashed`,
+        `}`,
+        ``,
+        // Hashing (scr_map_find_f64): the SameValueZero key word and its
+        // scr_key_word_hash mix. sc_map_entry_f64_int's probe takes the key's
+        // double with its integer view: its bits are the key word, never NaN,
+        // and +0 for the integer 0. The caller converts the integer: cvtsi2sd
+        // writes only the low half of its register, so in a function entry,
+        // where LLVM cannot clear the register first, it waits for the
+        // register's last value, in a loop of lookups often a value the
+        // previous iteration loaded.
+        ...hashedNumberProbe(`sc_map_entry_f64_hashed${flavor}`, loop, "double %k", sz, [
+            `  %kbits = bitcast double %k to i64`,
+            `  %kmag = and i64 %kbits, 9223372036854775807`,
+            `  %isnan = icmp ugt i64 %kmag, 9218868437227405312`,
+            `  %iszero = icmp eq i64 %kmag, 0`,
+            `  %ksn = select i1 %isnan, i64 9221120237041090560, i64 %kbits`,
+            `  %kb = select i1 %iszero, i64 0, i64 %ksn`,
+        ], "%k"),
+        ...hashedNumberProbe(`sc_map_entry_f64_hashed_int${flavor}`, loop, "i64 %ki, double %k", sz, [
+            `  %kzero = icmp eq i64 %ki, 0`,
+            `  %kbits = bitcast double %k to i64`,
+            `  %kb = select i1 %kzero, i64 0, i64 %kbits`,
+        ], "%k"),
+    ];
+}
+/** The hashed probe of a number-key map (scr_map_find_f64), out of line,
+ * or inlined with `loop`: `keyLines` define the normalized key word %kb
+ * from the parameter, and `runtimeKey` is the double passed to the runtime
+ * lookup. A map with at most SCR_MAP_LINEAR_LIMIT entries keeps no bucket
+ * table, and neither does a map with the direct index, whose lookups reach
+ * the probe only for a key outside the index, which is absent. In a small
+ * map the absent-key filter is tested before its entries are scanned.
+ * A bucket table is probed without the filter: the filter keeps two of its
+ * 64 bits per key, so past a few dozen keys it rejects few absent keys, and
+ * the probe's first bucket read answers those. Compact tables tag each
+ * 24-bit entry index (+1) with the hash's top byte; wider tables take the
+ * runtime lookup. */
+function hashedNumberProbe(name, loop, param, sz, keyLines, runtimeKey) {
+    return [
+        `define internal ptr @${name}(ptr %m, ${param}) ${loop ? INLINED_LOOKUP_ATTRS : "noinline"} ${FN_ATTRS} {`,
         `entry:`,
         ...keyLines,
-        `  %x1s = lshr i64 %kb, 30`,
+        `  %x1s = lshr i64 %kb, 32`,
         `  %x1 = xor i64 %kb, %x1s`,
-        `  %x2 = mul i64 %x1, -4658895280553007687`,
-        `  %x3s = lshr i64 %x2, 27`,
+        `  %x2 = mul i64 %x1, -7046029254386353131`,
+        `  %x3s = lshr i64 %x2, 29`,
         `  %x3 = xor i64 %x2, %x3s`,
-        `  %x4 = mul i64 %x3, -7723592293110705685`,
-        `  %x5s = lshr i64 %x4, 31`,
-        `  %x5 = xor i64 %x4, %x5s`,
-        `  %hz = icmp eq i64 %x5, 0`,
-        `  %h = select i1 %hz, i64 1, i64 %x5`,
+        `  %hz = icmp eq i64 %x3, 0`,
+        `  %h = select i1 %hz, i64 1, i64 %x3`,
         `  %nbp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 13`,
         ...(sz === "i64"
             ? [`  %nb32 = load i32, ptr %nbp`, `  %nb = zext i32 %nb32 to i64`]
@@ -1232,7 +1397,14 @@ function hashedNumberProbe(name, param, sz, keyLines, runtimeKey) {
         `  %eps = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 11`,
         `  %ents = load ptr, ptr %eps`,
         `  %linear = icmp eq ${sz} %nb, 0`,
-        `  br i1 %linear, label %filter, label %buckets`,
+        `  br i1 %linear, label %unbucketed, label %buckets`,
+        // A map without a bucket table either indexes its keys directly, and
+        // then holds none outside the index, or scans its entries.
+        `unbucketed:`,
+        `  %ndp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 16`,
+        `  %nd = load i32, ptr %ndp`,
+        `  %indexed = icmp ne i32 %nd, 0`,
+        `  br i1 %indexed, label %miss, label %filter`,
         `filter:`,
         `  %f1s = lshr i64 %h, 44`,
         `  %f1a = and i64 %f1s, 63`,
@@ -1366,42 +1538,19 @@ export function helperDefs(emitter) {
     }
     // Number-key Map/Set lookups probe the runtime's direct index inline:
     // while ndense is nonzero every live key is an integer below it, so an
-    // in-range integral key reads its slot (entry index + 1, 0 = absent) and
-    // any other key is absent. Maps without the index (ndense 0) hash the
-    // key in sc_map_entry_f64_hashed, which scans the entries of a small map
-    // or probes a compact bucket table as scr_map_find_f64 does, and calls
-    // the runtime lookup for a wide table. It stays out of line so that the
-    // direct-index path stays small at every call site. Both only read
-    // memory, so LLVM merges lookups of one key when nothing is stored in
-    // between (a has() followed by get(), for example).
-    if (emitter.decls.has(NUMBER_MAP_ENTRY_DECL)) {
-        const sz = emitter.sizeType;
-        let mapTmp = 0;
-        const mapIndex = exactIndexLines("%k", "%nd", sz, () => `%ix${mapTmp++}`);
-        defs.push(`define internal ptr @sc_map_entry_f64(ptr %m, double %k) ${FN_ATTRS} {`, `entry:`, `  %ndp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 16`, ...(sz === "i64"
-            ? [`  %nd32 = load i32, ptr %ndp`, `  %nd = zext i32 %nd32 to i64`]
-            : [`  %nd = load i32, ptr %ndp`]), 
-        // A non-integral key inside the table is as absent as one outside it.
-        ...mapIndex.lines.map((line) => `  ${line}`), `  br i1 ${mapIndex.ok}, label %slot, label %outside`, `slot:`, sz === "i64"
-            ? `  %i = add i64 ${mapIndex.wide}, 0`
-            : `  %i = trunc i64 ${mapIndex.wide} to ${sz}`, `  %dp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 15`, `  %dense = load ptr, ptr %dp`, `  %sp = getelementptr inbounds i32, ptr %dense, ${sz} %i`, `  %s = load i32, ptr %sp`, `  %empty = icmp eq i32 %s, 0`, `  br i1 %empty, label %miss, label %hit`, `hit:`, `  %e32 = sub i32 %s, 1`, sz === "i32" ? `  %e = add i32 %e32, 0` : `  %e = zext i32 %e32 to ${sz}`, `  %ep = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 11`, `  %entries = load ptr, ptr %ep`, `  %found = getelementptr inbounds %ScrMapNumEntry, ptr %entries, ${sz} %e`, `  ret ptr %found`, `miss:`, `  ret ptr null`, `outside:`, `  %dense.any = icmp ne ${sz} %nd, 0`, `  br i1 %dense.any, label %miss, label %slow`, `slow:`, `  %hashed = call ptr @sc_map_entry_f64_hashed(ptr %m, double %k)`, `  ret ptr %hashed`, `}`, ``, 
-        // The same lookup for a key the emitter knows is an integer (`%ki`,
-        // the double `%k`'s exact value): the direct index tests the integer
-        // itself, so the double is converted only by the hashed probe. A
-        // negative key is not below the index's size as an unsigned value.
-        `define internal ptr @sc_map_entry_f64_int(ptr %m, i64 %ki, double %k) ${FN_ATTRS} {`, `entry:`, `  %ndp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 16`, `  %nd32 = load i32, ptr %ndp`, `  %nd = zext i32 %nd32 to i64`, `  %inside = icmp ult i64 %ki, %nd`, `  br i1 %inside, label %slot, label %outside`, `slot:`, sz === "i64" ? `  %i = add i64 %ki, 0` : `  %i = trunc i64 %ki to ${sz}`, `  %dp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 15`, `  %dense = load ptr, ptr %dp`, `  %sp = getelementptr inbounds i32, ptr %dense, ${sz} %i`, `  %s = load i32, ptr %sp`, `  %empty = icmp eq i32 %s, 0`, `  br i1 %empty, label %miss, label %hit`, `hit:`, `  %e32 = sub i32 %s, 1`, sz === "i32" ? `  %e = add i32 %e32, 0` : `  %e = zext i32 %e32 to ${sz}`, `  %ep = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 11`, `  %entries = load ptr, ptr %ep`, `  %found = getelementptr inbounds %ScrMapNumEntry, ptr %entries, ${sz} %e`, `  ret ptr %found`, `miss:`, `  ret ptr null`, `outside:`, `  %dense.any = icmp ne i64 %nd, 0`, `  br i1 %dense.any, label %miss, label %slow`, `slow:`, `  %hashed = call ptr @sc_map_entry_f64_hashed_int(ptr %m, i64 %ki)`, `  ret ptr %hashed`, `}`, ``, 
-        // Hashing (scr_map_find_f64): the SameValueZero key word and its
-        // scr_key_mix hash. sc_map_entry_f64_int's probe converts the integer
-        // view to the key word, which is never NaN or -0, and so skips the
-        // normalization.
-        ...hashedNumberProbe("sc_map_entry_f64_hashed", "double %k", sz, [
-            `  %nan = fcmp uno double %k, %k`,
-            `  %zero = fcmp oeq double %k, 0.0`,
-            `  %kz = select i1 %zero, double 0.0, double %k`,
-            `  %kzb = bitcast double %kz to i64`,
-            `  %kb = select i1 %nan, i64 9221120237041090560, i64 %kzb`,
-        ], "%k"), ...hashedNumberProbe("sc_map_entry_f64_hashed_int", "i64 %ki", sz, [`  %kf = sitofp i64 %ki to double`, `  %kb = bitcast double %kf to i64`], "%kf"));
-    }
+    // in-range integral key reads its slot (entry index + 1, 0 = absent).
+    // Any other key goes to sc_map_entry_f64_hashed, which probes a compact
+    // bucket table as scr_map_find_f64 does, calls the runtime lookup for a
+    // wide table, and in a map without a bucket table answers absent when the
+    // map has the index (a map with buckets keeps no index, scr_map.h) and
+    // scans the entries otherwise. A lookup in a hashed map so tests the
+    // index once. The probe stays out of line so that the direct-index path
+    // stays small at every call site, except in the `.loop` flavor
+    // (numberMapEntryHelpers). Both only read memory, so LLVM merges lookups
+    // of one key when nothing is stored in between (a has() followed by
+    // get(), for example).
+    if (emitter.decls.has(NUMBER_MAP_ENTRY_DECL))
+        defs.push(...numberMapEntryHelpers(emitter.sizeType, false), ...numberMapEntryHelpers(emitter.sizeType, true));
     // Map/Set iteration steps read the entries array inline: an index is
     // live when it is below the entry count and its entry's cached hash is
     // nonzero (a tombstone's is zero), as scr_map_iter_live decides, or for