@scriptc/compiler 0.4.5 → 0.4.6

package/dist/backend/llvm/integer-clones.js

@@ -2,6 +2,7 @@ import { withinInt32 } from "../../ir/integer-ranges.js";
 import { mangleFunction } from "../mangle.js";
 import { BlockBuilder } from "./blocks.js";
 import { SCR_ARR_FIELD } from "./common.js";
+import { SCR_ELEM_F64, SCR_ELEM_I32 } from "./dense-array-access.js";
 export const NO_INTEGER_CLONES = {
     members: new Set(),
     dispatched: new Set(),
@@ -15,16 +16,28 @@ const BAIL = `${CLONE_RESULT} { i32 poison, i1 true }`;
 /** The stack one replaced recursion level is charged, in bytes. Ordinary
  * frames of these functions are about that size. */
 const VIRTUAL_FRAME = 32;
-/** The single-threaded stack guard, declared as the emitter's ordinary
- * stack checks declare it (clones only exist without workers). */
-export const STACK_GUARD_DECL = "@scr_stack_guard = external global ptr";
+/** Iterations between the polls of a loop over plain tail calls in worker
+ * executables (emitLoopPoll). */
+const POLL_INTERVAL = 64;
+/** The stack guard and the termination word, declared as the emitter's
+ * ordinary stack checks and polls declare them: process-wide in programs
+ * without workers, per thread in ELF worker executables (the only worker
+ * programs with clones). */
+export function stackGuardDecl(workers) {
+    return `@scr_stack_guard = external ${workers ? "thread_local " : ""}global ptr`;
+}
+const EXC_FLAG_DECL = "@scr_exc_flag = external thread_local(localexec) global i32, align 4";
 export const INTEGER_CLONE_SUFFIX = ".int";
 export const INTEGER_DISPATCH_SUFFIX = ".spec";
 function scalarKind(type) {
     return type.kind === "f64" ? "num" : type.kind === "bool" ? "bool" : null;
 }
-function valueKind(type) {
-    return type.kind === "array" && type.elem.kind === "f64" ? "arr" : scalarKind(type);
+function valueKind(type, refs) {
+    if (type.kind === "array" && type.elem.kind === "f64")
+        return "arr";
+    if (refs?.reference(type) === true)
+        return "ref";
+    return scalarKind(type);
 }
 /** The LLVM type of a value of `kind` inside a clone. */
 function kindType(kind) {
@@ -34,7 +47,7 @@ function isInt32Literal(value) {
     return (Number.isInteger(value) && value >= -0x80000000 && value <= 0x7fffffff && !Object.is(value, -0));
 }
 /** The call targets of `fn` when its body has the qualifying shape. */
-function cloneShape(fn, borrowed) {
+function cloneShape(fn, borrowed, refs) {
     if (fn.async || fn.generator !== undefined || fn.captures !== undefined)
         return null;
     if (fn.classCaptures !== undefined && fn.classCaptures.length > 0)
@@ -43,15 +56,18 @@ function cloneShape(fn, borrowed) {
     if (result === null)
         return null;
     const locals = new Map(fn.locals.map((l) => [l.id, l]));
-    // Array parameters are only read: never assigned, and their locals are
-    // the parameters themselves.
+    // Array and reference parameters are only read: never assigned, and their
+    // locals are the parameters themselves.
     const arrays = new Set();
-    for (const [index, p] of fn.params.entries())
-        if (valueKind(p.type) === "arr") {
-            if (borrowed?.(fn.name, index) !== true)
-                return null;
-            arrays.add(p.localId);
-        }
+    const references = new Set();
+    for (const [index, p] of fn.params.entries()) {
+        const kind = valueKind(p.type, refs);
+        if (kind !== "arr" && kind !== "ref")
+            continue;
+        if (borrowed?.(fn.name, index) !== true)
+            return null;
+        (kind === "arr" ? arrays : references).add(p.localId);
+    }
     const localKind = (id) => {
         const local = locals.get(id);
         if (!local || local.boxed || local.tdz)
@@ -60,13 +76,37 @@ function cloneShape(fn, borrowed) {
     };
     const arrayRef = (e) => e.kind === "varRef" && arrays.has(e.localId);
     for (const p of fn.params)
-        if (!arrays.has(p.localId) && (localKind(p.localId) === null || scalarKind(p.type) === null))
+        if (!arrays.has(p.localId) &&
+            !references.has(p.localId) &&
+            (localKind(p.localId) === null || scalarKind(p.type) === null))
             return null;
+    /** A field the clone reads in place, of a record a reference reads. */
+    const readsField = (e) => reference(e.obj) && refs?.field(e.shapeId, e.field) != null;
+    /** A reference parameter, a reference field of a record one reaches, or
+     * a narrow of a reference union to its record arm. */
+    const reference = (e) => {
+        if (refs?.reference(e.type) !== true)
+            return false;
+        switch (e.kind) {
+            case "varRef":
+                return references.has(e.localId);
+            case "recordGet":
+                return readsField(e);
+            case "unionNarrow":
+                return refs.arm(e.unionId, e.tag) === "record" && reference(e.value);
+            default:
+                return false;
+        }
+    };
     const callees = new Set();
     const expr = (e, want) => {
         if (scalarKind(e.type) !== want)
             return false;
         switch (e.kind) {
+            case "recordGet":
+                return readsField(e);
+            case "unionIsTag":
+                return want === "bool" && refs?.arm(e.unionId, e.tag) != null && reference(e.value);
             case "numLit":
                 return isInt32Literal(e.value);
             case "boolLit":
@@ -114,14 +154,20 @@ function cloneShape(fn, borrowed) {
             case "ternary":
                 return expr(e.cond, "bool") && expr(e.then, want) && expr(e.else_, want);
             case "arrIntrinsic":
+                if (e.method === "getNumber")
+                    return want === "num" && arrayRef(e.receiver) && expr(e.args[0], "num");
                 return (want === "num" && e.method === "length" && e.args.length === 0 && arrayRef(e.receiver));
             case "arrayGet":
                 return want === "num" && arrayRef(e.arr) && expr(e.index, "num");
             case "call":
                 callees.add(e.callee);
                 return e.args.every((a) => {
-                    const kind = valueKind(a.type);
-                    return kind === "arr" ? arrayRef(a) : kind !== null && expr(a, kind);
+                    const kind = valueKind(a.type, refs);
+                    if (kind === "arr")
+                        return arrayRef(a);
+                    if (kind === "ref")
+                        return reference(a);
+                    return kind !== null && expr(a, kind);
                 });
             default:
                 return false;
@@ -155,7 +201,7 @@ function cloneShape(fn, borrowed) {
     return stmts(fn.body) ? callees : null;
 }
 /** Every call in `fn`'s body has arguments of its callee's parameter kinds. */
-function callsMatch(fn, byName) {
+function callsMatch(fn, byName, refs) {
     let ok = true;
     const visit = (e) => {
         switch (e.kind) {
@@ -164,7 +210,7 @@ function callsMatch(fn, byName) {
                 if (!callee ||
                     callee.params.length !== e.args.length ||
                     scalarKind(callee.returnType) !== scalarKind(e.type) ||
-                    callee.params.some((p, i) => valueKind(p.type) !== valueKind(e.args[i].type)))
+                    callee.params.some((p, i) => valueKind(p.type, refs) !== valueKind(e.args[i].type, refs)))
                     ok = false;
                 e.args.forEach(visit);
                 return;
@@ -186,6 +232,10 @@ function callsMatch(fn, byName) {
             case "arrayGet":
                 visit(e.index);
                 return;
+            case "arrIntrinsic":
+                if (e.method === "getNumber")
+                    visit(e.args[0]);
+                return;
             default:
                 return;
         }
@@ -213,12 +263,12 @@ function callsMatch(fn, byName) {
     stmts(fn.body);
     return ok;
 }
-export function findIntegerClones(mod, borrowed) {
+export function findIntegerClones(mod, borrowed, refs) {
     const byName = new Map(mod.functions.map((fn) => [fn.name, fn]));
     const edges = new Map();
     for (const fn of mod.functions) {
-        const callees = cloneShape(fn, borrowed);
-        if (callees && callsMatch(fn, byName))
+        const callees = cloneShape(fn, borrowed, refs);
+        if (callees && callsMatch(fn, byName, refs))
             edges.set(fn.name, callees);
     }
     // Drop functions that call a non-member until none does.
@@ -351,6 +401,8 @@ function callsAnything(e) {
             return callsAnything(e.cond) || callsAnything(e.then) || callsAnything(e.else_);
         case "arrayGet":
             return callsAnything(e.index);
+        case "arrIntrinsic":
+            return e.method === "getNumber" && callsAnything(e.args[0]);
         default:
             return false;
     }
@@ -501,7 +553,9 @@ class CloneEmitter {
         };
         const params = new Set(fn.params.map((p) => p.localId));
         for (const local of fn.locals) {
-            const kind = params.has(local.id) ? valueKind(local.type) : scalarKind(local.type);
+            const kind = params.has(local.id)
+                ? valueKind(local.type, this.host.cloneReferences)
+                : scalarKind(local.type);
             if (kind === null || local.boxed)
                 continue;
             const slot = `%l${this.slotCount++}`;
@@ -515,6 +569,8 @@ class CloneEmitter {
         this.frame = frame;
         if (inline === null && this.stackCheck)
             this.emitStackCheck();
+        if (inline === null)
+            this.emitPoll();
         const tail = this.tailShape();
         if (!tail.loop) {
             this.assumeParamRanges();
@@ -526,11 +582,56 @@ class CloneEmitter {
             B.entryAllocas.push(`${frame.accSlot} = alloca i32 ; accumulator`);
             B.line(`store i32 0, ptr ${frame.accSlot}`);
         }
+        const budget = tail.plain && this.host.executableTls ? `%l${this.slotCount++}` : null;
+        if (budget !== null) {
+            B.entryAllocas.push(`${budget} = alloca i32 ; poll countdown`);
+            B.line(`store i32 0, ptr ${budget}`);
+        }
         frame.loopLabel = B.newLabel("tail");
         B.br(frame.loopLabel);
         B.startBlock(frame.loopLabel);
+        if (budget !== null)
+            this.emitLoopPoll(budget);
         this.assumeParamRanges();
     }
+    /** In an ELF worker executable, bails when the thread's pending word is
+     * set: an exception is pending, or the worker must stop. The dispatcher
+     * then forwards it. Ordinary bodies observe a stop through their pending
+     * tests after calls and in loops, which clones do not run, so a clone
+     * polls when it is called and every POLL_INTERVAL iterations of a loop
+     * over plain tail calls, which can run forever. An accumulating loop runs
+     * at most until its virtual stack pointer passes the guard. The load is
+     * ordered, so LLVM keeps it in loops it forms out of tail calls. */
+    emitPoll() {
+        if (!this.host.executableTls)
+            return;
+        const { B } = this;
+        this.host.declare(EXC_FLAG_DECL);
+        const word = B.tmp();
+        const clear = B.tmp();
+        B.line(`${word} = load atomic i32, ptr @scr_exc_flag monotonic, align 4`);
+        B.line(`${clear} = icmp eq i32 ${word}, 0`);
+        this.require(clear);
+    }
+    /** The poll of a loop iteration, every POLL_INTERVAL iterations. */
+    emitLoopPoll(budget) {
+        const { B } = this;
+        const left = B.tmp();
+        const rest = B.tmp();
+        const due = B.tmp();
+        B.line(`${left} = load i32, ptr ${budget}`);
+        B.line(`${rest} = sub i32 ${left}, 1`);
+        B.line(`store i32 ${rest}, ptr ${budget}`);
+        B.line(`${due} = icmp eq i32 ${left}, 0`);
+        const poll = B.newLabel("poll");
+        const next = B.newLabel("poll.next");
+        B.condBr(due, poll, next);
+        B.startBlock(poll);
+        B.line(`store i32 ${POLL_INTERVAL - 1}, ptr ${budget}`);
+        this.emitPoll();
+        B.br(next);
+        B.startBlock(next);
+    }
     /** Starts the virtual stack pointer of an activation with an accumulating
      * loop. */
     openVirtualStack(frame, parent) {
@@ -587,7 +688,7 @@ class CloneEmitter {
     }
     emit() {
         const { B, fn } = this;
-        const params = fn.params.map((p, i) => `${kindType(valueKind(p.type))} %a${i}`);
+        const params = fn.params.map((p, i) => `${kindType(valueKind(p.type, this.host.cloneReferences))} %a${i}`);
         this.openFrame(fn.params.map((_, i) => `%a${i}`), null);
         this.emitStmts(fn.body);
         // A number or boolean function never falls off its end; bail if it did.
@@ -630,9 +731,10 @@ class CloneEmitter {
     inlines(e) {
         return this.inlineSelf && this.frame.inline === null && this.isSelfCall(e);
     }
-    /** Whether the body has self tail calls, and whether one accumulates. */
+    /** Whether the body has self tail calls, whether one accumulates, and
+     * whether one is a plain tail call. */
     tailShape() {
-        let loop = false;
+        let plain = false;
         let accumulates = false;
         const ret = (e) => {
             if (e.kind === "ternary") {
@@ -640,9 +742,9 @@ class CloneEmitter {
                 ret(e.else_);
             }
             else if (this.isSelfCall(e))
-                loop = true;
+                plain = true;
             else if (this.accumulatingCall(e) !== null)
-                loop = accumulates = true;
+                accumulates = true;
         };
         const stmts = (list) => {
             for (const s of list) {
@@ -658,7 +760,7 @@ class CloneEmitter {
             }
         };
         stmts(this.fn.body);
-        return { loop, accumulates };
+        return { loop: plain || accumulates, accumulates, plain };
     }
     isSelfCall(e) {
         return e.kind === "call" && e.callee === this.fn.name;
@@ -692,7 +794,7 @@ class CloneEmitter {
         host.declare(`declare ptr @llvm.stacksave.p0()`);
         host.declare(`declare i1 @llvm.expect.i1(i1, i1)`);
         host.declare(`declare void @scr_stack_overflow()`);
-        host.declare(STACK_GUARD_DECL);
+        host.declare(stackGuardDecl(host.executableTls));
         const sp = B.tmp();
         const guard = B.tmp();
         const low = B.tmp();
@@ -720,7 +822,7 @@ class CloneEmitter {
      * continuing at the overflow block past the guard. */
     consumeFrame() {
         const { B } = this;
-        this.host.declare(STACK_GUARD_DECL);
+        this.host.declare(stackGuardDecl(this.host.executableTls));
         const depth = B.tmp();
         const next = B.tmp();
         const guardPtr = B.tmp();
@@ -899,7 +1001,7 @@ class CloneEmitter {
     }
     /** A call of `callee`'s clone with evaluated arguments. */
     emitCloneCall(callee, args, exprs, marker = "") {
-        const list = args.map((a, i) => `${kindType(valueKind(exprs[i].type))} ${a}`);
+        const list = args.map((a, i) => `${kindType(valueKind(exprs[i].type, this.host.cloneReferences))} ${a}`);
         const result = this.B.tmp();
         this.B.line(`${result} = ${marker}call ${CLONE_RESULT} @${mangleFunction(callee)}${INTEGER_CLONE_SUFFIX}(${list.join(", ")})`);
         return result;
@@ -912,6 +1014,41 @@ class CloneEmitter {
         this.B.line(`${value} = load ptr, ptr ${this.frame.slots.get(e.localId).slot}`);
         return value;
     }
+    /** The record pointer of a reference expression (cloneShape's
+     * `reference`): a parameter, a field read, or a narrow to the record arm,
+     * which bails on NULL so the ordinary body reports it. */
+    emitRef(e) {
+        const { B } = this;
+        switch (e.kind) {
+            case "varRef": {
+                const value = B.tmp();
+                B.line(`${value} = load ptr, ptr ${this.frame.slots.get(e.localId).slot}`);
+                return value;
+            }
+            case "recordGet": {
+                const value = B.tmp();
+                B.line(`${value} = load ptr, ptr ${this.emitFieldPtr(e)}`);
+                return value;
+            }
+            case "unionNarrow": {
+                const value = this.emitRef(e.value);
+                const present = B.tmp();
+                B.line(`${present} = icmp ne ptr ${value}, null`);
+                this.require(present);
+                return value;
+            }
+            default:
+                throw new Error(`integer clone: unexpected reference expression ${e.kind}`);
+        }
+    }
+    /** The address of the field a record read names. */
+    emitFieldPtr(e) {
+        const obj = this.emitRef(e.obj);
+        const { struct, index } = this.host.cloneReferences.field(e.shapeId, e.field);
+        const ptr = this.B.tmp();
+        this.B.line(`${ptr} = getelementptr inbounds %${struct}, ptr ${obj}, i64 0, i32 ${index}`);
+        return ptr;
+    }
     /** `a.length`, bailing beyond the int32 range. */
     emitLength(array) {
         const { B } = this;
@@ -930,10 +1067,10 @@ class CloneEmitter {
      * keeps every slot below `filled` a present value) whose element is
      * exactly an int32. Everything else (holes, indices outside the prefix,
      * other numbers) bails: the ordinary body reads it. */
-    emitElement(e) {
+    emitElement(arr, indexExpr) {
         const { B } = this;
-        const array = this.emitArray(e.arr);
-        const index = this.narrow(this.emitNum(e.index));
+        const array = this.emitArray(arr);
+        const index = this.narrow(this.emitNum(indexExpr));
         const filledField = B.tmp();
         const filled = B.tmp();
         const inside = B.tmp();
@@ -950,9 +1087,37 @@ class CloneEmitter {
         B.line(`${dataField} = getelementptr inbounds %ScrArr, ptr ${array}, i32 0, i32 ${SCR_ARR_FIELD.data}`);
         B.line(`${data} = load ptr, ptr ${dataField}`);
         B.line(`${offset} = zext i32 ${index} to i64`);
+        // A number array keeps 8-byte doubles, or 4-byte int32 slots when it is
+        // proven to hold only int32s (SCR_ELEM_I32); any other storage bails.
+        const kindField = B.tmp();
+        const kind = B.tmp();
+        const isInt32 = B.tmp();
+        B.line(`${kindField} = getelementptr inbounds %ScrArr, ptr ${array}, i32 0, i32 ${SCR_ARR_FIELD.elem}`);
+        B.line(`${kind} = load i8, ptr ${kindField}`);
+        B.line(`${isInt32} = icmp eq i8 ${kind}, ${SCR_ELEM_I32}`);
+        const int32Slots = B.newLabel("elem.i32");
+        const doubleSlots = B.newLabel("elem.f64");
+        const join = B.newLabel("elem.join");
+        B.condBr(isInt32, int32Slots, doubleSlots);
+        B.startBlock(int32Slots);
+        const int32Slot = B.tmp();
+        const int32 = B.tmp();
+        B.line(`${int32Slot} = getelementptr inbounds i32, ptr ${data}, i64 ${offset}`);
+        B.line(`${int32} = load i32, ptr ${int32Slot}`);
+        B.br(join);
+        B.startBlock(doubleSlots);
+        const isDouble = B.tmp();
+        B.line(`${isDouble} = icmp eq i8 ${kind}, ${SCR_ELEM_F64}`);
+        this.require(isDouble);
         B.line(`${slot} = getelementptr inbounds double, ptr ${data}, i64 ${offset}`);
         B.line(`${number} = load double, ptr ${slot}`);
-        return this.exactInt32(number);
+        const exact = this.exactInt32(number);
+        const doubleEnd = B.currentLabel();
+        B.br(join);
+        B.startBlock(join);
+        const value = B.tmp();
+        B.line(`${value} = phi i32 [ ${int32}, %${int32Slots} ], [ ${exact}, %${doubleEnd} ]`);
+        return value;
     }
     /** The int32 the double `number` equals bit for bit (never -0, NaN, a
      * fraction or out of range), bailing otherwise. */
@@ -974,12 +1139,14 @@ class CloneEmitter {
     }
     emitArgs(call) {
         return call.args.map((a) => {
-            const kind = valueKind(a.type);
+            const kind = valueKind(a.type, this.host.cloneReferences);
             return kind === "arr"
                 ? this.emitArray(a)
-                : kind === "num"
-                    ? this.narrow(this.emitNum(a))
-                    : this.emitBool(a);
+                : kind === "ref"
+                    ? this.emitRef(a)
+                    : kind === "num"
+                        ? this.narrow(this.emitNum(a))
+                        : this.emitBool(a);
         });
     }
     emitCall(call) {
@@ -1055,9 +1222,19 @@ class CloneEmitter {
                     return { name: this.emitInlineCall(e), wide: false };
                 return { name: this.emitCall(e), wide: true };
             case "arrIntrinsic":
+                // `getNumber` reads an element like `arrayGet`: a missing one bails,
+                // and the ordinary body reads it as NaN.
+                if (e.method === "getNumber")
+                    return { name: this.emitElement(e.receiver, e.args[0]), wide: false };
                 return { name: this.emitLength(this.emitArray(e.receiver)), wide: false };
             case "arrayGet":
-                return { name: this.emitElement(e), wide: false };
+                return { name: this.emitElement(e.arr, e.index), wide: false };
+            case "recordGet": {
+                // A number field that is exactly an int32; any other number bails.
+                const number = B.tmp();
+                B.line(`${number} = load double, ptr ${this.emitFieldPtr(e)}`);
+                return { name: this.exactInt32(number), wide: false };
+            }
             case "unary": {
                 const operand = this.narrow(this.emitNum(e.operand));
                 const value = B.tmp();
@@ -1201,6 +1378,21 @@ class CloneEmitter {
                 B.line(`${value} = xor i1 ${operand}, true`);
                 return value;
             }
+            case "recordGet": {
+                const byte = B.tmp();
+                const value = B.tmp();
+                B.line(`${byte} = load i8, ptr ${this.emitFieldPtr(e)}`);
+                B.line(`${value} = trunc i8 ${byte} to i1`);
+                return value;
+            }
+            case "unionIsTag": {
+                // NULL is the `null` arm; any other pointer the record arm.
+                const pointer = this.emitRef(e.value);
+                const isNull = this.host.cloneReferences.arm(e.unionId, e.tag) === "null";
+                const value = B.tmp();
+                B.line(`${value} = icmp ${isNull !== e.negated ? "eq" : "ne"} ptr ${pointer}, null`);
+                return value;
+            }
             case "toBool": {
                 if (scalarKind(e.operand.type) === "bool")
                     return this.emitBool(e.operand);
@@ -1285,9 +1477,10 @@ export function emitIntegerDispatcher(fn, host, entry, int32Params, int32Result)
     const cloneArgs = [];
     const exact = [];
     fn.params.forEach((p, i) => {
-        const kind = valueKind(p.type);
-        if (kind === "bool" || kind === "arr") {
-            // Borrowed arrays pass through to the clone and the ordinary entry.
+        const kind = valueKind(p.type, host.cloneReferences);
+        if (kind === "bool" || kind === "arr" || kind === "ref") {
+            // Borrowed arrays and records pass through to the clone and the
+            // ordinary entry.
             const type = kindType(kind);
             params.push(`${type} %p${i}`);
             entryArgs.push(`${type} %p${i}`);