knitting 0.1.60 → 0.1.62

package/src/runtime/dispatcher.js

@@ -1,5 +1,34 @@
 import { createRuntimeMessageChannel, } from "../common/worker-runtime.js";
-export const hostDispatcherLoop = ({ signalBox: { opView, txStatus, rxStatus, }, queue: { completeFrame, hasPendingFrames, flushToWorker, txIdle, }, channelHandler, dispatcherOptions, notifySignal, }) => {
+import { RUNTIME, SET_IMMEDIATE } from "../common/runtime.js";
+/**
+ * Macrotask primitive for the host pump, picked per runtime: a round trip costs
+ * 757ns on Bun via MessageChannel but 4662ns on Deno, where `setImmediate` is
+ * 1110ns. Bun keeps the channel; browser and Andromeda have no choice.
+ *
+ * `serial-channel` opts back into the channel explicitly (see `src/api.ts`): one
+ * hop drives every lane's check in turn and relies on the channel's delivery to
+ * do it, so a merely cheaper pump starves it -- it cost 25% throughput there.
+ */
+const IMMEDIATE_PUMP = RUNTIME === "deno" || RUNTIME === "node"
+    ? SET_IMMEDIATE
+    : undefined;
+/**
+ * Free drain hops before `scheduleNotify` stops re-arming the pump for free.
+ *
+ * The window has to match what the dispatcher escalates *to*, so these are
+ * chosen together and `canUseDoorbell` picks between them. Polling escalates to
+ * the `setTimeout` ladder, whose finest rung is ~1.1ms on every runtime, so it
+ * wants a wide window; that sleep is also load-bearing, since it batches
+ * completions. A doorbell escalates to `Atomics.waitAsync` at about the price of
+ * one hop, so it wants a narrow one.
+ *
+ * Mixing them is the trap: a doorbell behind the wide window keeps every poll
+ * hop *and* adds the arm, and loses to plain polling at every thread count.
+ * Measurements behind both values: `docs/host-doorbell-proposal.md`.
+ */
+const POLL_STALL_FREE_LOOPS = 128;
+const DOORBELL_STALL_FREE_LOOPS = 1;
+export const hostDispatcherLoop = ({ signalBox: { opView, txStatus, rxStatus, }, queue: { completeFrame, hasPendingFrames, flushToWorker, txIdle, waitForCompletion, setCompletionWaiterArmed, }, channelHandler, dispatcherOptions, notifySignal, crossProcess, }) => {
     const a_load = Atomics.load;
     const a_store = Atomics.store;
     const a_notify = Atomics.notify;
@@ -10,13 +39,81 @@ export const hostDispatcherLoop = ({ signalBox: { opView, txStatus, rxStatus, },
                 a_notify(opView, 0, 1);
         });
     const notify = () => channelHandler.notify();
+    // `crossProcess` is a capability, not a preference, so it overrides an
+    // explicit `doorbell: true`: V8's Atomics waiter list is per isolate, so a
+    // worker in another process can never ring the host's waiter, and an armed
+    // doorbell would just sleep to the watchdog.
+    const canUseDoorbell = crossProcess !== true &&
+        (dispatcherOptions?.doorbell ?? true) &&
+        (RUNTIME === "bun" || RUNTIME === "node") &&
+        typeof Atomics.waitAsync === "function";
+    let doorbellEnabled = canUseDoorbell;
+    let doorbellArmed = false;
+    let doorbellEpoch = 0 | 0;
+    const DOORBELL_WATCHDOG_MS = 1000;
     let stallCount = 0 | 0;
-    const STALL_FREE_LOOPS = Math.max(0, (dispatcherOptions?.stallFreeLoops ?? 128) | 0);
+    const requestedStallFreeLoops = dispatcherOptions?.stallFreeLoops;
+    let stallFreeLoops = requestedStallFreeLoops !== undefined
+        ? Math.max(0, requestedStallFreeLoops | 0)
+        : canUseDoorbell
+            ? DOORBELL_STALL_FREE_LOOPS
+            : POLL_STALL_FREE_LOOPS;
     const MAX_BACKOFF_MS = Math.max(0, (dispatcherOptions?.maxBackoffMs ?? 10) | 0);
     let backoffTimer;
     // inFlight prevents re-entrancy when pool.ts fires check() concurrently
     // from both send() and the channel callback. Cheaper than try/finally.
     let inFlight = false;
+    const cancelDoorbell = () => {
+        if (!doorbellArmed)
+            return;
+        doorbellEpoch = (doorbellEpoch + 1) | 0;
+        doorbellArmed = false;
+        setCompletionWaiterArmed(false);
+    };
+    const armDoorbell = () => {
+        if (!doorbellEnabled || doorbellArmed === true) {
+            if (!doorbellEnabled)
+                notify();
+            return;
+        }
+        const token = (doorbellEpoch + 1) | 0;
+        doorbellEpoch = token;
+        doorbellArmed = true;
+        let woke = false;
+        const wake = () => {
+            if (!doorbellArmed || doorbellEpoch !== token || woke)
+                return;
+            woke = true;
+            doorbellArmed = false;
+            doorbellEpoch = (doorbellEpoch + 1) | 0;
+            setCompletionWaiterArmed(false);
+            // Keep the existing macrotask boundary. Calling check directly from a
+            // resolved waitAsync promise can chain microtasks under a hot workload
+            // and starve host I/O.
+            notify();
+        };
+        let supported = false;
+        try {
+            supported = waitForCompletion(wake, DOORBELL_WATCHDOG_MS);
+        }
+        catch {
+            supported = false;
+        }
+        // Some runtimes reject waitAsync on their SharedArrayBuffer variants; fall
+        // back to polling rather than hanging the pool.
+        if (!supported) {
+            doorbellEnabled = false;
+            // The window was narrowed for a doorbell that no longer exists; widen it
+            // back or the ladder fires after a single fruitless drain.
+            if (requestedStallFreeLoops === undefined) {
+                stallFreeLoops = POLL_STALL_FREE_LOOPS;
+            }
+            doorbellArmed = false;
+            doorbellEpoch = (doorbellEpoch + 1) | 0;
+            setCompletionWaiterArmed(false);
+            notify();
+        }
+    };
     const check = () => {
         if (inFlight) {
             // Another check() is already mid-drain; mark that a re-run is needed
@@ -24,6 +121,16 @@ export const hostDispatcherLoop = ({ signalBox: { opView, txStatus, rxStatus, },
             check.rerun = true;
             return;
         }
+        // A waitAsync cannot be cancelled, so bumping its epoch makes the eventual
+        // callback inert instead.
+        cancelDoorbell();
+        // Idle lane: skip the drain. Otherwise it pays a txStatus write, an rxStatus
+        // load, and an Atomics.notify that wakes a parked worker for nothing — the
+        // waste that makes serial-channel latency grow with thread count.
+        if (txIdle()) {
+            check.isRunning = false;
+            return;
+        }
         inFlight = true;
         if (backoffTimer !== undefined) {
             clearTimeout(backoffTimer);
@@ -37,25 +144,27 @@ export const hostDispatcherLoop = ({ signalBox: { opView, txStatus, rxStatus, },
                 a_store(opView, 0, 1);
                 wakeSignal();
             }
-            // Drain loop: local vars so V8 keeps them as unboxed int32.
-            let anyProgressed = false;
+            // Local vars so V8 keeps them as unboxed int32. Only a reaped completion
+            // counts as progress for `stallCount`: the pump exists to notice work a
+            // worker cannot announce, and letting a flush reset the counter would
+            // leave the escalation unreachable for a pool that never runs dry.
+            let completed = false;
             let progressed = true;
             while (progressed) {
                 progressed = false;
                 if (completeFrame() > 0) {
                     progressed = true;
-                    anyProgressed = true;
+                    completed = true;
                 }
                 while (hasPendingFrames()) {
                     if (!flushToWorker())
                         break;
                     progressed = true;
-                    anyProgressed = true;
                 }
             }
             txStatus[0] = 0;
             if (!txIdle()) {
-                if (anyProgressed || hasPendingFrames()) {
+                if (completed || hasPendingFrames()) {
                     stallCount = 0 | 0;
                 }
                 else {
@@ -74,14 +183,22 @@ export const hostDispatcherLoop = ({ signalBox: { opView, txStatus, rxStatus, },
     check.isRunning = false;
     check.rerun = false;
     const scheduleNotify = () => {
-        if (stallCount <= STALL_FREE_LOOPS) {
+        if (stallCount <= stallFreeLoops) {
             notify();
             return;
         }
+        if (doorbellEnabled) {
+            // Release isRunning during the wait so a fresh send() can restart the
+            // dispatcher immediately. The epoch in cancelDoorbell() invalidates the
+            // old waiter in that case.
+            check.isRunning = false;
+            armDoorbell();
+            return;
+        }
         // One delayed wakeup at a time; fresh send() calls preempt via check directly.
         if (backoffTimer !== undefined)
             return;
-        let delay = (stallCount - STALL_FREE_LOOPS - 1) | 0;
+        let delay = (stallCount - stallFreeLoops - 1) | 0;
         if (delay < 0)
             delay = 0;
         else if (delay > MAX_BACKOFF_MS)
@@ -99,25 +216,47 @@ export const hostDispatcherLoop = ({ signalBox: { opView, txStatus, rxStatus, },
     return { check };
 };
 export class ChannelHandler {
+    // Set only when the pump is a MessageChannel. The `setImmediate` pump has no
+    // ports, and nothing outside this class reads them.
     channel;
     port1;
     port2;
-    #post2;
-    constructor() {
-        this.channel = createRuntimeMessageChannel();
-        this.port1 = this.channel.port1;
-        this.port2 = this.channel.port2;
-        this.#post2 = (message) => this.port2.postMessage(message);
+    #handler;
+    #notify;
+    constructor(pump = "auto") {
+        if (pump === "auto" && IMMEDIATE_PUMP !== undefined) {
+            const immediate = IMMEDIATE_PUMP;
+            // Allocated once; the pump fires hundreds of thousands of times a second.
+            // Routing via `#handler` also makes a callback outliving `close()` inert.
+            const run = () => {
+                this.#handler?.();
+            };
+            this.#notify = () => {
+                immediate(run);
+            };
+            return;
+        }
+        const channel = createRuntimeMessageChannel();
+        const port2 = channel.port2;
+        this.channel = channel;
+        this.port1 = channel.port1;
+        this.port2 = port2;
+        this.#notify = () => {
+            port2.postMessage(null);
+        };
     }
     notify() {
-        this.#post2(null);
+        this.#notify();
     }
     /**
-     * Opens the channel (if not already open) and sets the onmessage handler.
-     * This is the setup so `notify` can send a message to the port 1.
+     * Registers the handler the pump calls back into. On the channel pump this is
+     * also where the ports are opened, so `notify` can reach port 1.
      */
     open(f) {
+        this.#handler = f;
         const port1 = this.port1;
+        if (port1 === undefined)
+            return;
         if (typeof port1.on === "function") {
             port1.on("message", f);
         }
@@ -125,13 +264,16 @@ export class ChannelHandler {
             // @ts-ignore
             port1.onmessage = f;
         }
-        this.port1.start?.();
-        this.port2.start?.();
+        this.port1?.start?.();
+        this.port2?.start?.();
     }
     /**
-     * Closes the channel if it is open.
+     * Detaches the handler, and closes the channel if this pump has one.
      */
     close() {
+        this.#handler = undefined;
+        if (this.port1 === undefined || this.port2 === undefined)
+            return;
         //@ts-ignore
         this.port1.onmessage = null;
         //@ts-ignore

package/src/runtime/inline-executor.js

@@ -81,9 +81,14 @@ const composeInlineCallable = (fn, timeout, useAbortToolkit = false) => {
     };
 };
 export const createInlineExecutor = ({ tasks, genTaskID, batchSize, }) => {
+    // `createPool` passes an already name-sorted array, which is the order
+    // `fnNumber` indexes into. The object form has to reach the same order on its
+    // own: task ids identify a task, they do not rank it, so sorting by id would
+    // hand callers a different lane than the one they addressed.
     const entries = Array.isArray(tasks)
         ? tasks
-        : Object.values(tasks).sort((a, b) => a.id - b.id);
+        : Object.values(tasks)
+            .sort((a, b) => a.name.localeCompare(b.name));
     const runners = entries.map((entry) => {
         // Imported tasks must never execute on the host inline lane: their module
         // import is meant to stay inside the worker so worker permission policies

package/src/runtime/pool.d.ts

@@ -1,12 +1,115 @@
-import { type ProcessSharedMemoryBacking } from "./process-worker.js";
+import { createHostTxQueue } from "./tx-queue.js";
+import { type ProcessSharedMemoryBacking, type ProcessStealMemoryLayout, type ResolvedProcessSharedMemorySettings } from "./process-worker.js";
 import { type Sab } from "../ipc/transport/shared-memory.js";
 import { ChannelHandler, type DispatcherCheck } from "./dispatcher.js";
-import { lock2 } from "../memory/lock.js";
+import { lock2, type Task } from "../memory/lock.js";
 import "../memory/payloadCodec.js";
-import type { DebugOptions, DispatcherSettings, WorkerContext, WorkerData, WorkerSettings } from "../types.js";
+import type { DebugOptions, DispatcherSettings, LockBuffers, WorkerContext, WorkerData, WorkerSettings } from "../types.js";
 import "../worker/loop.js";
 import { type PayloadBufferOptions } from "../memory/payload-config.js";
-export declare const spawnWorkerContext: ({ list, ids, names, sab, thread, debug, hostDebug, totalNumberOfThread, source, at, workerOptions, workerExecArgv, permission, host, payload, bufferReferenceReturn, abortSignalCapacity, usesAbortSignal, sharedChannelHandler, }: {
+/**
+ * Build the buffers, host-side locks and pending registry that a stealing pool
+ * shares. One submit region for every worker to claim from, one private return
+ * region per worker, and a single pool-global queue so a response arriving on
+ * any lane settles the right promise.
+ *
+ * Region width follows paper §6.1: the largest power-of-two `g` leaving a spare
+ * region for a delayed claimant (`slots / g >= consumers + 1`). At 32 slots that
+ * caps how wide a region can be well before it caps the worker count.
+ */
+export declare const resolveStealRegionLanes: (consumers: number) => number;
+/** Maximum claimants that leave the protocol's required spare region. */
+export declare const MAX_STEAL_CONSUMERS: number;
+export declare const createStealPoolBuffers: ({ threads, payload, regionLanes, abortSignalCapacity, usesAbortSignal, processWorker, }: {
+    threads: number;
+    payload?: PayloadBufferOptions;
+    regionLanes?: number;
+    abortSignalCapacity?: number;
+    usesAbortSignal?: boolean;
+    processWorker?: {
+        signalBytes: number;
+        sharedMemory: ResolvedProcessSharedMemorySettings;
+    };
+}) => {
+    submitBuffers: LockBuffers;
+    returnBuffers: LockBuffers[];
+    hostSubmitLock: {
+        enlist: (task: Task) => true;
+        encode: (task: Task, state?: number) => boolean;
+        encodeManyFrom: (list: import("../ipc/tools/ring-queue.js").default<Task>) => number;
+        encodeAll: () => boolean;
+        publish: (task: Task) => boolean;
+        flushPending: () => boolean;
+        decode: () => boolean;
+        hasSpace: () => boolean;
+        resolved: import("../ipc/tools/ring-queue.js").default<Task>;
+        hostBits: Int32Array<import("../common/shared-buffer-region.js").SharedBuffer>;
+        workerBits: Int32Array<import("../common/shared-buffer-region.js").SharedBuffer>;
+        recyclecList: import("../ipc/tools/ring-queue.js").default<Task>;
+        resolveHost: ({ queue, onResolved, shouldSettle, activeRejectPlaceholder, }: {
+            queue: Task[];
+            onResolved?: (task: Task) => void;
+            shouldSettle?: (task: Task) => boolean;
+            activeRejectPlaceholder?: Task["reject"];
+        }) => () => number;
+        waitForHostChange: (timeoutMs?: number) => import("../memory/lock.js").WaitAsyncResult | undefined;
+        setHostWaiterArmed: (armed: boolean) => void;
+        hasPendingFrames: () => boolean;
+        getPendingFrameCount: () => number;
+        getPendingPromiseCount: () => number;
+        resetPendingState: () => void;
+        deactivateStealConsumer: (id: number) => boolean;
+        takeDeferredCount: () => number;
+        setPromiseHandler: (handler?: import("../memory/lock.js").PromisePayloadHandler) => void;
+    };
+    hostReturnLocks: {
+        enlist: (task: Task) => true;
+        encode: (task: Task, state?: number) => boolean;
+        encodeManyFrom: (list: import("../ipc/tools/ring-queue.js").default<Task>) => number;
+        encodeAll: () => boolean;
+        publish: (task: Task) => boolean;
+        flushPending: () => boolean;
+        decode: () => boolean;
+        hasSpace: () => boolean;
+        resolved: import("../ipc/tools/ring-queue.js").default<Task>;
+        hostBits: Int32Array<import("../common/shared-buffer-region.js").SharedBuffer>;
+        workerBits: Int32Array<import("../common/shared-buffer-region.js").SharedBuffer>;
+        recyclecList: import("../ipc/tools/ring-queue.js").default<Task>;
+        resolveHost: ({ queue, onResolved, shouldSettle, activeRejectPlaceholder, }: {
+            queue: Task[];
+            onResolved?: (task: Task) => void;
+            shouldSettle?: (task: Task) => boolean;
+            activeRejectPlaceholder?: Task["reject"];
+        }) => () => number;
+        waitForHostChange: (timeoutMs?: number) => import("../memory/lock.js").WaitAsyncResult | undefined;
+        setHostWaiterArmed: (armed: boolean) => void;
+        hasPendingFrames: () => boolean;
+        getPendingFrameCount: () => number;
+        getPendingPromiseCount: () => number;
+        resetPendingState: () => void;
+        deactivateStealConsumer: (id: number) => boolean;
+        takeDeferredCount: () => number;
+        setPromiseHandler: (handler?: import("../memory/lock.js").PromisePayloadHandler) => void;
+    }[];
+    sharedQueue: {
+        rejectAll: (reason: string) => void;
+        hasPendingFrames: () => boolean;
+        txIdle: () => boolean;
+        completeFrame: () => number;
+        waitForCompletion: (onWake: () => void, timeoutMs?: number) => boolean;
+        setCompletionWaiterArmed: (armed: boolean) => void;
+        setReturnHooks: (lane: number, hooks: (import("../connections/buffer-reference.js").BufferReferenceReturnHooks | ((token: bigint) => void)) | undefined) => void;
+        enqueue: (functionID: number, timeout?: import("../types.js").TaskTimeout, abortSignal?: import("../types.js").AbortSignalOption) => (rawArgs: unknown) => Promise<never> | import("../common/with-resolvers.js").PromiseWithMaybeReject<unknown>;
+        flushToWorker: () => boolean;
+        enqueueKnown: (task: Task) => boolean;
+        settlePromisePayload: (task: Task, isRejected: boolean, value: unknown) => boolean;
+    };
+    regionLanes: number;
+    processMemory: ProcessStealMemoryLayout;
+    abortSignalSAB: SharedArrayBuffer | import("../types.js").SharedBufferRegion;
+    abortSignalMax: number;
+};
+export declare const spawnWorkerContext: ({ list, ids, names, sab, thread, debug, hostDebug, totalNumberOfThread, source, at, workerOptions, workerExecArgv, permission, host, payload, bufferReferenceReturn, abortSignalCapacity, usesAbortSignal, sharedChannelHandler, stealPool, }: {
     list: string[];
     ids: number[];
     names: string[];
@@ -30,6 +133,26 @@ export declare const spawnWorkerContext: ({ list, ids, names, sab, thread, debug
      * macro channel that runs all lane checks.
      */
     sharedChannelHandler?: ChannelHandler;
+    /**
+     * Work-stealing wiring. When present the submit region, both host-side locks
+     * and the pending registry are owned by `createPool` and shared: every worker
+     * claims from one request region, and the host demultiplexes responses by
+     * task id. This lane only contributes its private return region and its
+     * worker. The dispatcher is owned by the pool too, so none is built here.
+     */
+    stealPool?: {
+        submitBuffers: LockBuffers;
+        returnBuffers: LockBuffers;
+        hostSubmitLock: ReturnType<typeof lock2>;
+        hostReturnLock: ReturnType<typeof lock2>;
+        sharedQueue: ReturnType<typeof createHostTxQueue>;
+        consumers: number;
+        consumerId: number;
+        regionLanes: number;
+        abortSignalSAB?: LockBuffers["headers"];
+        abortSignalMax?: number;
+        processMemory?: ProcessStealMemoryLayout;
+    };
 }) => (WorkerContext & {
     dispatcherCheck?: never;
     laneWake?: never;