knitting 0.1.61 → 0.1.63

package/src/runtime/process-worker.js

@@ -191,6 +191,106 @@ export const createProcessWorkerMemoryLayout = ({ signalBytes, abortBytes, paylo
         },
     };
 };
+export const createProcessStealMemoryLayout = ({ threads, signalBytes, abortBytes, abortSignalMax, payloadBytes, sharedMemory, }) => {
+    if (!Number.isInteger(threads) || threads < 2) {
+        throw new RangeError("process stealing needs at least two workers");
+    }
+    const carpet = createByteCarpet();
+    const requestLockSlice = carpet.take("requestLockSector", LOCK_SECTOR_BYTE_LENGTH);
+    const requestHeadersSlice = carpet.take("requestHeaders", getHeaderBlockByteLength({
+        slotCount: LockBound.slots,
+        slotStrideU32: HEADER_SLOT_STRIDE_U32,
+        alignTo: 64,
+    }));
+    const abortSignalsSlice = carpet.take("abortSignals", abortBytes);
+    const requestPayloadSlice = carpet.take("requestPayload", payloadBytes);
+    const workerSlices = Array.from({ length: threads }, (_, thread) => ({
+        signal: carpet.take(`signals-${thread}`, signalBytes),
+        returnLock: carpet.take(`returnLockSector-${thread}`, LOCK_SECTOR_BYTE_LENGTH),
+        returnHeaders: carpet.take(`returnHeaders-${thread}`, getHeaderBlockByteLength({
+            slotCount: LockBound.slots,
+            slotStrideU32: HEADER_SLOT_STRIDE_U32,
+            alignTo: 64,
+        })),
+        returnPayload: carpet.take(`returnPayload-${thread}`, payloadBytes),
+    }));
+    const primitives = getProcessWorkerSharedMemoryPrimitives();
+    const forceNamed = sharedMemory.mode === "named" || isWindowsRuntimeHost();
+    const mapping = primitives.createSharedMemory(forceNamed
+        ? {
+            size: carpet.byteLength(),
+            mode: "create",
+            name: makeProcessWorkerMemoryName(0, sharedMemory.namePrefix),
+        }
+        : { size: carpet.byteLength() });
+    const descriptor = FileDescriptor.fromMapping(mapping);
+    if (isWindowsRuntimeHost() && descriptor.name === undefined) {
+        throw new Error("Windows process worker shared memory must use a named mapping");
+    }
+    if (sharedMemory.mode === "named" && descriptor.name === undefined) {
+        throw new Error("processSharedMemory mode named needs a named shared-memory backend");
+    }
+    const buffer = mapping.buffer;
+    const bind = (slice) => makeSharedBufferRegion(buffer, slice.byteOffset, slice.byteLength);
+    const abortSignals = bind(abortSignalsSlice);
+    const requestLock = {
+        headers: bind(requestHeadersSlice),
+        headerSlotStrideU32: HEADER_SLOT_STRIDE_U32,
+        lockSector: bind(requestLockSlice),
+        payloadSector: bind(requestLockSlice),
+    };
+    // Each worker calls cleanup, so named-memory unlink happens only after every
+    // endpoint has released its view. The per-worker mapping wrappers below do
+    // not expose close(); the host views remain attached until host exit because
+    // Bun's external ArrayBuffer finalizer can race an explicit munmap.
+    let remaining = threads;
+    let cleaned = false;
+    const cleanupSharedMapping = () => {
+        if (cleaned)
+            return;
+        remaining--;
+        if (remaining > 0)
+            return;
+        cleaned = true;
+        const name = descriptor.name;
+        if (name !== undefined && sharedMemory.unlinkOnShutdown) {
+            primitives.unlinkSharedMemory?.(name);
+        }
+    };
+    const workerMapping = {
+        ...mapping,
+        close: undefined,
+    };
+    const workers = workerSlices.map((slices) => {
+        const returnLock = {
+            headers: bind(slices.returnHeaders),
+            headerSlotStrideU32: HEADER_SLOT_STRIDE_U32,
+            lockSector: bind(slices.returnLock),
+            payloadSector: bind(slices.returnLock),
+        };
+        return {
+            mapping: workerMapping,
+            descriptor,
+            controlLayout: {
+                controlSAB: buffer,
+                signals: bind(slices.signal),
+                abortSignals,
+                lock: requestLock,
+                returnLock,
+                slices: carpet.slices,
+            },
+            lockPayload: bind(requestPayloadSlice),
+            returnPayload: bind(slices.returnPayload),
+            cleanup: cleanupSharedMapping,
+        };
+    });
+    return {
+        mapping,
+        descriptor,
+        workers,
+        abortSignalMax,
+    };
+};
 const toChildProcessSharedBufferMetadata = (source, descriptor) => {
     const region = toSharedBufferRegion(source);
     const metadata = descriptor.toMetadata();
@@ -562,9 +662,23 @@ const spawnNodeHostedProcessWorker = ({ workerUrl, bootPayload, memory, processR
         queueMicrotask(() => child.send?.(bootPayload));
     }
     child.on("error", events.emitError);
-    child.on("exit", (code) => events.emitExit(code ?? -1));
+    let resolveExit = () => { };
+    const exited = new Promise((resolve) => {
+        resolveExit = resolve;
+    });
+    child.on("exit", (code) => {
+        resolveExit();
+        events.emitExit(code ?? -1);
+    });
     return {
-        terminate: () => child.kill(),
+        terminate: () => {
+            try {
+                child.kill();
+            }
+            catch {
+            }
+            return exited;
+        },
         unref: () => child.unref?.(),
         on: events.on,
     };

package/src/runtime/tx-queue.d.ts

@@ -33,6 +33,8 @@ export declare function createHostTxQueue({ max, lock, returnLock, extraReturnLo
     hasPendingFrames: () => boolean;
     txIdle: () => boolean;
     completeFrame: () => number;
+    waitForCompletion: (onWake: () => void, timeoutMs?: number) => boolean;
+    setCompletionWaiterArmed: (armed: boolean) => void;
     setReturnHooks: (lane: number, hooks: ReturnHooks | undefined) => void;
     enqueue: (functionID: FunctionID, timeout?: TaskTimeout, abortSignal?: AbortSignalOption) => (rawArgs: RawArguments) => Promise<never> | import("../common/with-resolvers.js").PromiseWithMaybeReject<unknown>;
     flushToWorker: () => boolean;

package/src/runtime/tx-queue.js

@@ -54,6 +54,18 @@ export function createHostTxQueue({ max, lock, returnLock, extraReturnLocks, rel
         activeRejectPlaceholder: PLACE_HOLDER,
         onResolved: onReturnResolved,
     }));
+    const returnWaiters = [
+        returnLock,
+        ...(extraReturnLocks ?? []),
+    ].map((each) => typeof each.waitForHostChange === "function"
+        ? each.waitForHostChange
+        : () => undefined);
+    const returnArmers = [
+        returnLock,
+        ...(extraReturnLocks ?? []),
+    ].map((each) => typeof each.setHostWaiterArmed === "function"
+        ? each.setHostWaiterArmed
+        : (_armed) => { });
     // A stealing queue drains one private return lock per worker. Borrowed
     // BufferReferences must therefore be claimed with the hooks belonging to the
     // worker that produced that particular return. The hooks are bound after the
@@ -84,6 +96,60 @@ export function createHostTxQueue({ max, lock, returnLock, extraReturnLocks, rel
             }
             return resolved;
         };
+    // A stealing queue has one private return lock per worker. Keep at most one
+    // waitAsync waiter per lane: after lane A rings, lane B's unresolved waiter
+    // must survive the next arm or Atomics.notify(..., 1) could wake an obsolete
+    // waiter and starve the live one until the watchdog timeout.
+    const completionArmed = new Uint8Array(returnWaiters.length);
+    let completionWake;
+    let completionGeneration = 0 | 0;
+    const setCompletionWaiterArmed = (armed) => {
+        for (const setArmed of returnArmers)
+            setArmed(armed);
+    };
+    const waitForCompletion = (onWake, timeoutMs) => {
+        completionWake = onWake;
+        // A persistent waiter may still be pending after a send preempted the
+        // dispatcher. Re-arm its shared gate before relying on that waiter again.
+        setCompletionWaiterArmed(true);
+        let supported = true;
+        for (let index = 0; index < returnWaiters.length; index++) {
+            if (completionArmed[index] !== 0)
+                continue;
+            let wait;
+            try {
+                wait = returnWaiters[index](timeoutMs);
+            }
+            catch {
+                wait = undefined;
+            }
+            if (wait === undefined) {
+                supported = false;
+                break;
+            }
+            completionArmed[index] = 1;
+            const generation = completionGeneration;
+            const wakeLane = () => {
+                if (completionArmed[index] === 0)
+                    return;
+                completionArmed[index] = 0;
+                returnArmers[index](false);
+                if (generation !== completionGeneration)
+                    return;
+                completionWake?.();
+            };
+            if (!wait.async)
+                wakeLane();
+            else
+                Promise.resolve(wait.value).then(wakeLane, wakeLane);
+        }
+        if (!supported) {
+            completionWake = undefined;
+            setCompletionWaiterArmed(false);
+            completionGeneration = (completionGeneration + 1) | 0;
+        }
+        return supported;
+    };
     const hasActiveTasks = () => {
         const count = (inUsed - getPendingPromiseCount()) | 0;
         return count > 0;
@@ -116,6 +182,8 @@ export function createHostTxQueue({ max, lock, returnLock, extraReturnLocks, rel
         hasPendingFrames,
         txIdle,
         completeFrame,
+        waitForCompletion,
+        setCompletionWaiterArmed,
         setReturnHooks: (lane, hooks) => {
             if (!Number.isInteger(lane) || lane < 0 || lane >= returnHooks.length) {
                 throw new RangeError(`return lane ${lane} out of range`);

package/src/runtime/worker-common.d.ts

@@ -10,4 +10,5 @@ export type NodeWorkerLike = {
 export declare const toWorkerSafeExecArgv: (flags: string[] | undefined) => string[] | undefined;
 export declare const toWorkerCompatExecArgv: (flags: string[] | undefined) => string[] | undefined;
 export declare const serializeWorkerBootstrapData: (options: WorkerSettings) => WorkerSettings;
-export declare const terminateWorkerQuietly: (worker: SpawnedWorker) => void;
+/** Terminate a worker, optionally waiting for it to exit. */
+export declare const terminateWorkerQuietly: (worker: SpawnedWorker, awaitExit?: boolean) => Promise<void>;

package/src/runtime/worker-common.js

@@ -90,13 +90,28 @@ export const serializeWorkerBootstrapData = (options) => {
         },
     };
 };
-export const terminateWorkerQuietly = (worker) => {
+// Bound the wait for process-worker exit.
+const WORKER_EXIT_WAIT_MS = 2_000;
+/** Terminate a worker, optionally waiting for it to exit. */
+export const terminateWorkerQuietly = (worker, awaitExit = false) => {
     try {
-        // Runaway worker termination can be slow or stuck on some runtimes; once the
-        // pool is closing it must not keep the host process alive.
-        worker.unref?.();
-        void Promise.resolve(worker.terminate()).catch(() => { });
+        if (!awaitExit)
+            worker.unref?.();
+        const exited = Promise.resolve(worker.terminate()).then(() => { }, () => { });
+        if (!awaitExit)
+            return exited;
+        return new Promise((resolve) => {
+            const timer = setTimeout(() => {
+                worker.unref?.();
+                resolve();
+            }, WORKER_EXIT_WAIT_MS);
+            void exited.then(() => {
+                clearTimeout(timer);
+                resolve();
+            });
+        });
     }
     catch {
+        return Promise.resolve();
     }
 };

package/src/types.d.ts

@@ -13,6 +13,8 @@ type WorkerInvoke = (args: Uint8Array) => Promise<unknown>;
 interface WorkerContext {
     txIdle(): boolean;
     call(descriptor: WorkerCall): WorkerInvoke;
+    /** Ask the worker to leave its dispatch loop before termination. */
+    requestStop?(): Promise<void>;
     kills(): Promise<void>;
 }
 type CreateContext = WorkerContext;
@@ -34,6 +36,8 @@ type WorkerData = {
     payloadConfig?: PayloadBufferOptions;
     bufferReferenceReturn?: "copy" | "borrow";
     permission?: ResolvedPermissionProtocol;
+    /** Whether this host can arm an async completion waiter on the return lock. */
+    notifyOnHostPublish?: boolean;
     /**
      * Work stealing. When present, `lock` is a submit region shared by every
      * worker and this worker claims from it as consumer `consumerId` of
@@ -341,13 +345,46 @@ type WorkerTimers = {
 };
 type DispatcherSettings = {
     /**
-     * How many immediate notify loops before backoff kicks in.
+     * How many immediate notify loops before the dispatcher stops re-arming the
+     * pump for free.
+     *
+     * The default depends on what the dispatcher escalates *to*. Polling
+     * escalates to a `setTimeout` ladder costing ~1.1ms even at delay 0, so it
+     * defaults to 128 — escalating is expensive and the wide window also batches
+     * completions. A doorbell escalates to `Atomics.waitAsync` at roughly the
+     * price of one hop, so pools that have one default to 1. Pools without a
+     * doorbell — Deno, or `doorbell: false` — keep 128.
+     *
+     * Setting this explicitly opts out of that coupling for every pool shape.
      */
     stallFreeLoops?: number;
     /**
      * Max backoff delay (milliseconds).
      */
     maxBackoffMs?: number;
+    /**
+     * Replace idle completion polling with an `Atomics.waitAsync` doorbell when
+     * the host runtime supports it.
+     *
+     * Defaults to enabled on Node and Bun at any worker count, where it costs
+     * 1.6-3.9x less host CPU per completed call. Under HTTP load, where the host
+     * has real work of its own, that converts to +13% to +36% throughput.
+     *
+     * Turn it off for a pool that oversubscribes its machine. The doorbell only
+     * progresses when the host gets scheduled, so once workers occupy every core
+     * a wake must preempt one: measured +5% to +12.6% rps while workers+host fit
+     * within the cores, -22% to -32% once they do not. That is not gated
+     * automatically because the core count cannot be probed portably.
+     *
+     * Forced off, overriding an explicit `true`, where a doorbell cannot work:
+     * on Deno, whose `waitAsync` does not wake an idle event loop, and for
+     * process workers, which live in another process and so cannot ring a host
+     * waiter at all — V8 keeps its Atomics waiter list per isolate, which is why
+     * they wake through a native futex addon instead. Compiled (Porffor) workers
+     * never reach this path: they reject `host` outright and use pipes rather
+     * than shared memory.
+     */
+    doorbell?: boolean;
     /**
      * Host dispatcher topology.
      * - `"per-thread"`: each worker owns its dispatcher and macro channel.
@@ -366,14 +403,12 @@ type DispatcherSettings = {
      * claim from, private return lanes, and a pool-global pending registry. The
      * endpoint that claims a task owns its response.
      *
-     * Enabled by default for multi-worker thread pools unless a balancer or
-     * private-lane dispatcher was explicitly selected. One-worker pools,
-     * inliners, compiled/Porffor workers, process workers, and pools above the
-     * current 31-claimant protocol limit retain their existing transport.
-     * Explicitly requesting `true` with process workers is rejected because
-     * their transport uses separately mapped process-shared memory. Set `false`
-     * (or `KNITTING_STEAL=0`) to opt out; `KNITTING_STEAL=1` explicitly opts in
-     * and overrides a balancer/dispatcher selection.
+     * Enabled by default for compatible multi-worker thread and process pools
+     * unless a balancer or private-lane dispatcher was explicitly selected.
+     * One-worker pools, inliners, compiled/Porffor workers, and pools above the
+     * current 31-claimant protocol limit retain their existing transport. Set
+     * `false` (or `KNITTING_STEAL=0`) to opt out; `KNITTING_STEAL=1` explicitly
+     * opts in and overrides a balancer/dispatcher selection.
      */
     steal?: boolean;
     /**

package/src/worker/loop.js

@@ -2,7 +2,7 @@ import { addRuntimeDataListener, createRuntimeMessageChannel, RUNTIME_IS_MAIN_TH
 import { isSharedBufferSource } from "../common/shared-buffer-region.js";
 import { isLockBufferTextCompat } from "../common/shared-buffer-text.js";
 import { createWorkerRxQueue } from "./rx-queue.js";
-import { createSharedMemoryTransport } from "../ipc/transport/shared-memory.js";
+import { createSharedMemoryTransport, WORKER_STOP, } from "../ipc/transport/shared-memory.js";
 import { lock2 } from "../memory/lock.js";
 // Side-effect import: registers the payload codec (cycle break for Andromeda;
 // see lock.ts). Must run before any lock2() call.
@@ -73,7 +73,7 @@ export const workerMainLoop = async (startupData) => {
     installTerminationGuard();
     installUnhandledRejectionSilencer();
     installPerformanceNowGuard();
-    const { debug, sab, thread, startAt, workerOptions, lock, returnLock, abortSignalSAB, abortSignalMax, payloadConfig, bufferReferenceReturn, permission, totalNumberOfThread, list, ids, names, at, steal, } = startupData;
+    const { debug, sab, thread, startAt, workerOptions, lock, returnLock, abortSignalSAB, abortSignalMax, payloadConfig, bufferReferenceReturn, permission, notifyOnHostPublish, totalNumberOfThread, list, ids, names, at, steal, } = startupData;
     scrubWorkerDataSensitiveBuffers(startupData);
     assertWorkerSharedMemoryBootData({ sab, lock, returnLock });
     const debugNamespaces = resolveDebugNamespaces(debug);
@@ -120,21 +120,23 @@ export const workerMainLoop = async (startupData) => {
         payloadConfig,
         textCompat: returnLock.textCompat,
         processBoundary: RUNTIME_IS_PROCESS_WORKER,
+        // The host parks on this lock's publication word when it has no work to
+        // flush. Request locks are host-produced and must not wake that waiter.
+        notifyOnHostPublish,
     });
     const timers = workerOptions?.timers;
     const spinMicroseconds = timers?.spinMicroseconds ??
-        Math.max(1, totalNumberOfThread) * 50;
+        (totalNumberOfThread <= 1 ? 50 : 0);
     const parkMs = dbg !== undefined
         ? Number.POSITIVE_INFINITY
-        : (timers?.parkMs ??
-            Math.max(1, totalNumberOfThread) * 50);
+        : (timers?.parkMs ?? 1);
     const pauseSpin = (() => {
         const fn = typeof timers?.pauseNanoseconds === "number"
             ? whilePausing({ pauseInNanoseconds: timers.pauseNanoseconds })
             : pauseGeneric;
         return () => fn(); // always a closure wrapper
     })();
-    const { opView, rxStatus, txStatus } = signals;
+    const { opView, rxStatus, txStatus, stopView } = signals;
     const a_store = Atomics.store;
     const a_load = Atomics.load;
     const nativeWaitU32 = getProcessWorkerNativeWaitU32();
@@ -258,17 +260,33 @@ export const workerMainLoop = async (startupData) => {
         }
         : pauseUntil;
     const flushBeforeClaim = steal !== undefined;
+    /** Leave the dispatch loop and acknowledge shutdown. */
+    const stopLoop = () => {
+        a_store(stopView, 0, WORKER_STOP.acknowledged);
+        a_store(rxStatus, 0, 0);
+        try {
+            port1.close?.();
+            port2.close?.();
+        }
+        catch { }
+    };
     const loop = () => {
         isInMacro = false;
         let progressed = true;
         let awaiting = 0;
         while (true) {
+            if (stopView[0] !== WORKER_STOP.running)
+                return stopLoop();
             if (flushBeforeClaim) {
                 // Stealing only. Flush finished work before taking more on: a computed
                 // response should not wait behind a claim, and that claim can block on
                 // a peer withdrawing its intent. Measured to hurt the per-lane path,
                 // where a claim is a cheap private decode and picking work up promptly
                 // matters more, so the classic order is kept below.
+                // Reordering must not make `progressed` sticky: it answers "did this
+                // iteration move anything", so it has to start false every pass or the
+                // park below is unreachable and the worker spins a core forever.
+                progressed = false;
                 if (_hasCompleted()) {
                     if (_writeBatch(WRITE_MAX) > 0)
                         progressed = true;