knitting 0.1.60 → 0.1.62

package/src/runtime/worker-common.js

@@ -0,0 +1,103 @@
+// Worker helpers shared by every worker mode. Kept out of `process-worker.ts`
+// so the thread and web-worker paths do not pull the process-worker module —
+// and, through it, the whole native connections tree — into their bundles.
+import { ProcessSharedBuffer } from "../connections/process-shared-buffer.js";
+const execFlagKey = (flag) => flag.split("=", 1)[0];
+const NODE_PERMISSION_EXEC_FLAGS = new Set([
+    "--permission",
+    "--experimental-permission",
+    "--allow-fs-read",
+    "--allow-fs-write",
+    "--allow-worker",
+    "--allow-child-process",
+    "--allow-net",
+    "--allow-addons",
+    "--allow-ffi",
+    "--allow-wasi",
+]);
+const NODE_WORKER_SAFE_EXEC_FLAGS = new Set([
+    "--experimental-ffi",
+    "--experimental-transform-types",
+    "--expose-gc",
+    "--no-warnings",
+    ...NODE_PERMISSION_EXEC_FLAGS,
+]);
+const isNodeWorkerSafeExecFlag = (flag) => NODE_WORKER_SAFE_EXEC_FLAGS.has(execFlagKey(flag));
+const isNodePermissionExecFlag = (flag) => NODE_PERMISSION_EXEC_FLAGS.has(execFlagKey(flag));
+export const toWorkerSafeExecArgv = (flags) => {
+    if (!flags || flags.length === 0)
+        return undefined;
+    const filtered = flags.filter(isNodeWorkerSafeExecFlag);
+    if (filtered.length === 0)
+        return undefined;
+    const seen = new Set();
+    const deduped = [];
+    for (const flag of filtered) {
+        if (seen.has(flag))
+            continue;
+        seen.add(flag);
+        deduped.push(flag);
+    }
+    return deduped;
+};
+export const toWorkerCompatExecArgv = (flags) => {
+    const safe = toWorkerSafeExecArgv(flags);
+    if (!safe || safe.length === 0)
+        return undefined;
+    const compat = safe.filter((flag) => !isNodePermissionExecFlag(flag));
+    return compat.length > 0 ? compat : undefined;
+};
+const isPlainRecord = (value) => {
+    if (value === null || typeof value !== "object")
+        return false;
+    const prototype = Object.getPrototypeOf(value);
+    return prototype === Object.prototype || prototype === null;
+};
+const serializeWorkerBootstrapValue = (value, seen = new WeakMap()) => {
+    if (value instanceof ProcessSharedBuffer)
+        return value.toMetadata();
+    if (value === null || typeof value !== "object")
+        return value;
+    const existing = seen.get(value);
+    if (existing !== undefined)
+        return existing;
+    if (Array.isArray(value)) {
+        const out = [];
+        seen.set(value, out);
+        for (const item of value) {
+            out.push(serializeWorkerBootstrapValue(item, seen));
+        }
+        return out;
+    }
+    if (!isPlainRecord(value))
+        return value;
+    const out = {};
+    seen.set(value, out);
+    for (const [key, item] of Object.entries(value)) {
+        out[key] = serializeWorkerBootstrapValue(item, seen);
+    }
+    return out;
+};
+export const serializeWorkerBootstrapData = (options) => {
+    const bootstrap = options.bootstrap;
+    if (bootstrap === undefined || bootstrap.data === undefined)
+        return options;
+    return {
+        ...options,
+        bootstrap: {
+            ...bootstrap,
+            data: serializeWorkerBootstrapValue(bootstrap.data),
+        },
+    };
+};
+export const terminateWorkerQuietly = (worker) => {
+    try {
+        // Runaway worker termination can be slow or stuck on some runtimes; once the
+        // pool is closing it must not keep the host process alive.
+        worker.unref?.();
+        return Promise.resolve(worker.terminate()).then(() => { }, () => { });
+    }
+    catch {
+        return Promise.resolve();
+    }
+};

package/src/types.d.ts

@@ -34,6 +34,19 @@ type WorkerData = {
     payloadConfig?: PayloadBufferOptions;
     bufferReferenceReturn?: "copy" | "borrow";
     permission?: ResolvedPermissionProtocol;
+    /** Whether this host can arm an async completion waiter on the return lock. */
+    notifyOnHostPublish?: boolean;
+    /**
+     * Work stealing. When present, `lock` is a submit region shared by every
+     * worker and this worker claims from it as consumer `consumerId` of
+     * `consumers`, rather than owning a private request lane. `returnLock` stays
+     * private — the endpoint that claims a task owns its response.
+     */
+    steal?: {
+        consumers: number;
+        consumerId: number;
+        regionLanes: number;
+    };
 };
 type UnsafeOptions = {
     /**
@@ -330,24 +343,82 @@ type WorkerTimers = {
 };
 type DispatcherSettings = {
     /**
-     * How many immediate notify loops before backoff kicks in.
+     * How many immediate notify loops before the dispatcher stops re-arming the
+     * pump for free.
+     *
+     * The default depends on what the dispatcher escalates *to*. Polling
+     * escalates to a `setTimeout` ladder costing ~1.1ms even at delay 0, so it
+     * defaults to 128 — escalating is expensive and the wide window also batches
+     * completions. A doorbell escalates to `Atomics.waitAsync` at roughly the
+     * price of one hop, so pools that have one default to 1. Pools without a
+     * doorbell — Deno, or `doorbell: false` — keep 128.
+     *
+     * Setting this explicitly opts out of that coupling for every pool shape.
      */
     stallFreeLoops?: number;
     /**
      * Max backoff delay (milliseconds).
      */
     maxBackoffMs?: number;
+    /**
+     * Replace idle completion polling with an `Atomics.waitAsync` doorbell when
+     * the host runtime supports it.
+     *
+     * Defaults to enabled on Node and Bun at any worker count, where it costs
+     * 1.6-3.9x less host CPU per completed call. Under HTTP load, where the host
+     * has real work of its own, that converts to +13% to +36% throughput.
+     *
+     * Turn it off for a pool that oversubscribes its machine. The doorbell only
+     * progresses when the host gets scheduled, so once workers occupy every core
+     * a wake must preempt one: measured +5% to +12.6% rps while workers+host fit
+     * within the cores, -22% to -32% once they do not. That is not gated
+     * automatically because the core count cannot be probed portably.
+     *
+     * Forced off, overriding an explicit `true`, where a doorbell cannot work:
+     * on Deno, whose `waitAsync` does not wake an idle event loop, and for
+     * process workers, which live in another process and so cannot ring a host
+     * waiter at all — V8 keeps its Atomics waiter list per isolate, which is why
+     * they wake through a native futex addon instead. Compiled (Porffor) workers
+     * never reach this path: they reject `host` outright and use pipes rather
+     * than shared memory.
+     */
+    doorbell?: boolean;
     /**
      * Host dispatcher topology.
      * - `"per-thread"`: each worker owns its dispatcher and macro channel.
      * - `"serial-channel"`: each worker keeps its own dispatcher check state, but
      *   one shared channel runs all lane checks from first to last.
      *
-     * Experimental default: `"per-thread"` on Bun or with one worker, otherwise
-     * `"serial-channel"` for multi-worker Node/Deno pools. Can also be forced
-     * with the `KNITTING_DISPATCHER` env var (`serial-channel` or `per-thread`).
+     * Used by private-lane pools. Its experimental default is `"per-thread"` on
+     * Bun or with one worker, otherwise `"serial-channel"` for multi-worker
+     * Node/Deno pools. Selecting it explicitly keeps private lanes unless
+     * `steal: true` is also explicit. Can also be forced with the
+     * `KNITTING_DISPATCHER` env var (`serial-channel` or `per-thread`).
      */
     dispatcher?: "per-thread" | "serial-channel";
+    /**
+     * Work stealing: one shared submit region that any worker may
+     * claim from, private return lanes, and a pool-global pending registry. The
+     * endpoint that claims a task owns its response.
+     *
+     * Enabled by default for compatible multi-worker thread and process pools
+     * unless a balancer or private-lane dispatcher was explicitly selected.
+     * One-worker pools, inliners, compiled/Porffor workers, and pools above the
+     * current 31-claimant protocol limit retain their existing transport. Set
+     * `false` (or `KNITTING_STEAL=0`) to opt out; `KNITTING_STEAL=1` explicitly
+     * opts in and overrides a balancer/dispatcher selection.
+     */
+    steal?: boolean;
+    /**
+     * Lanes claimed per stealing handshake (a power of two, `slots / g >=
+     * workers + 1`). Defaults to the widest region the lane budget allows, which
+     * amortises arbitration best for cheap tasks.
+     *
+     * **A region is a batch.** For expensive tasks, a wide region lets one worker
+     * claim work the others could have run in parallel; set this to `1` (or a
+     * small value) when per-task cost dominates arbitration cost.
+     */
+    stealRegionLanes?: number;
 };
 type CreatePool = {
     /** Number of workers. Default: 1. */

package/src/worker/loop.js

@@ -73,7 +73,7 @@ export const workerMainLoop = async (startupData) => {
     installTerminationGuard();
     installUnhandledRejectionSilencer();
     installPerformanceNowGuard();
-    const { debug, sab, thread, startAt, workerOptions, lock, returnLock, abortSignalSAB, abortSignalMax, payloadConfig, bufferReferenceReturn, permission, totalNumberOfThread, list, ids, names, at, } = startupData;
+    const { debug, sab, thread, startAt, workerOptions, lock, returnLock, abortSignalSAB, abortSignalMax, payloadConfig, bufferReferenceReturn, permission, notifyOnHostPublish, totalNumberOfThread, list, ids, names, at, steal, } = startupData;
     scrubWorkerDataSensitiveBuffers(startupData);
     assertWorkerSharedMemoryBootData({ sab, lock, returnLock });
     const debugNamespaces = resolveDebugNamespaces(debug);
@@ -105,6 +105,11 @@ export const workerMainLoop = async (startupData) => {
         payloadConfig,
         textCompat: lock.textCompat,
         processBoundary: RUNTIME_IS_PROCESS_WORKER,
+        // Under stealing the request region is shared, so decode() becomes a
+        // region-Dekker claim against the other endpoints.
+        consumers: steal?.consumers,
+        consumerId: steal?.consumerId,
+        regionLanes: steal?.regionLanes,
     });
     const returnLockState = lock2({
         headers: returnLock.headers,
@@ -115,6 +120,9 @@ export const workerMainLoop = async (startupData) => {
         payloadConfig,
         textCompat: returnLock.textCompat,
         processBoundary: RUNTIME_IS_PROCESS_WORKER,
+        // The host parks on this lock's publication word when it has no work to
+        // flush. Request locks are host-produced and must not wake that waiter.
+        notifyOnHostPublish,
     });
     const timers = workerOptions?.timers;
     const spinMicroseconds = timers?.spinMicroseconds ??
@@ -163,6 +171,7 @@ export const workerMainLoop = async (startupData) => {
         returnLock: returnLockState,
         borrowReturnedBufferReferences: bufferReferenceReturn === "borrow",
         hasAborted: abortSignals?.hasAborted,
+        stealing: steal !== undefined,
     });
     installBufferReferenceReleaseListener(releaseReturnedBufferReference);
     a_store(rxStatus, 0, 1);
@@ -175,6 +184,7 @@ export const workerMainLoop = async (startupData) => {
         pauseInNanoseconds: timers?.pauseNanoseconds,
         enqueueLock,
         write: () => hasCompleted() ? writeBatch(WRITE_MAX) : 0,
+        flushBeforeClaim: steal !== undefined,
         nativeWaitU32,
         useSharedMemoryWait: !(RUNTIME_IS_PROCESS_WORKER &&
             RUNTIME === "node" &&
@@ -250,15 +260,34 @@ export const workerMainLoop = async (startupData) => {
             pauseUntil(value, spinMicroseconds, parkMs);
         }
         : pauseUntil;
+    const flushBeforeClaim = steal !== undefined;
     const loop = () => {
         isInMacro = false;
         let progressed = true;
         let awaiting = 0;
         while (true) {
-            progressed = _enqueueLock();
-            if (_hasCompleted()) {
-                if (_writeBatch(WRITE_MAX) > 0)
-                    progressed = true;
+            if (flushBeforeClaim) {
+                // Stealing only. Flush finished work before taking more on: a computed
+                // response should not wait behind a claim, and that claim can block on
+                // a peer withdrawing its intent. Measured to hurt the per-lane path,
+                // where a claim is a cheap private decode and picking work up promptly
+                // matters more, so the classic order is kept below.
+                // Reordering must not make `progressed` sticky: it answers "did this
+                // iteration move anything", so it has to start false every pass or the
+                // park below is unreachable and the worker spins a core forever.
+                progressed = false;
+                if (_hasCompleted()) {
+                    if (_writeBatch(WRITE_MAX) > 0)
+                        progressed = true;
+                }
+                progressed = _enqueueLock() || progressed;
+            }
+            else {
+                progressed = _enqueueLock();
+                if (_hasCompleted()) {
+                    if (_writeBatch(WRITE_MAX) > 0)
+                        progressed = true;
+                }
             }
             _drainReturnReleases();
             if (_hasPending()) {

package/src/worker/rx-queue.d.ts

@@ -9,9 +9,19 @@ type ArgumentsForCreateWorkerQueue = {
     borrowReturnedBufferReferences?: boolean;
     hasAborted?: (signal: number) => boolean;
     now?: () => number;
+    /**
+     * Work-stealing discipline: only claim when this worker has nothing left to
+     * run. Without it a worker keeps claiming whole regions on top of a backlog
+     * it has not started, which is exactly the hoarding stealing exists to
+     * prevent — the queued tasks would be better taken by an idle peer.
+     *
+     * Off for the classic per-lane path, where the host has already assigned
+     * these tasks to this lane and batching them is a win.
+     */
+    stealing?: boolean;
 };
 export type CreateWorkerRxQueue = ReturnType<typeof createWorkerRxQueue>;
-export declare const createWorkerRxQueue: ({ listOfFunctions, workerOptions, lock, returnLock, borrowReturnedBufferReferences, hasAborted, now, }: ArgumentsForCreateWorkerQueue) => {
+export declare const createWorkerRxQueue: ({ listOfFunctions, workerOptions, lock, returnLock, borrowReturnedBufferReferences, hasAborted, now, stealing, }: ArgumentsForCreateWorkerQueue) => {
     hasCompleted: () => boolean;
     hasPending: () => boolean;
     writeBatch: (max: number) => number;

package/src/worker/rx-queue.js

@@ -2,7 +2,7 @@ import RingQueue from "../ipc/tools/ring-queue.js";
 import { attachPayloadTransportFinalizer, runTaskFinalizers, TaskFlag, TaskIndex, } from "../memory/lock.js";
 import { composeWorkerRunner } from "./composable-runners.js";
 import { BUFFER_REFERENCE_RETURN_RELEASE_TOKEN } from "../connections/buffer-reference.js";
-export const createWorkerRxQueue = ({ listOfFunctions, workerOptions, lock, returnLock, borrowReturnedBufferReferences, hasAborted, now, }) => {
+export const createWorkerRxQueue = ({ listOfFunctions, workerOptions, lock, returnLock, borrowReturnedBufferReferences, hasAborted, now, stealing, }) => {
     const PLACE_HOLDER = (_) => {
         throw ("UNREACHABLE FROM PLACE HOLDER (thread)");
     };
@@ -71,6 +71,9 @@ export const createWorkerRxQueue = ({ listOfFunctions, workerOptions, lock, retu
     const { decode, resolved } = lock;
     const resolvedShift = () => resolved.shiftNoClear();
     const enqueueLock = () => {
+        // Steal only when idle: leaving work unclaimed lets a free peer take it.
+        if (stealing && toWork.size !== 0)
+            return false;
         if (!decode())
             return false;
         let task = resolvedShift();

package/src/worker/timers.d.ts

@@ -4,7 +4,7 @@ type PauseOptions = {
 export type NativeWaitU32 = (buffer: ArrayBuffer | SharedArrayBuffer, byteOffset: number, expected: number, timeoutMs?: number) => unknown;
 export declare const whilePausing: ({ pauseInNanoseconds }: PauseOptions) => () => void;
 export declare const pauseGeneric: () => void;
-export declare const sleepUntilChanged: ({ at, opView, pauseInNanoseconds, rxStatus, txStatus, enqueueLock, write, nativeWaitU32, useSharedMemoryWait, }: {
+export declare const sleepUntilChanged: ({ at, opView, pauseInNanoseconds, rxStatus, txStatus, enqueueLock, write, nativeWaitU32, useSharedMemoryWait, flushBeforeClaim, }: {
     opView: Int32Array;
     rxStatus: Int32Array;
     txStatus: Int32Array;
@@ -14,5 +14,12 @@ export declare const sleepUntilChanged: ({ at, opView, pauseInNanoseconds, rxSta
     write?: () => number | boolean;
     nativeWaitU32?: NativeWaitU32;
     useSharedMemoryWait?: boolean;
+    /**
+     * Stealing only: write results back before attempting to claim more work.
+     * A claim can spin waiting for a peer to withdraw its intent, so claiming
+     * first puts arbitration latency in front of a finished response. Measured
+     * to hurt the per-lane path, so it stays off by default.
+     */
+    flushBeforeClaim?: boolean;
 }) => (value: number, spinMicroseconds: number, parkMs?: number) => void;
 export {};

package/src/worker/timers.js

@@ -30,7 +30,9 @@ const a_load = Atomics.load;
 const a_store = Atomics.store;
 const a_wait = typeof Atomics.wait === "function" ? Atomics.wait : undefined;
 const p_now = performance.now.bind(performance);
-const waitFallbackView = a_wait === undefined
+// `SharedArrayBuffer` is undeclared on pages that are not cross-origin
+// isolated, and this module has to load anyway so the pool can report why.
+const waitFallbackView = a_wait === undefined || typeof SharedArrayBuffer !== "function"
     ? undefined
     : new Int32Array(new SharedArrayBuffer(Int32Array.BYTES_PER_ELEMENT));
 const a_pause = "pause" in Atomics
@@ -59,26 +61,27 @@ export const whilePausing = ({ pauseInNanoseconds }) => {
     return () => a_pause(forNanoseconds);
 };
 export const pauseGeneric = whilePausing({});
-export const sleepUntilChanged = ({ at, opView, pauseInNanoseconds, rxStatus, txStatus, enqueueLock, write, nativeWaitU32, useSharedMemoryWait = true, }) => {
+export const sleepUntilChanged = ({ at, opView, pauseInNanoseconds, rxStatus, txStatus, enqueueLock, write, nativeWaitU32, useSharedMemoryWait = true, flushBeforeClaim = false, }) => {
     const pause = pauseInNanoseconds === undefined
         ? pauseGeneric
         : whilePausing({ pauseInNanoseconds });
-    const tryProgress = () => {
-        let progressed = false;
-        if (enqueueLock())
-            progressed = true;
-        if (write) {
-            const wrote = write();
-            if (typeof wrote === "number") {
-                if (wrote > 0)
-                    progressed = true;
-            }
-            else if (wrote === true) {
-                progressed = true;
-            }
-        }
-        return progressed;
+    const flushWrite = () => {
+        if (!write)
+            return false;
+        const wrote = write();
+        if (typeof wrote === "number")
+            return wrote > 0;
+        return wrote === true;
     };
+    const tryProgress = flushBeforeClaim
+        ? () => {
+            const wrote = flushWrite();
+            return enqueueLock() || wrote;
+        }
+        : () => {
+            const claimed = enqueueLock();
+            return flushWrite() || claimed;
+        };
     return (value, spinMicroseconds, parkMs) => {
         const until = p_now() + (spinMicroseconds / 1000);
         maybeGc();