knitting 0.1.54 → 0.1.55

package/src/worker/loop.js

@@ -1,382 +0,0 @@
-import { addRuntimeDataListener, createRuntimeMessageChannel, RUNTIME_IS_MAIN_THREAD, RUNTIME_IS_PROCESS_WORKER, RUNTIME_PARENT_PORT, RUNTIME_WORKER_DATA, } from "../common/worker-runtime.js";
-import { isSharedBufferSource } from "../common/shared-buffer-region.js";
-import { isLockBufferTextCompat } from "../common/shared-buffer-text.js";
-import { createWorkerRxQueue } from "./rx-queue.js";
-import { createSharedMemoryTransport } from "../ipc/transport/shared-memory.js";
-import { lock2 } from "../memory/lock.js";
-import { getFunctions } from "./task-loader.js";
-import { pauseGeneric, sleepUntilChanged, whilePausing } from "./timers.js";
-import { RUNTIME, SET_IMMEDIATE } from "../common/runtime.js";
-import { getNodeProcess } from "../common/node-compat.js";
-import { assertWorkerImportsResolved, assertWorkerSharedMemoryBootData, installPerformanceNowGuard, installTerminationGuard, installUnhandledRejectionSilencer, scrubWorkerDataSensitiveBuffers, } from "./safety/index.js";
-import { signalAbortFactory } from "../shared/abortSignal.js";
-import { runWorkerBootstrap } from "./bootstrap.js";
-import { getProcessWorkerNativeWaitU32, installProcessWorkerBootstrap, } from "./process-worker-bootstrap.js";
-import { readBufferReferenceReturnReleaseMessage, } from "../connections/buffer-reference.js";
-import { resolveDebugNamespaces } from "../debug/gate.js";
-const WORKER_FATAL_MESSAGE_KEY = "__knittingWorkerFatal";
-const reportWorkerStartupFatal = (error) => {
-    const message = String(error?.message ?? error);
-    const payload = {
-        [WORKER_FATAL_MESSAGE_KEY]: message,
-    };
-    let reported = false;
-    try {
-        RUNTIME_PARENT_PORT?.postMessage?.(payload);
-        reported = RUNTIME_PARENT_PORT !== undefined;
-    }
-    catch {
-    }
-    if (!reported) {
-        try {
-            globalThis.postMessage(payload);
-            reported = true;
-        }
-        catch {
-        }
-    }
-    if (!reported) {
-        try {
-            console.error(`Worker startup failed: ${message}`);
-        }
-        catch {
-        }
-        if (RUNTIME_IS_PROCESS_WORKER) {
-            try {
-                getNodeProcess()?.exit?.(1);
-            }
-            catch {
-            }
-        }
-    }
-};
-const installBufferReferenceReleaseListener = (releaseReturnedBufferReference) => {
-    const handleMessage = (message) => {
-        const token = readBufferReferenceReturnReleaseMessage(message);
-        if (token !== undefined)
-            releaseReturnedBufferReference(token);
-    };
-    if (RUNTIME_PARENT_PORT !== undefined) {
-        addRuntimeDataListener(RUNTIME_PARENT_PORT, handleMessage);
-        return;
-    }
-    const scope = globalThis;
-    scope.addEventListener?.("message", (event) => {
-        handleMessage(event?.data);
-    });
-};
-export const workerMainLoop = async (startupData) => {
-    // Startup-only safety layer: no per-iteration checks in the hot loop.
-    installTerminationGuard();
-    installUnhandledRejectionSilencer();
-    installPerformanceNowGuard();
-    const { debug, sab, thread, startAt, workerOptions, lock, returnLock, abortSignalSAB, abortSignalMax, payloadConfig, bufferReferenceReturn, permission, totalNumberOfThread, list, ids, names, at, } = startupData;
-    scrubWorkerDataSensitiveBuffers(startupData);
-    assertWorkerSharedMemoryBootData({ sab, lock, returnLock });
-    const debugNamespaces = resolveDebugNamespaces(debug);
-    const dbg = debugNamespaces.size > 0
-        ? await import("../debug/handle.js").then((module) => module.initDebug({
-            name: `w${thread}`,
-            runtime: RUNTIME,
-            namespaces: debugNamespaces,
-        }))
-        : undefined;
-    let Comment;
-    (function (Comment) {
-        Comment[Comment["thisIsAHint"] = 0] = "thisIsAHint";
-    })(Comment || (Comment = {}));
-    const signals = createSharedMemoryTransport({
-        sabObject: {
-            sharedSab: sab,
-        },
-        isMain: false,
-        thread,
-        startTime: startAt,
-    });
-    const lockState = lock2({
-        headers: lock.headers,
-        headerSlotStrideU32: lock.headerSlotStrideU32,
-        LockBoundSector: lock.lockSector,
-        payload: lock.payload,
-        payloadSector: lock.payloadSector,
-        payloadConfig,
-        textCompat: lock.textCompat,
-    });
-    const returnLockState = lock2({
-        headers: returnLock.headers,
-        headerSlotStrideU32: returnLock.headerSlotStrideU32,
-        LockBoundSector: returnLock.lockSector,
-        payload: returnLock.payload,
-        payloadSector: returnLock.payloadSector,
-        payloadConfig,
-        textCompat: returnLock.textCompat,
-    });
-    const timers = workerOptions?.timers;
-    const spinMicroseconds = timers?.spinMicroseconds ??
-        Math.max(1, totalNumberOfThread) * 50;
-    const parkMs = dbg !== undefined
-        ? Number.POSITIVE_INFINITY
-        : (timers?.parkMs ??
-            Math.max(1, totalNumberOfThread) * 50);
-    const pauseSpin = (() => {
-        const fn = typeof timers?.pauseNanoseconds === "number"
-            ? whilePausing({ pauseInNanoseconds: timers.pauseNanoseconds })
-            : pauseGeneric;
-        return () => fn(); // always a closure wrapper
-    })();
-    const { opView, rxStatus, txStatus } = signals;
-    const a_store = Atomics.store;
-    const a_load = Atomics.load;
-    const nativeWaitU32 = getProcessWorkerNativeWaitU32();
-    await runWorkerBootstrap({
-        bootstrap: workerOptions?.bootstrap,
-        thread,
-        totalNumberOfThread,
-    });
-    dbg?.envPhase("bootstrap");
-    const listOfFunctions = await getFunctions({
-        list,
-        isWorker: true,
-        ids,
-        names,
-        at,
-        permission,
-    });
-    dbg?.envPhase("tasks");
-    dbg?.log("imports", `${listOfFunctions.length} task(s) from ${list.map((spec) => spec.split(/[\\/]/).pop() || spec).join(", ")}`);
-    assertWorkerImportsResolved({ list, ids, names, listOfFunctions });
-    const abortSignals = abortSignalSAB
-        ? signalAbortFactory({
-            sab: abortSignalSAB,
-            maxSignals: abortSignalMax,
-        })
-        : undefined;
-    const { enqueueLock, serviceBatchImmediate, hasCompleted, writeBatch, hasPending, getAwaiting, drainReturnReleases, releaseReturnedBufferReference, } = createWorkerRxQueue({
-        listOfFunctions,
-        workerOptions,
-        lock: lockState,
-        returnLock: returnLockState,
-        borrowReturnedBufferReferences: bufferReferenceReturn === "borrow",
-        hasAborted: abortSignals?.hasAborted,
-    });
-    installBufferReferenceReleaseListener(releaseReturnedBufferReference);
-    a_store(rxStatus, 0, 1);
-    const WRITE_MAX = 64;
-    const pauseUntil = sleepUntilChanged({
-        opView,
-        at: 0,
-        rxStatus,
-        txStatus,
-        pauseInNanoseconds: timers?.pauseNanoseconds,
-        enqueueLock,
-        write: () => hasCompleted() ? writeBatch(WRITE_MAX) : 0,
-        nativeWaitU32,
-        useSharedMemoryWait: !(RUNTIME_IS_PROCESS_WORKER &&
-            RUNTIME === "node" &&
-            nativeWaitU32 === undefined),
-    });
-    const channel = createRuntimeMessageChannel();
-    const port1 = channel.port1;
-    const port2 = channel.port2;
-    const post2 = (message) => port2.postMessage(message);
-    let isInMacro = false;
-    let awaitingSpins = 0;
-    let lastAwaiting = 0;
-    const MAX_AWAITING_MS = 10;
-    let wakeToken = a_load(opView, 0);
-    const scheduleMacro = () => {
-        if (isInMacro)
-            return;
-        isInMacro = true;
-        post2(null);
-    };
-    const scheduleTimer = (delayMs) => {
-        if (isInMacro)
-            return;
-        isInMacro = true;
-        if (delayMs <= 0 && typeof SET_IMMEDIATE === "function") {
-            SET_IMMEDIATE(loop);
-            return;
-        }
-        if (delayMs <= 0) {
-            post2(null);
-            return;
-        }
-        if (typeof setTimeout === "function") {
-            setTimeout(loop, delayMs);
-            return;
-        }
-        post2(null);
-    };
-    const traceSignals = dbg?.enabled("signals") === true;
-    const _hasCompleted = hasCompleted;
-    const _hasPending = hasPending;
-    const _getAwaiting = getAwaiting;
-    const _drainReturnReleases = drainReturnReleases;
-    const _pauseSpin = pauseSpin;
-    const _enqueueLock = traceSignals
-        ? () => {
-            const progressed = enqueueLock();
-            if (progressed) {
-                dbg.log("signals", "work from=host");
-            }
-            return progressed;
-        }
-        : enqueueLock;
-    const _writeBatch = traceSignals
-        ? (max) => {
-            const wrote = writeBatch(max);
-            if (wrote > 0)
-                dbg.log("signals", `result count=${wrote}`);
-            return wrote;
-        }
-        : writeBatch;
-    const _serviceBatchImmediate = traceSignals
-        ? () => {
-            const ran = serviceBatchImmediate();
-            if (ran > 0)
-                dbg.log("signals", `run count=${ran}`);
-            return ran;
-        }
-        : serviceBatchImmediate;
-    const _pauseUntil = traceSignals
-        ? (value, spinMicroseconds, parkMs) => {
-            dbg.log("signals", `idle token=${value}`);
-            pauseUntil(value, spinMicroseconds, parkMs);
-        }
-        : pauseUntil;
-    const loop = () => {
-        isInMacro = false;
-        let progressed = true;
-        let awaiting = 0;
-        while (true) {
-            progressed = _enqueueLock();
-            if (_hasCompleted()) {
-                if (_writeBatch(WRITE_MAX) > 0)
-                    progressed = true;
-            }
-            _drainReturnReleases();
-            if (_hasPending()) {
-                if (_serviceBatchImmediate() > 0)
-                    progressed = true;
-            }
-            if ((awaiting = _getAwaiting()) > 0) {
-                if (awaiting !== lastAwaiting)
-                    awaitingSpins = 0;
-                lastAwaiting = awaiting;
-                awaitingSpins++;
-                const delay = Math.min(MAX_AWAITING_MS, Math.max(0, awaitingSpins - 1));
-                scheduleTimer(delay);
-                return;
-            }
-            awaitingSpins = lastAwaiting = 0;
-            if (!progressed) {
-                if (txStatus[Comment.thisIsAHint] === 1) {
-                    _pauseSpin();
-                    continue;
-                }
-                _pauseUntil(wakeToken, spinMicroseconds, parkMs);
-                wakeToken = a_load(opView, 0);
-            }
-        }
-    };
-    const port1Any = port1;
-    if (typeof port1Any.on === "function") {
-        port1Any.on("message", loop);
-    }
-    else {
-        // @ts-ignore
-        port1Any.onmessage = loop;
-    }
-    port1Any.start?.();
-    port2.start?.();
-    dbg?.log("lifecycle", `ready: ${listOfFunctions.length} task(s) on thread ${thread}/${totalNumberOfThread}, entering dispatch loop`);
-    scheduleMacro();
-};
-const isWorkerGlobalScope = () => {
-    const scopeCtor = globalThis.WorkerGlobalScope;
-    if (typeof scopeCtor !== "function")
-        return false;
-    try {
-        return globalThis instanceof
-            scopeCtor;
-    }
-    catch {
-        return false;
-    }
-};
-const isLockBuffers = (value) => {
-    if (!value || typeof value !== "object")
-        return false;
-    const candidate = value;
-    return isSharedBufferSource(candidate.headers) &&
-        isSharedBufferSource(candidate.lockSector) &&
-        isSharedBufferSource(candidate.payload) &&
-        isSharedBufferSource(candidate.payloadSector) &&
-        (candidate.textCompat === undefined ||
-            isLockBufferTextCompat(candidate.textCompat));
-};
-const isWorkerBootPayload = (value) => {
-    if (!value || typeof value !== "object")
-        return false;
-    const candidate = value;
-    return isSharedBufferSource(candidate.sab) &&
-        Array.isArray(candidate.list) &&
-        Array.isArray(candidate.ids) &&
-        Array.isArray(candidate.names) &&
-        Array.isArray(candidate.at) &&
-        typeof candidate.thread === "number" &&
-        typeof candidate.totalNumberOfThread === "number" &&
-        typeof candidate.startAt === "number" &&
-        (candidate.abortSignalSAB === undefined ||
-            isSharedBufferSource(candidate.abortSignalSAB)) &&
-        isLockBuffers(candidate.lock) &&
-        isLockBuffers(candidate.returnLock);
-};
-const installWorkerGlobalBootstrap = () => {
-    const g = globalThis;
-    const start = (data) => {
-        if (!isWorkerBootPayload(data))
-            return;
-        void workerMainLoop(data).catch(reportWorkerStartupFatal);
-    };
-    if (typeof g.addEventListener === "function" &&
-        typeof g.removeEventListener === "function") {
-        const onMessage = (event) => {
-            const data = event?.data;
-            if (!isWorkerBootPayload(data))
-                return;
-            try {
-                g.removeEventListener?.("message", onMessage);
-            }
-            catch {
-            }
-            start(data);
-        };
-        g.addEventListener("message", onMessage);
-        return;
-    }
-    g.onmessage = (event) => {
-        const data = event?.data;
-        if (!isWorkerBootPayload(data))
-            return;
-        g.onmessage = null;
-        start(data);
-    };
-};
-if (RUNTIME_IS_MAIN_THREAD === false &&
-    isWorkerBootPayload(RUNTIME_WORKER_DATA)) {
-    void workerMainLoop(RUNTIME_WORKER_DATA).catch(reportWorkerStartupFatal);
-}
-else if (RUNTIME_IS_PROCESS_WORKER) {
-    installProcessWorkerBootstrap({
-        isWorkerBootPayload,
-        reportWorkerStartupFatal,
-        startWorker: (data) => {
-            void workerMainLoop(data).catch(reportWorkerStartupFatal);
-        },
-    });
-}
-else if (isWorkerGlobalScope()) {
-    installWorkerGlobalBootstrap();
-}

package/src/worker/process-worker-bootstrap.d.ts

@@ -1,8 +0,0 @@
-import type { WorkerData } from "../types.js";
-import type { NativeWaitU32 } from "./timers.js";
-export declare const getProcessWorkerNativeWaitU32: () => NativeWaitU32 | undefined;
-export declare const installProcessWorkerBootstrap: ({ isWorkerBootPayload, reportWorkerStartupFatal, startWorker, }: {
-    isWorkerBootPayload: (value: unknown) => value is WorkerData;
-    reportWorkerStartupFatal: (error: unknown) => void;
-    startWorker: (data: WorkerData) => void;
-}) => void;

package/src/worker/process-worker-bootstrap.js

@@ -1,160 +0,0 @@
-import { RUNTIME } from "../common/runtime.js";
-import { RUNTIME_IS_PROCESS_WORKER, RUNTIME_PARENT_PORT, RUNTIME_PROCESS_WORKER_BOOT_ENV, RUNTIME_PROCESS_WORKER_BOOT_VERSION, } from "../common/worker-runtime.js";
-import { getNodeProcess } from "../common/node-compat.js";
-import { isLockBufferTextCompat } from "../common/shared-buffer-text.js";
-import { getDefaultProcessSharedBufferPrimitives, ProcessSharedBuffer, setDefaultProcessSharedBufferPrimitives, } from "../connections/process-shared-buffer.js";
-import { createBunConnectionPrimitives } from "../connections/bun.js";
-import { createDenoConnectionPrimitives } from "../connections/deno.js";
-import { loadNodeFutexAddon } from "../connections/node.js";
-export const getProcessWorkerNativeWaitU32 = () => {
-    if (!RUNTIME_IS_PROCESS_WORKER || RUNTIME !== "node")
-        return undefined;
-    try {
-        return loadNodeFutexAddon().waitU32;
-    }
-    catch {
-        return undefined;
-    }
-};
-const isProcessSharedBufferMetadata = (value) => {
-    try {
-        ProcessSharedBuffer.fromMetadata(value);
-        return true;
-    }
-    catch {
-        return false;
-    }
-};
-const isProcessWorkerWireLockBuffers = (value) => {
-    if (!value || typeof value !== "object")
-        return false;
-    const candidate = value;
-    return isProcessSharedBufferMetadata(candidate.headers) &&
-        isProcessSharedBufferMetadata(candidate.lockSector) &&
-        isProcessSharedBufferMetadata(candidate.payload) &&
-        isProcessSharedBufferMetadata(candidate.payloadSector) &&
-        (candidate.textCompat === undefined ||
-            isLockBufferTextCompat(candidate.textCompat));
-};
-const isProcessWorkerBootPayload = (value) => {
-    if (!value || typeof value !== "object")
-        return false;
-    const candidate = value;
-    const workerData = candidate.workerData;
-    return candidate.version === RUNTIME_PROCESS_WORKER_BOOT_VERSION &&
-        !!workerData &&
-        isProcessSharedBufferMetadata(workerData.sab) &&
-        Array.isArray(workerData.list) &&
-        Array.isArray(workerData.ids) &&
-        Array.isArray(workerData.names) &&
-        Array.isArray(workerData.at) &&
-        typeof workerData.thread === "number" &&
-        typeof workerData.totalNumberOfThread === "number" &&
-        typeof workerData.startAt === "number" &&
-        (workerData.abortSignalSAB === undefined ||
-            isProcessSharedBufferMetadata(workerData.abortSignalSAB)) &&
-        isProcessWorkerWireLockBuffers(workerData.lock) &&
-        isProcessWorkerWireLockBuffers(workerData.returnLock);
-};
-const getProcessWorkerPrimitives = () => {
-    const primitives = RUNTIME === "bun"
-        ? createBunConnectionPrimitives()
-        : RUNTIME === "deno"
-            ? createDenoConnectionPrimitives()
-            : getDefaultProcessSharedBufferPrimitives();
-    setDefaultProcessSharedBufferPrimitives(primitives);
-    return primitives;
-};
-const reviveProcessWorkerData = (wire) => {
-    const primitives = getProcessWorkerPrimitives();
-    const mappings = new Map();
-    const mappingKey = (descriptor) => descriptor.name === undefined
-        ? `fd:${descriptor.fd}:${descriptor.size}:${descriptor.runtime ?? ""}`
-        : `name:${descriptor.name}:${descriptor.size}:${descriptor.runtime ?? ""}`;
-    const reviveRegion = (metadata) => {
-        const processBuffer = ProcessSharedBuffer.fromMetadata(metadata);
-        const descriptor = processBuffer.descriptor;
-        const key = mappingKey(descriptor);
-        let mapping = mappings.get(key);
-        if (mapping === undefined) {
-            mapping = descriptor.map(primitives);
-            mappings.set(key, mapping);
-        }
-        else {
-            descriptor.attach(mapping);
-        }
-        return {
-            sab: mapping.buffer,
-            byteOffset: processBuffer.byteOffset,
-            byteLength: processBuffer.byteLength,
-        };
-    };
-    const reviveLockBuffers = (lock) => ({
-        ...lock,
-        headers: reviveRegion(lock.headers),
-        lockSector: reviveRegion(lock.lockSector),
-        payload: reviveRegion(lock.payload),
-        payloadSector: reviveRegion(lock.payloadSector),
-    });
-    return {
-        ...wire,
-        sab: reviveRegion(wire.sab),
-        abortSignalSAB: wire.abortSignalSAB === undefined
-            ? undefined
-            : reviveRegion(wire.abortSignalSAB),
-        lock: reviveLockBuffers(wire.lock),
-        returnLock: reviveLockBuffers(wire.returnLock),
-    };
-};
-export const installProcessWorkerBootstrap = ({ isWorkerBootPayload, reportWorkerStartupFatal, startWorker, }) => {
-    const processLike = getNodeProcess();
-    const start = (payload) => {
-        const data = reviveProcessWorkerData(payload.workerData);
-        if (!isWorkerBootPayload(data)) {
-            throw new TypeError("invalid process worker boot payload");
-        }
-        startWorker(data);
-    };
-    const envBoot = processLike?.env?.[RUNTIME_PROCESS_WORKER_BOOT_ENV];
-    if (typeof envBoot === "string" && envBoot.length > 0) {
-        try {
-            const payload = JSON.parse(envBoot);
-            if (!isProcessWorkerBootPayload(payload)) {
-                throw new TypeError("invalid process worker boot payload");
-            }
-            try {
-                delete processLike?.env?.[RUNTIME_PROCESS_WORKER_BOOT_ENV];
-            }
-            catch {
-            }
-            start(payload);
-        }
-        catch (error) {
-            reportWorkerStartupFatal(error);
-        }
-        return;
-    }
-    if (RUNTIME_PARENT_PORT === undefined) {
-        reportWorkerStartupFatal(new TypeError("missing process worker boot payload"));
-        return;
-    }
-    if (typeof processLike?.on !== "function")
-        return;
-    const onMessage = (message) => {
-        if (!isProcessWorkerBootPayload(message))
-            return;
-        try {
-            processLike.off?.("message", onMessage);
-            processLike.removeListener?.("message", onMessage);
-        }
-        catch {
-        }
-        try {
-            start(message);
-        }
-        catch (error) {
-            reportWorkerStartupFatal(error);
-        }
-    };
-    processLike.on("message", onMessage);
-};

package/src/worker/rx-queue.d.ts

@@ -1,25 +0,0 @@
-import { type Lock2 } from "../memory/lock.js";
-import type { WorkerComposedWithKey } from "./task-loader.js";
-import type { WorkerSettings } from "../types.js";
-type ArgumentsForCreateWorkerQueue = {
-    listOfFunctions: WorkerComposedWithKey[];
-    workerOptions?: WorkerSettings;
-    lock: Lock2;
-    returnLock: Lock2;
-    borrowReturnedBufferReferences?: boolean;
-    hasAborted?: (signal: number) => boolean;
-    now?: () => number;
-};
-export type CreateWorkerRxQueue = ReturnType<typeof createWorkerRxQueue>;
-export declare const createWorkerRxQueue: ({ listOfFunctions, workerOptions, lock, returnLock, borrowReturnedBufferReferences, hasAborted, now, }: ArgumentsForCreateWorkerQueue) => {
-    hasCompleted: () => boolean;
-    hasPending: () => boolean;
-    writeBatch: (max: number) => number;
-    serviceBatchImmediate: () => number;
-    enqueueLock: () => boolean;
-    drainReturnReleases: () => void;
-    releaseReturnedBufferReference: (token: bigint) => void;
-    hasAwaiting: () => boolean;
-    getAwaiting: () => number;
-};
-export {};