knitting 0.1.51 → 0.1.53

package/src/connections/shared-array-buffer-payload.js

@@ -0,0 +1,235 @@
+import { RUNTIME } from "../common/runtime.js";
+import { getNodeProcess } from "../common/node-compat.js";
+import { getBufferReferenceCapabilities } from "./buffer-reference-native.js";
+// Thread-worker SharedArrayBuffer transport by process-local pointer.
+// SABs are shared, not moved or detached; process workers reject them.
+export const SHARED_ARRAY_BUFFER_CODEC_ID = "knitting.sharedArrayBuffer";
+export const SHARED_ARRAY_BUFFER_NUMERIC_TRANSFER = Symbol.for("knitting.sharedArrayBuffer.numericTransfer");
+export const SHARED_ARRAY_BUFFER_NUMERIC_WORDS = 8;
+const SHARED_ARRAY_BUFFER_TOKEN_NUMERIC_WORDS = 2;
+const EXTERNAL_PAYLOAD_BRAND = Symbol.for("knitting.payloadCodec");
+const getProcessId = () => {
+    const proc = getNodeProcess();
+    if (proc !== undefined && typeof proc.pid === "number")
+        return proc.pid;
+    const deno = globalThis.Deno;
+    if (typeof deno?.pid === "number")
+        return deno.pid;
+    return 0;
+};
+const PROCESS_ORIGIN = `${RUNTIME}:${getProcessId()}`;
+const hasSharedArrayBuffer = typeof SharedArrayBuffer === "function";
+export const isSharedArrayBufferValue = (value) => hasSharedArrayBuffer && value instanceof SharedArrayBuffer;
+const pinnedBySab = new WeakMap();
+const payloadBySharedBuffer = new WeakMap();
+const warmedTokensByTransport = new WeakMap();
+const cachedSharedBuffersByToken = new Map();
+const pinFinalizer = typeof FinalizationRegistry === "function"
+    ? new FinalizationRegistry((token) => {
+        try {
+            getBufferReferenceCapabilities().releaseShared(token);
+        }
+        catch {
+            // best effort
+        }
+    })
+    : undefined;
+const splitU64 = (value) => [
+    Number(value & 0xffffffffn) >>> 0,
+    Number((value >> 32n) & 0xffffffffn) >>> 0,
+];
+const joinU64 = (low, high) => (BigInt(high >>> 0) << 32n) | BigInt(low >>> 0);
+const encodeRuntime = (runtime) => {
+    switch (runtime) {
+        case "node":
+            return 1;
+        case "deno":
+            return 2;
+        case "bun":
+            return 3;
+        default:
+            return 0;
+    }
+};
+const decodeRuntime = (value) => {
+    switch (value) {
+        case 1:
+            return "node";
+        case 2:
+            return "deno";
+        case 3:
+            return "bun";
+        default:
+            return undefined;
+    }
+};
+const getWarmTokens = (transportKey) => {
+    if (transportKey === undefined)
+        return undefined;
+    let warmTokens = warmedTokensByTransport.get(transportKey);
+    if (warmTokens === undefined) {
+        warmTokens = new Set();
+        warmedTokensByTransport.set(transportKey, warmTokens);
+    }
+    return warmTokens;
+};
+const pinSab = (sab) => {
+    let pin = pinnedBySab.get(sab);
+    if (pin === undefined) {
+        const produced = getBufferReferenceCapabilities().produceShared(sab);
+        pin = {
+            token: produced.token,
+            pointer: produced.pointer,
+            byteLength: produced.byteLength,
+        };
+        pinnedBySab.set(sab, pin);
+        pinFinalizer?.register(sab, pin.token);
+    }
+    return pin;
+};
+const makeMetadata = (pin) => ({
+    kind: SHARED_ARRAY_BUFFER_CODEC_ID,
+    origin: PROCESS_ORIGIN,
+    runtime: RUNTIME,
+    pointer: pin.pointer.toString(),
+    token: pin.token.toString(),
+    byteLength: pin.byteLength,
+});
+const makeFullNumericMetadata = (pin) => {
+    if (pin.byteLength > 0xffffffff)
+        return undefined;
+    const [tokenLow, tokenHigh] = splitU64(pin.token);
+    const [pointerLow, pointerHigh] = splitU64(pin.pointer);
+    return [
+        tokenLow,
+        tokenHigh,
+        pointerLow,
+        pointerHigh,
+        pin.byteLength >>> 0,
+        encodeRuntime(RUNTIME),
+        getProcessId() >>> 0,
+        0,
+    ];
+};
+const makeTokenNumericMetadata = (pin) => {
+    const [tokenLow, tokenHigh] = splitU64(pin.token);
+    return [tokenLow, tokenHigh];
+};
+/** Wrap a SAB as external payload; GC-managed pins mean no settle finalizer. */
+export const wrapSharedArrayBufferPayload = (sab) => {
+    let payload = payloadBySharedBuffer.get(sab);
+    if (payload !== undefined)
+        return payload;
+    const pin = pinSab(sab);
+    payload = createSharedArrayBufferPayload(sab, pin, makeMetadata(pin));
+    return payload;
+};
+const createSharedArrayBufferPayload = (buffer, pin, metadata) => {
+    let payload = payloadBySharedBuffer.get(buffer);
+    if (payload !== undefined)
+        return payload;
+    const fullNumeric = makeFullNumericMetadata(pin);
+    const tokenOnlyNumeric = makeTokenNumericMetadata(pin);
+    payload = {
+        [EXTERNAL_PAYLOAD_BRAND]: SHARED_ARRAY_BUFFER_CODEC_ID,
+        toMetadata: () => metadata,
+        [SHARED_ARRAY_BUFFER_NUMERIC_TRANSFER]: (transportKey) => {
+            if (fullNumeric === undefined) {
+                return undefined;
+            }
+            const warmTokens = getWarmTokens(transportKey);
+            if (warmTokens === undefined)
+                return fullNumeric;
+            if (warmTokens.has(pin.token))
+                return tokenOnlyNumeric;
+            warmTokens.add(pin.token);
+            return fullNumeric;
+        },
+    };
+    payloadBySharedBuffer.set(buffer, payload);
+    return payload;
+};
+export const getSharedArrayBufferPayload = (value) => {
+    if (isSharedArrayBufferValue(value))
+        return wrapSharedArrayBufferPayload(value);
+    return payloadBySharedBuffer.get(value);
+};
+const isSharedArrayBufferMetadata = (value) => {
+    if (value === null || typeof value !== "object")
+        return false;
+    const meta = value;
+    return (meta.kind === SHARED_ARRAY_BUFFER_CODEC_ID &&
+        typeof meta.origin === "string" &&
+        typeof meta.runtime === "string" &&
+        typeof meta.pointer === "string" &&
+        typeof meta.token === "string" &&
+        typeof meta.byteLength === "number" &&
+        Number.isInteger(meta.byteLength) &&
+        meta.byteLength >= 0);
+};
+const materializeSharedBuffer = (metadata, warmOnly) => {
+    if (metadata.origin !== PROCESS_ORIGIN) {
+        throw new Error(`SharedArrayBuffer cannot cross a process boundary (origin ${metadata.origin} ` +
+            `!= ${PROCESS_ORIGIN}); it is shared by reference to thread workers only.`);
+    }
+    const token = BigInt(metadata.token);
+    const cached = cachedSharedBuffersByToken.get(token);
+    if (cached !== undefined)
+        return cached;
+    if (warmOnly) {
+        throw new TypeError("SharedArrayBuffer cache miss for warm token payload");
+    }
+    const region = getBufferReferenceCapabilities().adoptShared({
+        token,
+        pointer: BigInt(metadata.pointer),
+        byteOffset: 0,
+        byteLength: metadata.byteLength,
+    });
+    cachedSharedBuffersByToken.set(token, region.buffer);
+    createSharedArrayBufferPayload(region.buffer, {
+        token,
+        pointer: BigInt(metadata.pointer),
+        byteLength: metadata.byteLength,
+    }, metadata);
+    return region.buffer;
+};
+const decode = (metadata) => {
+    if (!isSharedArrayBufferMetadata(metadata)) {
+        throw new TypeError("Invalid SharedArrayBuffer payload metadata");
+    }
+    return materializeSharedBuffer(metadata, false);
+};
+const decodeNumeric = (words) => {
+    if (words.length === SHARED_ARRAY_BUFFER_TOKEN_NUMERIC_WORDS) {
+        const token = joinU64(words[0] ?? 0, words[1] ?? 0);
+        const cached = cachedSharedBuffersByToken.get(token);
+        if (cached !== undefined)
+            return cached;
+        throw new TypeError("SharedArrayBuffer cache miss for warm token payload");
+    }
+    const runtime = decodeRuntime(words[5] ?? 0);
+    if (runtime === undefined) {
+        throw new TypeError("Invalid SharedArrayBuffer numeric runtime");
+    }
+    const originPid = words[6];
+    if (originPid === undefined ||
+        !Number.isInteger(originPid) ||
+        originPid < 0) {
+        throw new TypeError("Invalid SharedArrayBuffer numeric origin");
+    }
+    if (words.length !== SHARED_ARRAY_BUFFER_NUMERIC_WORDS) {
+        throw new TypeError("Invalid SharedArrayBuffer numeric word count");
+    }
+    const metadata = {
+        kind: SHARED_ARRAY_BUFFER_CODEC_ID,
+        origin: `${runtime}:${originPid >>> 0}`,
+        runtime,
+        pointer: joinU64(words[2] ?? 0, words[3] ?? 0).toString(),
+        token: joinU64(words[0] ?? 0, words[1] ?? 0).toString(),
+        byteLength: words[4] ?? 0,
+    };
+    return materializeSharedBuffer(metadata, false);
+};
+const codecGlobal = globalThis;
+const codecs = codecGlobal.__KNITTING_PAYLOAD_CODECS__ ??= Object.create(null);
+codecs[SHARED_ARRAY_BUFFER_CODEC_ID] = { decode, decodeNumeric };

package/src/debug/env-diff.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Environment-diff primitive: snapshot the keys present on `globalThis`, then
+ * later compare to see what the loaded code added, removed, or redefined.
+ *
+ * This is the core of "check what happens" debugging — a concrete before/after
+ * of the real runtime a worker's modules created, not an aggregate metric. The
+ * same snapshot/diff shape generalises to other ambient state (process
+ * listeners, open handles, prototype patches); `globalThis` keys are the first
+ * instance.
+ */
+export type EnvSnapshot = {
+    readonly keys: ReadonlySet<string | symbol>;
+};
+/** Capture every own key on `globalThis`, including symbols. */
+export declare const snapshotGlobals: () => EnvSnapshot;
+export type GlobalsDiff = {
+    readonly added: (string | symbol)[];
+    readonly removed: (string | symbol)[];
+};
+export declare const diffGlobals: (before: EnvSnapshot, after: EnvSnapshot) => GlobalsDiff;
+/**
+ * Render a key with enough provenance to tell a fresh global from a
+ * monkeypatch: its `typeof`/accessor kind plus writable/configurable/enumerable
+ * flags (`w`/`c`/`e`).
+ */
+export declare const describeGlobalKey: (key: string | symbol) => string;

package/src/debug/env-diff.js

@@ -0,0 +1,49 @@
+/**
+ * Environment-diff primitive: snapshot the keys present on `globalThis`, then
+ * later compare to see what the loaded code added, removed, or redefined.
+ *
+ * This is the core of "check what happens" debugging — a concrete before/after
+ * of the real runtime a worker's modules created, not an aggregate metric. The
+ * same snapshot/diff shape generalises to other ambient state (process
+ * listeners, open handles, prototype patches); `globalThis` keys are the first
+ * instance.
+ */
+/** Capture every own key on `globalThis`, including symbols. */
+export const snapshotGlobals = () => ({
+    keys: new Set(Reflect.ownKeys(globalThis)),
+});
+export const diffGlobals = (before, after) => {
+    const added = [];
+    const removed = [];
+    for (const key of after.keys) {
+        if (!before.keys.has(key))
+            added.push(key);
+    }
+    for (const key of before.keys) {
+        if (!after.keys.has(key))
+            removed.push(key);
+    }
+    return { added, removed };
+};
+/**
+ * Render a key with enough provenance to tell a fresh global from a
+ * monkeypatch: its `typeof`/accessor kind plus writable/configurable/enumerable
+ * flags (`w`/`c`/`e`).
+ */
+export const describeGlobalKey = (key) => {
+    const name = typeof key === "symbol" ? key.toString() : key;
+    let descriptor;
+    try {
+        descriptor = Object.getOwnPropertyDescriptor(globalThis, key);
+    }
+    catch {
+        return name;
+    }
+    if (descriptor === undefined)
+        return name;
+    const kind = descriptor.get !== undefined || descriptor.set !== undefined
+        ? "accessor"
+        : typeof descriptor.value;
+    const flags = `${descriptor.writable === false ? "" : "w"}${descriptor.configurable ? "c" : ""}${descriptor.enumerable ? "e" : ""}`;
+    return flags.length > 0 ? `${name} (${kind} ${flags})` : `${name} (${kind})`;
+};

package/src/debug/gate.d.ts

@@ -0,0 +1,18 @@
+import type { DebugOptions } from "../types.js";
+/** Namespaces explicitly requested via `KNITTING_DEBUG`. */
+export declare const DEBUG_NAMESPACES: ReadonlySet<string>;
+/**
+ * True when at least one namespace is requested via the env var alone. Guards
+ * the env-only paths; callers that also accept a `debug` option should use
+ * {@link resolveDebugNamespaces} instead.
+ */
+export declare const DEBUG_ENABLED: boolean;
+/**
+ * Merge the namespaces requested through the `debug` pool option with those from
+ * `KNITTING_DEBUG`; either source can enable a namespace. Returns an empty set
+ * when nothing is requested, so callers use `.size === 0` to keep the rest of
+ * `src/debug` unloaded — zero cost when off.
+ */
+export declare const resolveDebugNamespaces: (debug?: DebugOptions) => Set<string>;
+/** True when `namespace` (or `"*"`) is active for the given `debug` config + env. */
+export declare const debugHas: (debug: DebugOptions | undefined, namespace: string) => boolean;

package/src/debug/gate.js

@@ -0,0 +1,69 @@
+/**
+ * Zero-cost debug gate.
+ *
+ * Read once at module load from `KNITTING_DEBUG`. This module is deliberately
+ * tiny and dependency-light: importing it must never pull in the logger or the
+ * environment-diff machinery. Callers branch on {@link DEBUG_ENABLED} and only
+ * then `await import("./handle.ts")`, so when debug is off nothing else under
+ * `src/debug` is ever loaded — literally zero cost, not merely cheap.
+ *
+ * `KNITTING_DEBUG` is a comma-separated list of namespaces:
+ *   KNITTING_DEBUG=host,imports      # only those
+ *   KNITTING_DEBUG=*                 # everything
+ */
+import { getNodeProcess } from "../common/node-compat.js";
+const readEnv = (key) => {
+    // Deno: `process.env` exists under node-compat, but reading it may require
+    // --allow-env. Prefer the typed `Deno.env` and swallow permission errors.
+    const denoEnv = globalThis.Deno?.env;
+    if (typeof denoEnv?.get === "function") {
+        try {
+            return denoEnv.get(key);
+        }
+        catch {
+            /* env permission denied — fall through to node/bun */
+        }
+    }
+    try {
+        return getNodeProcess()?.env?.[key];
+    }
+    catch {
+        return undefined;
+    }
+};
+const raw = readEnv("KNITTING_DEBUG");
+/** Namespaces explicitly requested via `KNITTING_DEBUG`. */
+export const DEBUG_NAMESPACES = new Set((raw ?? "")
+    .split(",")
+    .map((part) => part.trim())
+    .filter((part) => part.length > 0));
+/**
+ * True when at least one namespace is requested via the env var alone. Guards
+ * the env-only paths; callers that also accept a `debug` option should use
+ * {@link resolveDebugNamespaces} instead.
+ */
+export const DEBUG_ENABLED = DEBUG_NAMESPACES.size > 0;
+/**
+ * Merge the namespaces requested through the `debug` pool option with those from
+ * `KNITTING_DEBUG`; either source can enable a namespace. Returns an empty set
+ * when nothing is requested, so callers use `.size === 0` to keep the rest of
+ * `src/debug` unloaded — zero cost when off.
+ */
+export const resolveDebugNamespaces = (debug) => {
+    const namespaces = new Set(DEBUG_NAMESPACES);
+    if (debug === true) {
+        namespaces.add("*");
+    }
+    else if (debug !== undefined && debug !== false) {
+        for (const [key, value] of Object.entries(debug)) {
+            if (value === true)
+                namespaces.add(key);
+        }
+    }
+    return namespaces;
+};
+/** True when `namespace` (or `"*"`) is active for the given `debug` config + env. */
+export const debugHas = (debug, namespace) => {
+    const namespaces = resolveDebugNamespaces(debug);
+    return namespaces.has("*") || namespaces.has(namespace);
+};

package/src/debug/handle.d.ts

@@ -0,0 +1,23 @@
+export type DebugInit = {
+    /** Identity prefix for log tags, e.g. `"w0"` for a worker or `"main"`. */
+    readonly name: string;
+    readonly runtime: string;
+    readonly namespaces: ReadonlySet<string>;
+};
+export type Debug = {
+    /**
+     * Is a namespace active? Capture this once before a hot loop and branch on
+     * the boolean — never call per-iteration.
+     */
+    enabled: (namespace: string) => boolean;
+    /** Emit a tagged line to stderr when `namespace` is active. */
+    log: (namespace: string, message: string) => void;
+    /**
+     * Re-snapshot `globalThis` and report what changed since the previous phase.
+     * Drives two-phase pollution attribution (e.g. `"bootstrap"` then
+     * `"tasks"`), so you can see which loader injected which global. No-op unless
+     * the `globals` namespace is active.
+     */
+    envPhase: (label: string) => void;
+};
+export declare const initDebug: ({ name, runtime, namespaces }: DebugInit) => Debug;

package/src/debug/handle.js

@@ -0,0 +1,48 @@
+/**
+ * Worker-side debug handle. Lazily imported (see {@link ./gate.ts}) only when
+ * `KNITTING_DEBUG` names at least one namespace, so neither this code nor the
+ * baseline snapshot it takes exists when debug is off.
+ *
+ * Diagnostics go to stderr so they never corrupt a worker's stdout, and every
+ * line is tagged with the worker id, runtime, and a clock relative to when this
+ * worker's debug initialised. The clock is worker-local on purpose: a main-thread
+ * timestamp can't be compared against `performance.now()` here because the time
+ * origins differ across the thread/process boundary (it would read negative).
+ */
+import { describeGlobalKey, diffGlobals, snapshotGlobals, } from "./env-diff.js";
+export const initDebug = ({ name, runtime, namespaces }) => {
+    const all = namespaces.has("*");
+    const enabled = (namespace) => all || namespaces.has(namespace);
+    const base = performance.now();
+    const tag = `${name}·${runtime}`;
+    const log = (namespace, message) => {
+        if (!enabled(namespace))
+            return;
+        const elapsed = (performance.now() - base).toFixed(1);
+        console.error(`[${tag}·+${elapsed}ms] ${namespace}: ${message}`);
+    };
+    // Baseline for the environment diff, taken the moment debug initialises:
+    // before worker bootstrap and before any task module is imported. Only paid
+    // when `globals` tracing is actually on.
+    let previous = enabled("globals")
+        ? snapshotGlobals()
+        : undefined;
+    const envPhase = (label) => {
+        if (previous === undefined)
+            return;
+        const current = snapshotGlobals();
+        const { added, removed } = diffGlobals(previous, current);
+        previous = current;
+        if (added.length === 0 && removed.length === 0) {
+            log("globals", `${label}: no new globals`);
+            return;
+        }
+        if (added.length > 0) {
+            log("globals", `${label} +${added.length}: ${added.map(describeGlobalKey).join("  ")}`);
+        }
+        if (removed.length > 0) {
+            log("globals", `${label} -${removed.length}: ${removed.map(String).join("  ")}`);
+        }
+    };
+    return { enabled, log, envPhase };
+};

package/src/ipc/transport/shared-memory.d.ts

@@ -1,6 +1,5 @@
 export type SignalArguments = ReturnType<typeof createSharedMemoryTransport>;
 import { type SharedBufferSource } from "../../common/shared-buffer-region.js";
-import { type DebugOptions } from "../../types.js";
 export declare const TRANSPORT_SIGNAL_BYTES: number;
 export type Sab = {
     size?: number;
@@ -10,11 +9,10 @@ type SignalForWorker = {
     sabObject?: Sab;
     isMain: boolean;
     thread: number;
-    debug?: DebugOptions;
     startTime?: number;
 };
 export declare const createSharedMemoryTransport: ({ sabObject, isMain, startTime }: SignalForWorker) => {
-    sab: import("../../types.js").SharedBufferRegion;
+    sab: import("../../common/shared-buffer-region.js").SharedBufferRegion;
     op: Int32Array<import("../../common/shared-buffer-region.js").SharedBuffer>;
     startAt: number;
     opView: Int32Array<import("../../common/shared-buffer-region.js").SharedBuffer>;