@earendil-works/chord 1.5.0 → 1.5.1

package/dist/next-node/workers/workers.js

@@ -1,572 +0,0 @@
-// The workers component (slice 10 §3, §4; decision "Slice 10's points"): a forked worker process per address of one
-// host kind, started on demand by the route it owns, stopped when idle, never restarted by itself (R2): a crash, a
-// `fatal`, a link lost while its child lives or a failed start leaves the address failed until `retry` or a rollout.
-// Per address, a child's events are recorded on that child as they happen; starts and stops run one after another.
-import { awaitWithContext, BACKGROUND_CONTEXT, withAbortSignal } from "../../context/index.js";
-import { ChordError } from "../../next/data/index.js";
-import { Batch, Source } from "../../next/state/internal.js";
-import { uncrossed, uncrossedReport } from "./control.js";
-import { WorkerProcess } from "./process.js";
-/** Creates the workers component on `host`: it routes `Kind` itself (`host.route`, before `load`). */
-export function createWorkers(host, Kind, options, context) {
-    // Its addresses are `Address<HostKind<N, Q>>`, as the route hands them over.
-    const component = new WorkersComponent(options, context);
-    const route = host.route(Kind, {
-        authorize: (principal, address) => {
-            // A throw would reach the holder as the hold's failure: the app's error stays private.
-            try {
-                return options.authorize(principal, address);
-            }
-            catch (error) {
-                options.onError({ kind: "authorize_failed", address: address.key, error });
-                return false;
-            }
-        },
-        // The principal is `options.principal`'s, a `P`.
-        resolve: (address, routing) => component.resolve(address, routing),
-    }, context);
-    component.route = route;
-    return component;
-}
-/** Starts a worker may take until `started` (fork, lock, connect): 60 s. */
-const START_TIMEOUT = 60_000;
-/** How long a stopping worker (or one whose link is gone) gets before its tree is killed: 5 s. */
-const GRACE = 5_000;
-/** How often idle workers are looked for: every second. */
-const IDLE_CHECK = 1_000;
-/** How long a worker idles before it is stopped unless `idleMs` says otherwise: 5 minutes. */
-const IDLE_MS = 5 * 60_000;
-/** The public error of a start that failed. */
-const START_FAILED = { message: "worker failed to start", code: "unavailable" };
-class WorkersComponent {
-    /** Set by `createWorkers` once the route exists. */
-    route;
-    state;
-    #state;
-    #options;
-    #addresses = new Map();
-    /** Aborted at `close`: starts waiting in the app's `start` or for `started` end. */
-    #closing = new AbortController();
-    /** `#closing`'s signal as a context: what a start's waits are bounded by. */
-    #closingContext = withAbortSignal(this.#closing.signal, BACKGROUND_CONTEXT);
-    /** Whether the route's context closes it already (its first `resolve` attached that). */
-    #routing = false;
-    #idleTimer;
-    constructor(options, context) {
-        this.#options = options;
-        this.#state = new WorkersState(this.#addresses, new Batch((error) => options.onError(error), undefined));
-        // Its value is a `Record<string, WorkerState>`.
-        this.state = this.#state;
-        this.#closeWith(context);
-    }
-    /**
-     * The route's `resolve`: a failed address rejects at once with its (public) failure, nothing forks; otherwise a
-     * start, after the previous child's exit.
-     */
-    resolve(address, routing) {
-        if (!this.#routing) {
-            this.#routing = true;
-            this.#closeWith(routing);
-        }
-        const routed = this.#routed(address);
-        if (routed.failure !== undefined)
-            return Promise.reject(uncrossed(routed.failure.error));
-        if (this.#closing.signal.aborted)
-            return Promise.reject(closed());
-        const started = this.#enqueue(routed, () => this.#start(routed, routing));
-        routed.starting = started;
-        void started.then(() => this.#settled(routed, started), () => this.#settled(routed, started));
-        return started;
-    }
-    async stop(address, context) {
-        this.route.suspend(address);
-        const routed = this.#addresses.get(address.key);
-        if (routed === undefined)
-            return;
-        await awaitWithContext(this.#enqueue(routed, () => this.#stop(routed)), context);
-    }
-    async retry(address, _context) {
-        const routed = this.#addresses.get(address.key);
-        if (routed !== undefined) {
-            routed.failure = undefined;
-            this.#state.refresh();
-            this.#forget(routed);
-        }
-        this.route.retry(address);
-    }
-    step(name) {
-        return { name, kinds: [this.#options.kind], reload: (_built, context) => this.#rollout(context) };
-    }
-    async close(context) {
-        if (!this.#closing.signal.aborted)
-            this.#closing.abort(closed());
-        clearInterval(this.#idleTimer);
-        this.#idleTimer = undefined;
-        const stops = [...this.#addresses.values()].map((routed) => this.#enqueue(routed, () => this.#stop(routed)));
-        await awaitWithContext(Promise.all(stops), context);
-    }
-    /** `context`'s abort closes the component (the route's is aborted when the host closes). */
-    #closeWith(context) {
-        const signal = context.abortSignal;
-        if (signal === undefined || signal.aborted)
-            return;
-        signal.addEventListener("abort", () => void this.close(BACKGROUND_CONTEXT).catch(() => { }), { once: true });
-    }
-    /** The record of `address`, made by its first use. */
-    #routed(address) {
-        let routed = this.#addresses.get(address.key);
-        if (routed === undefined) {
-            routed = {
-                address,
-                queue: Promise.resolve(),
-                pending: 0,
-                child: undefined,
-                failure: undefined,
-                starting: undefined,
-            };
-            this.#addresses.set(address.key, routed);
-        }
-        return routed;
-    }
-    /** Runs `operation` after the address's earlier ones. */
-    #enqueue(routed, operation) {
-        routed.pending++;
-        const run = routed.queue.then(operation);
-        routed.queue = run.then(() => this.#done(routed), () => this.#done(routed));
-        return run;
-    }
-    /** An operation ended. */
-    #done(routed) {
-        routed.pending--;
-        this.#forget(routed);
-    }
-    /** A start settled: no longer starting. */
-    #settled(routed, started) {
-        if (routed.starting === started)
-            routed.starting = undefined;
-    }
-    /** Forgets an address with no operation, no child and no failure. */
-    #forget(routed) {
-        if (routed.pending === 0 && routed.child === undefined && routed.failure === undefined) {
-            this.#addresses.delete(routed.address.key);
-        }
-    }
-    /**
-     * A start: the app's start data, the fork, the start message with the kind's current set, then `started` within the
-     * start timer (fork, lock and connect only). Its failure fails the address; `close` ends it at any of its waits (an
-     * app's `start` that ignores its signal too), a forked child is stopped.
-     */
-    async #start(routed, routing) {
-        const { address } = routed;
-        // After the previous child's exit (its link may close first): a crash recorded meanwhile rejects this start.
-        await routed.child?.process.exited;
-        const crashed = routed.failure;
-        if (crashed !== undefined)
-            throw uncrossed(crashed.error);
-        if (this.#closing.signal.aborted)
-            throw closed();
-        const context = withAbortSignal(this.#closing.signal, routing);
-        let data;
-        try {
-            data = await awaitWithContext(Promise.resolve(this.#options.start(address, context)), this.#closingContext);
-        }
-        catch (error) {
-            if (this.#closing.signal.aborted)
-                throw closed();
-            throw this.#fail(routed, { error, public: START_FAILED }, undefined);
-        }
-        if (this.#closing.signal.aborted)
-            throw closed();
-        let child;
-        try {
-            child = this.#fork(routed);
-        }
-        catch (error) {
-            throw this.#fail(routed, { error, public: START_FAILED }, undefined);
-        }
-        const timer = setTimeout(() => {
-            const error = new Error(`worker ${address.key}: did not start within ${START_TIMEOUT / 1000} s`);
-            child.cause ??= { error, killed: true };
-            child.process.kill();
-        }, START_TIMEOUT);
-        // Every app hook of the start runs in here: what one throws stays private (`worker_failed`), its holders and
-        // `state` get the fixed public message (Astra's recheck R3: `principal` threw through to a guest).
-        try {
-            const scripts = await this.#options.plugins.scripts(this.#options.kind, context);
-            child.shipped = keyOf(scripts);
-            const lock = this.#options.lock(address);
-            child.process.send({ kind: "start", address, start: data, scripts, lock });
-            await awaitWithContext(child.started.promise, this.#closingContext);
-            return { link: child.process.link, principal: this.#options.principal(address), trusted: true };
-        }
-        catch (error) {
-            // Closing is a stop (no failure); anything else fails the start.
-            if (this.#closing.signal.aborted)
-                child.stopping = true;
-            else
-                child.cause ??= { error, public: START_FAILED, killed: true };
-            child.process.kill();
-            await child.process.exited;
-            if (this.#closing.signal.aborted)
-                throw closed();
-            throw uncrossed(routed.failure?.error ?? START_FAILED);
-        }
-        finally {
-            clearTimeout(timer);
-        }
-    }
-    /** Forks the address's worker and records its events on it as they happen. */
-    #fork(routed) {
-        const { address } = routed;
-        const onOutput = this.#options.onOutput;
-        let child;
-        const process = new WorkerProcess({
-            entry: this.#options.entry,
-            ...(this.#options.execArgv === undefined ? {} : { execArgv: this.#options.execArgv }),
-            ...(this.#options.env === undefined ? {} : { env: this.#options.env }),
-            ...(onOutput === undefined ? {} : { onOutput: (stream, chunk) => onOutput(address, stream, chunk) }),
-        }, {
-            control: (message) => this.#control(routed, child, message),
-            linkLost: () => this.#linkLost(child),
-            linkClosed: () => this.#linkClosed(child),
-            exited: (exit) => this.#exited(routed, child, exit),
-        });
-        child = {
-            process,
-            status: "starting",
-            stopping: false,
-            cause: undefined,
-            started: Promise.withResolvers(),
-            ready: Promise.withResolvers(),
-            idleSince: undefined,
-            idleStop: false,
-            shipped: "",
-            reloading: Promise.resolve(),
-            reloaded: undefined,
-            ended: undefined,
-            grace: undefined,
-        };
-        // Nobody may wait for them.
-        child.started.promise.catch(() => { });
-        child.ready.promise.catch(() => { });
-        routed.child = child;
-        this.#state.refresh();
-        this.#watchIdle();
-        return child;
-    }
-    /** A control message of `child`. */
-    #control(routed, child, message) {
-        switch (message.kind) {
-            case "started":
-                child.started.resolve();
-                return;
-            case "ready":
-                child.status = "running";
-                child.idleSince = message.idle ? Date.now() : undefined;
-                child.ready.resolve(message.report);
-                if (routed.child === child)
-                    this.#state.refresh();
-                return;
-            case "idle":
-                child.idleSince = message.idle ? (child.idleSince ?? Date.now()) : undefined;
-                return;
-            case "fatal":
-                // Redacted by the child's `redact`: public as it is.
-                child.cause ??= { error: uncrossed(message.error), public: message.error };
-                child.started.reject(uncrossed(message.error));
-                child.process.kill();
-                return;
-            case "reloaded":
-                child.reloaded?.(message);
-        }
-    }
-    /** This side closed `child`'s link (the heartbeat, a protocol error): its child is stopped, its exit a crash. */
-    #linkLost(child) {
-        this.#ended(child);
-        if (child.stopping)
-            return;
-        child.cause ??= { error: new Error("worker: link lost") };
-        this.#shutdown(child);
-        this.#state.refresh();
-    }
-    /**
-     * The child's side closed its link (it is exiting, or it closed its IPC and lives on): its reloads end; one that does
-     * not exit within the grace is killed, its exit a crash (link lost).
-     */
-    #linkClosed(child) {
-        this.#ended(child);
-        if (child.stopping || child.grace !== undefined)
-            return;
-        child.grace = setTimeout(() => {
-            child.cause ??= { error: new Error("worker: link lost") };
-            child.process.kill();
-        }, GRACE);
-    }
-    /**
-     * `child` exited: a stop's exit forgets it; any other is a crash that fails the address (`worker_exited` and
-     * `worker_failed` once, the full error and stderr tail for the app's logs only), except once the component closes.
-     */
-    #exited(routed, child, exit) {
-        clearTimeout(child.grace);
-        const clean = child.stopping && child.cause === undefined;
-        const cause = child.cause ?? { error: exitError(routed.address, exit, child) };
-        const error = cause.public ?? publicExit(exit, child);
-        child.started.reject(uncrossed(error));
-        child.ready.reject(uncrossed(error));
-        this.#ended(child);
-        if (routed.child !== child)
-            return;
-        routed.child = undefined;
-        if (!clean && !this.#closing.signal.aborted) {
-            const { code, signal } = exit;
-            // A failed start's kill is the component's own act (Opus's second recheck): only `worker_failed`.
-            if (cause.killed !== true) {
-                this.#options.onError({ kind: "worker_exited", address: routed.address.key, code, signal });
-            }
-            this.#fail(routed, { ...cause, public: error }, child.process.stderrTail);
-        }
-        this.#state.refresh();
-        this.#forget(routed);
-    }
-    /**
-     * `child`'s link is gone or it exited (the first of these): its outstanding reload and every one queued behind it
-     * settle, `stopped` when a stop asked for it, else failed.
-     */
-    #ended(child) {
-        if (child.ended !== undefined)
-            return;
-        child.ended =
-            child.stopping && child.cause === undefined
-                ? { kind: "stopped" }
-                : { kind: "failed", error: new ChordError({ code: "unavailable", message: "worker: its link is gone" }) };
-        child.reloaded?.({ kind: "reloaded", error: { message: "worker: its link is gone" } });
-    }
-    /**
-     * Fails `routed` with `cause` (`worker_failed` once, with the full error and `stderrTail`); returns the public error
-     * a start rejects with.
-     */
-    #fail(routed, cause, stderrTail) {
-        routed.failure = { error: cause.public, at: Date.now() };
-        this.#options.onError({
-            kind: "worker_failed",
-            address: routed.address.key,
-            error: cause.error,
-            ...(stderrTail === undefined ? {} : { stderrTail }),
-        });
-        this.#state.refresh();
-        return uncrossed(cause.public);
-    }
-    /** A stop: `shutdown`, the grace, then the tree killed; resolves at its exit. */
-    async #stop(routed) {
-        const child = routed.child;
-        if (child === undefined)
-            return;
-        child.stopping = true;
-        this.#shutdown(child);
-        this.#state.refresh();
-        await child.process.exited;
-    }
-    /** Asks `child` to shut down and kills its tree after the grace. */
-    #shutdown(child) {
-        child.status = "stopping";
-        child.process.send({ kind: "shutdown" });
-        clearTimeout(child.grace);
-        child.grace = setTimeout(() => child.process.kill(), GRACE);
-    }
-    /** Starts the idle check (every second) while any worker exists. */
-    #watchIdle() {
-        if (this.#idleTimer !== undefined)
-            return;
-        this.#idleTimer = setInterval(() => this.#checkIdle(), IDLE_CHECK);
-        this.#idleTimer.unref();
-    }
-    /** Queues a stop for every worker idle since `idleMs()`; stops checking once no worker is left. */
-    #checkIdle() {
-        const idleMs = this.#idleMs();
-        const now = Date.now();
-        let workers = 0;
-        for (const routed of this.#addresses.values()) {
-            const child = routed.child;
-            if (child === undefined)
-                continue;
-            workers++;
-            if (child.idleStop || child.status !== "running")
-                continue;
-            if (child.idleSince === undefined || now - child.idleSince < idleMs)
-                continue;
-            child.idleStop = true;
-            void this.#enqueue(routed, () => this.#idleStop(routed, child));
-        }
-        if (workers === 0) {
-            clearInterval(this.#idleTimer);
-            this.#idleTimer = undefined;
-        }
-    }
-    /** An idle stop when it runs: only if `child` is still the address's and still idle (a hold may have come since). */
-    async #idleStop(routed, child) {
-        child.idleStop = false;
-        if (routed.child !== child || child.idleSince === undefined)
-            return;
-        if (Date.now() - child.idleSince < this.#idleMs())
-            return;
-        await this.#stop(routed);
-    }
-    /** How long a worker idles before it is stopped. */
-    #idleMs() {
-        return this.#options.idleMs?.() ?? IDLE_MS;
-    }
-    /**
-     * The rollout step (§4): the set as this rollout applied it, read now (before another rollout changes it); the
-     * failures recorded before it cleared and the held ones retried (listed once ready or failed); the set shipped to
-     * every worker whose last applied set differs, in parallel. `context`'s abort ends only this wait: the entries
-     * still waiting are listed with its error.
-     */
-    async #rollout(context) {
-        const { kind, plugins } = this.#options;
-        const scripts = plugins.current(kind) ?? (await plugins.scripts(kind, context));
-        const key = keyOf(scripts);
-        const entries = [];
-        for (const routed of [...this.#addresses.values()]) {
-            if (routed.failure === undefined)
-                continue;
-            routed.failure = undefined;
-            this.#state.refresh();
-            this.route.retry(routed.address);
-            const started = routed.starting;
-            if (started === undefined)
-                this.#forget(routed);
-            else
-                entries.push({ host: routed.address.key, entry: this.#retried(routed, started) });
-        }
-        for (const routed of this.#addresses.values()) {
-            const child = routed.child;
-            if (child === undefined || child.status === "stopping" || child.shipped === key)
-                continue;
-            entries.push({ host: routed.address.key, entry: this.#reload(routed.address.key, child, scripts, key) });
-        }
-        return { hosts: await Promise.all(entries.map(({ host, entry }) => bounded(host, entry, context))) };
-    }
-    /** A retried address's entry: its start's load report once ready, or its error. */
-    async #retried(routed, started) {
-        const host = routed.address.key;
-        try {
-            await started;
-            const child = routed.child;
-            if (child === undefined)
-                throw new Error("worker: exited");
-            return { host, report: uncrossedReport(await child.ready.promise) };
-        }
-        catch (error) {
-            return { host, error };
-        }
-    }
-    /**
-     * Ships `scripts` to `child` after its outstanding reload; settles on `reloaded`, its link's loss or its exit. Only a
-     * reload that applied without failures marks the set shipped (a failed plugin is shipped again next time).
-     */
-    #reload(host, child, scripts, key) {
-        const reload = child.reloading.then(() => new Promise((resolve) => {
-            const settle = (ended) => {
-                resolve(ended.kind === "stopped" ? { host, stopped: true } : { host, error: ended.error });
-            };
-            if (child.ended !== undefined) {
-                settle(child.ended);
-                return;
-            }
-            child.reloaded = (reply) => {
-                child.reloaded = undefined;
-                if ("report" in reply) {
-                    if (reply.report.failed.length === 0)
-                        child.shipped = key;
-                    resolve({ host, report: uncrossedReport(reply.report) });
-                }
-                else if (child.ended !== undefined) {
-                    settle(child.ended);
-                }
-                else {
-                    resolve({ host, error: uncrossed(reply.error) });
-                }
-            };
-            child.process.send({ kind: "reload", scripts });
-        }));
-        child.reloading = reload.then(() => { }, () => { });
-        return reload;
-    }
-}
-/** What new starts reject with once the component closes. */
-function closed() {
-    return new ChordError({ code: "released", reason: "closed", message: "workers: closed" });
-}
-/** The full error of an exit nobody asked for (the app's logs). */
-function exitError(address, { code, signal }, child) {
-    const how = signal === null ? `code ${code}` : `signal ${signal}`;
-    const when = child.status === "starting" ? "exited before it was ready" : "exited";
-    return new ChordError({ code: "unavailable", message: `worker ${address.key}: ${when} (${how})` });
-}
-/** The public error of an exit: a start that failed, or the exit's code or signal. */
-function publicExit({ code, signal }, child) {
-    if (child.status === "starting")
-        return START_FAILED;
-    return { message: `worker exited (${signal ?? `code ${code}`})`, code: "unavailable" };
-}
-/** A set's identity: its plugins and digests. */
-function keyOf(scripts) {
-    return scripts
-        .map(({ plugin, built }) => `${plugin.name}@${built.kind === "script" ? built.digest : "failed"}`)
-        .sort()
-        .join("\n");
-}
-/** `entry`, or `host` listed with the abort's error once `context` aborts first (the entry goes on). */
-async function bounded(host, entry, context) {
-    try {
-        return await awaitWithContext(entry, context);
-    }
-    catch (error) {
-        return { host, error };
-    }
-}
-/** The workers' state: a JSON watchable of every address's `WorkerState`, published once per microtask turn. */
-class WorkersState extends Source {
-    #addresses;
-    #value;
-    #scheduled = false;
-    constructor(addresses, batch) {
-        super(batch);
-        this.#addresses = addresses;
-    }
-    get value() {
-        this.#value ??= stateOf(this.#addresses);
-        // Plain JSON: statuses, numbers and crossed errors.
-        return this.#value;
-    }
-    deliver(context) {
-        const value = this.value;
-        this.publish(value, [["r", value]], false, context);
-    }
-    /** Something changed: published with the turn's other changes while someone watches or derives from it. */
-    refresh() {
-        this.#value = undefined;
-        if (this.#scheduled || (!this.watched && this.dependents.size === 0))
-            return;
-        this.#scheduled = true;
-        queueMicrotask(() => {
-            this.#scheduled = false;
-            this.batch.within(() => this.changed(), BACKGROUND_CONTEXT);
-        });
-    }
-}
-/** Every address's state: its child's, or its failure; none for an address with neither. */
-function stateOf(addresses) {
-    const state = {};
-    for (const [key, { child, failure }] of addresses) {
-        if (failure !== undefined) {
-            state[key] = { status: "failed", ...failure };
-        }
-        else if (child !== undefined) {
-            const { pid, startedAt } = child.process;
-            state[key] = { status: child.status, pid, startedAt };
-        }
-    }
-    return state;
-}
-//# sourceMappingURL=workers.js.map