agent-hitch 0.2.5 → 0.2.7

package/dist/src/control-plane/remote-worker-runner.js

@@ -0,0 +1,214 @@
+import { HitchError, safeDiagnosticMessage } from "../foundation/index.js";
+import { ResourceLedger } from "./resources.js";
+const REQUIRED_INPUTS = new Set(["work-spec", "harness-artifact", "controller-runtime", "task-input"]);
+export class RemoteWorkerRunner {
+    client;
+    execute;
+    signal;
+    once;
+    pollIntervalMs;
+    heartbeatIntervalMs;
+    retryIntervalMs;
+    onError;
+    releaseUnknown;
+    ledger;
+    jobs = new Map();
+    handled = 0;
+    lastHeartbeat = 0;
+    constructor(options) {
+        this.client = options.client;
+        this.execute = options.execute;
+        this.signal = options.signal;
+        this.once = options.once ?? false;
+        this.pollIntervalMs = interval(options.pollIntervalMs ?? 1_000, "poll");
+        this.heartbeatIntervalMs = interval(options.heartbeatIntervalMs ?? 10_000, "heartbeat");
+        this.retryIntervalMs = interval(options.retryIntervalMs ?? 1_000, "retry");
+        this.onError = options.onError ?? (() => undefined);
+        this.releaseUnknown = options.releaseUnknown ?? (() => Promise.resolve());
+        this.ledger = new ResourceLedger(options.capacity);
+    }
+    async run() {
+        while (!this.signal?.aborted) {
+            try {
+                await this.tick();
+            }
+            catch (error) {
+                this.onError(error);
+            }
+            if (this.once && this.handled > 0 && this.jobs.size === 0)
+                return;
+            await delay(this.pollIntervalMs, this.signal);
+        }
+        for (const job of this.jobs.values())
+            job.controller.abort(this.signal?.reason);
+        await Promise.allSettled([...this.jobs.values()].map((job) => job.promise));
+    }
+    async tick() {
+        const offers = await this.client.listOffers();
+        const visible = new Map(offers.map((offer) => [offer.offer_id, offer]));
+        const released = new Set();
+        for (const job of this.jobs.values()) {
+            const current = visible.get(job.offer.offer_id);
+            if (!current)
+                continue;
+            job.offer = current;
+            if (current.state === "cancel-requested")
+                job.controller.abort(new Error("remote work cancellation requested"));
+            if (current.state === "release-requested") {
+                await this.releaseJob(job);
+                released.add(current.offer_id);
+            }
+        }
+        for (const offer of offers) {
+            if (this.jobs.has(offer.offer_id) || released.has(offer.offer_id))
+                continue;
+            if (offer.state === "release-requested" || offer.state === "cancel-requested") {
+                await this.releaseUnknown(offer);
+                await this.client.release(offer);
+                this.handled += 1;
+            }
+            else if (offer.state === "offered" && (!this.once || this.handled === 0)) {
+                this.startOffer(offer);
+            }
+        }
+        if (Date.now() - this.lastHeartbeat >= this.heartbeatIntervalMs)
+            await this.heartbeat();
+    }
+    async heartbeat(health = "healthy") {
+        const active = [...this.jobs.values()].filter((job) => job.accepted).map((job) => ({
+            lease_id: job.offer.lease.lease_id, epoch: job.offer.lease.epoch,
+        }));
+        await this.client.heartbeat(this.ledger.snapshot().allocated, active, health);
+        this.lastHeartbeat = Date.now();
+    }
+    startOffer(offer) {
+        const allocation = this.ledger.tryAcquire(offer.lease.lease_id, "eval", offer.lease.reservation);
+        if (!allocation)
+            return;
+        const controller = new AbortController();
+        const job = {
+            offer, controller, allocation, accepted: false, settled: false, cleanup: undefined,
+            promise: Promise.resolve(),
+        };
+        this.jobs.set(offer.offer_id, job);
+        job.promise = this.runOffer(job).catch((error) => {
+            this.onError(error);
+            if (!job.accepted)
+                this.finishJob(job);
+        });
+    }
+    async runOffer(job) {
+        let inputs;
+        try {
+            inputs = await this.downloadInputs(job.offer);
+        }
+        catch (error) {
+            await this.client.reject(job.offer, rejectionCode(error));
+            this.handled += 1;
+            this.finishJob(job);
+            return;
+        }
+        const accepted = await retry(() => this.client.accept(job.offer), this.retryIntervalMs, this.signal);
+        if (accepted.state !== "accepted") {
+            this.handled += 1;
+            this.finishJob(job);
+            return;
+        }
+        job.offer = accepted;
+        job.accepted = true;
+        await this.heartbeat();
+        const credentials = new Map(Object.entries((await retry(() => this.client.credentials(job.offer), this.retryIntervalMs, job.controller.signal)).credentials));
+        let sequence = 0;
+        const emit = (type, payload) => retry(() => this.client.emit(job.offer, ++sequence, type, payload), this.retryIntervalMs, job.controller.signal);
+        let result;
+        try {
+            result = await this.execute({ offer: job.offer, inputs, credentials, signal: job.controller.signal, emit });
+            validateResult(result);
+            job.cleanup = result.release;
+        }
+        catch (error) {
+            // Executor setup can fail before it has a chance to return its normal
+            // release callback. Keep the offer releasable through the same
+            // ownership-fenced recovery path used after a worker restart.
+            job.cleanup = () => this.releaseUnknown(job.offer);
+            result = {
+                status: job.controller.signal.aborted ? "cancelled" : "failed",
+                artifacts: [{ kind: "diagnostic", body: diagnostic(error, [...credentials.values()]) }],
+            };
+        }
+        finally {
+            credentials.clear();
+        }
+        const artifacts = [];
+        for (const artifact of result.artifacts ?? []) {
+            artifacts.push(await retry(() => this.client.uploadArtifact(job.offer, artifact.kind, artifact.body), this.retryIntervalMs, this.signal));
+        }
+        job.offer = await retry(() => this.client.complete(job.offer, result.status, artifacts), this.retryIntervalMs, this.signal);
+        job.settled = true;
+        await this.heartbeat();
+    }
+    async downloadInputs(offer) {
+        const refs = offer.inputs ?? [];
+        if (refs.length !== REQUIRED_INPUTS.size || refs.some((ref) => !REQUIRED_INPUTS.has(ref.kind))
+            || new Set(refs.map((ref) => ref.kind)).size !== REQUIRED_INPUTS.size)
+            throw runnerError("remote work offer is missing required inputs");
+        const inputs = new Map();
+        for (const ref of refs)
+            inputs.set(ref.kind, await this.client.downloadInput(offer, ref));
+        return inputs;
+    }
+    async releaseJob(job) {
+        await job.cleanup?.();
+        job.offer = await this.client.release(job.offer);
+        this.handled += 1;
+        this.finishJob(job);
+        await this.heartbeat();
+    }
+    finishJob(job) {
+        job.allocation.release();
+        this.jobs.delete(job.offer.offer_id);
+    }
+}
+function validateResult(value) {
+    if (!value || !new Set(["succeeded", "failed", "cancelled"]).has(value.status)
+        || value.release !== undefined && typeof value.release !== "function"
+        || value.artifacts !== undefined && (!Array.isArray(value.artifacts) || value.artifacts.some((entry) => !entry
+            || !new Set(["result-bundle", "diagnostic"]).has(entry.kind) || !Buffer.isBuffer(entry.body)))) {
+        throw runnerError("remote worker executor returned an invalid result");
+    }
+}
+function diagnostic(error, credentialValues) {
+    return Buffer.from(`${JSON.stringify({ schema_version: "1", error: safeDiagnosticMessage(error, credentialValues, 2_048), at: new Date().toISOString() })}\n`);
+}
+function rejectionCode(error) {
+    const code = error?.code;
+    return typeof code === "string" && /^[a-z0-9][a-z0-9._-]{0,127}$/.test(code) ? code : "invalid-input";
+}
+async function retry(operation, retryMs, signal) {
+    while (true) {
+        try {
+            return await operation();
+        }
+        catch (error) {
+            if (signal?.aborted)
+                throw error;
+            await delay(retryMs, signal);
+        }
+    }
+}
+function delay(milliseconds, signal) {
+    if (signal?.aborted)
+        return Promise.resolve();
+    return new Promise((resolve) => {
+        const timer = setTimeout(done, milliseconds);
+        function done() { signal?.removeEventListener("abort", done); clearTimeout(timer); resolve(); }
+        signal?.addEventListener("abort", done, { once: true });
+    });
+}
+function interval(value, label) {
+    if (!Number.isSafeInteger(value) || value < 50 || value > 5 * 60_000)
+        throw new TypeError(`remote worker ${label} interval is invalid`);
+    return value;
+}
+function runnerError(message) { return new HitchError(message, { code: "remote_worker_invalid_input", exitCode: 12 }); }
+//# sourceMappingURL=remote-worker-runner.js.map

package/dist/src/control-plane/remote-worker-runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"remote-worker-runner.js","sourceRoot":"","sources":["../../../src/control-plane/remote-worker-runner.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,qBAAqB,EAAE,MAAM,wBAAwB,CAAC;AAC3E,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AA2ChD,MAAM,eAAe,GAAG,IAAI,GAAG,CAA+B,CAAC,WAAW,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,YAAY,CAAC,CAAC,CAAC;AAErI,MAAM,OAAO,kBAAkB;IACZ,MAAM,CAAyB;IAC/B,OAAO,CAAuB;IAC9B,MAAM,CAA0B;IAChC,IAAI,CAAU;IACd,cAAc,CAAS;IACvB,mBAAmB,CAAS;IAC5B,eAAe,CAAS;IACxB,OAAO,CAA2B;IAClC,cAAc,CAA8C;IAC5D,MAAM,CAAiB;IACvB,IAAI,GAAG,IAAI,GAAG,EAAqB,CAAC;IAC7C,OAAO,GAAG,CAAC,CAAC;IACZ,aAAa,GAAG,CAAC,CAAC;IAE1B,YAAY,OAAkC;QAC5C,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;QAC7B,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;QAC/B,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;QAC7B,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,KAAK,CAAC;QAClC,IAAI,CAAC,cAAc,GAAG,QAAQ,CAAC,OAAO,CAAC,cAAc,IAAI,KAAK,EAAE,MAAM,CAAC,CAAC;QACxE,IAAI,CAAC,mBAAmB,GAAG,QAAQ,CAAC,OAAO,CAAC,mBAAmB,IAAI,MAAM,EAAE,WAAW,CAAC,CAAC;QACxF,IAAI,CAAC,eAAe,GAAG,QAAQ,CAAC,OAAO,CAAC,eAAe,IAAI,KAAK,EAAE,OAAO,CAAC,CAAC;QAC3E,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QACpD,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;QAC1E,IAAI,CAAC,MAAM,GAAG,IAAI,cAAc,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACrD,CAAC;IAED,KAAK,CAAC,GAAG;QACP,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;YAC7B,IAAI,CAAC;gBAAC,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;YAAC,CAAC;YAC1B,OAAO,KAAK,EAAE,CAAC;gBAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YAAC,CAAC;YACtC,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,GAAG,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC;gBAAE,OAAO;YAClE,MAAM,KAAK,CAAC,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAChD,CAAC;QACD,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAAE,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAChF,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;IAC9E,CAAC;IAED,KAAK,CAAC,IAAI;QACR,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;QAC9C,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;QACxE,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;QACnC,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;YACrC,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YAChD,IAAI,CAAC,OAAO;gBAAE,SAAS;YACvB,GAAG,CAAC,KAAK,GAAG,OAAO,CAAC;YACpB,IAAI,OAAO,CAAC,KAAK,KAAK,kBAAkB;gBAAE,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC,CAAC;YAChH,IAAI,OAAO,CAAC,KAAK,KAAK,mBAAmB,EAAE,CAAC;gBAAC,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;gBAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YAAC,CAAC;QAC5G,CAAC;QACD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC;gBAAE,SAAS;YAC5E,IAAI,KAAK,CAAC,KAAK,KAAK,mBAAmB,IAAI,KAAK,CAAC,KAAK,KAAK,kBAAkB,EAAE,CAAC;gBAC9E,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;gBACjC,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;gBACjC,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC;YACpB,CAAC;iBAAM,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,KAAK,CAAC,CAAC,EAAE,CAAC;gBAC3E,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YACzB,CAAC;QACH,CAAC;QACD,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,mBAAmB;YAAE,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;IAC1F,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,SAAiD,SAAS;QACxE,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACjF,QAAQ,EAAE,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK;SACjE,CAAC,CAAC,CAAC;QACJ,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9E,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAClC,CAAC;IAEO,UAAU,CAAC,KAAwB;QACzC,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QACjG,IAAI,CAAC,UAAU;YAAE,OAAO;QACxB,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,GAAG,GAAc;YACrB,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS;YAClF,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE;SAC3B,CAAC;QACF,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QACnC,GAAG,CAAC,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;YAC/C,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACpB,IAAI,CAAC,GAAG,CAAC,QAAQ;gBAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QACzC,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,QAAQ,CAAC,GAAc;QACnC,IAAI,MAAiD,CAAC;QACtD,IAAI,CAAC;YACH,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAChD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC;YAC1D,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC;YAClB,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YACpB,OAAO;QACT,CAAC;QACD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QACrG,IAAI,QAAQ,CAAC,KAAK,KAAK,UAAU,EAAE,CAAC;YAClC,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC;YAClB,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YACpB,OAAO;QACT,CAAC;QACD,GAAG,CAAC,KAAK,GAAG,QAAQ,CAAC;QACrB,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC;QACpB,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QACvB,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CACrD,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,eAAe,EAAE,GAAG,CAAC,UAAU,CAAC,MAAM,CACtF,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC;QACjB,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,MAAM,IAAI,GAAG,CAAC,IAAY,EAAE,OAAiC,EAAE,EAAE,CAAC,KAAK,CACrE,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,CAAC,EAAE,IAAI,CAAC,eAAe,EAAE,GAAG,CAAC,UAAU,CAAC,MAAM,CAC1G,CAAC;QACF,IAAI,MAAmC,CAAC;QACxC,IAAI,CAAC;YACH,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,GAAG,CAAC,UAAU,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;YAC5G,cAAc,CAAC,MAAM,CAAC,CAAC;YACvB,GAAG,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QAC/B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,sEAAsE;YACtE,+DAA+D;YAC/D,8DAA8D;YAC9D,GAAG,CAAC,OAAO,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YACnD,MAAM,GAAG;gBACP,MAAM,EAAE,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ;gBAC9D,SAAS,EAAE,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,UAAU,CAAC,KAAK,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC;aACxF,CAAC;QACJ,CAAC;gBAAS,CAAC;YACT,WAAW,CAAC,KAAK,EAAE,CAAC;QACtB,CAAC;QACD,MAAM,SAAS,GAA8B,EAAE,CAAC;QAChD,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,IAAI,EAAE,EAAE,CAAC;YAC9C,SAAS,CAAC,IAAI,CAAC,MAAM,KAAK,CACxB,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,MAAM,CAC7G,CAAC,CAAC;QACL,CAAC;QACD,GAAG,CAAC,KAAK,GAAG,MAAM,KAAK,CACrB,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,EAAE,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,MAAM,CACnG,CAAC;QACF,GAAG,CAAC,OAAO,GAAG,IAAI,CAAC;QACnB,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;IACzB,CAAC;IAEO,KAAK,CAAC,cAAc,CAAC,KAAwB;QACnD,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,IAAI,EAAE,CAAC;QAChC,IAAI,IAAI,CAAC,MAAM,KAAK,eAAe,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;eACzF,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,KAAK,eAAe,CAAC,IAAI;YAAE,MAAM,WAAW,CAAC,8CAA8C,CAAC,CAAC;QAC3I,MAAM,MAAM,GAAG,IAAI,GAAG,EAAwC,CAAC;QAC/D,KAAK,MAAM,GAAG,IAAI,IAAI;YAAE,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;QAC1F,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,GAAc;QACrC,MAAM,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC;QACtB,GAAG,CAAC,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACjD,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC;QAClB,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QACpB,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;IACzB,CAAC;IAEO,SAAS,CAAC,GAAc;QAC9B,GAAG,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;QACzB,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IACvC,CAAC;CACF;AAED,SAAS,cAAc,CAAC,KAAkC;IACxD,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,WAAW,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC;WACzE,KAAK,CAAC,OAAO,KAAK,SAAS,IAAI,OAAO,KAAK,CAAC,OAAO,KAAK,UAAU;WAClE,KAAK,CAAC,SAAS,KAAK,SAAS,IAAI,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK;eACzG,CAAC,IAAI,GAAG,CAAC,CAAC,eAAe,EAAE,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;QACnG,MAAM,WAAW,CAAC,mDAAmD,CAAC,CAAC;IACzE,CAAC;AACH,CAAC;AAED,SAAS,UAAU,CAAC,KAAc,EAAE,gBAAmC;IACrE,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,cAAc,EAAE,GAAG,EAAE,KAAK,EAAE,qBAAqB,CAAC,KAAK,EAAE,gBAAgB,EAAE,KAAK,CAAC,EAAE,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;AACjK,CAAC;AAED,SAAS,aAAa,CAAC,KAAc;IACnC,MAAM,IAAI,GAAI,KAA4B,EAAE,IAAI,CAAC;IACjD,OAAO,OAAO,IAAI,KAAK,QAAQ,IAAI,8BAA8B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,eAAe,CAAC;AACxG,CAAC;AAED,KAAK,UAAU,KAAK,CAAI,SAA2B,EAAE,OAAe,EAAE,MAAoB;IACxF,OAAO,IAAI,EAAE,CAAC;QACZ,IAAI,CAAC;YAAC,OAAO,MAAM,SAAS,EAAE,CAAC;QAAC,CAAC;QACjC,OAAO,KAAK,EAAE,CAAC;YACb,IAAI,MAAM,EAAE,OAAO;gBAAE,MAAM,KAAK,CAAC;YACjC,MAAM,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,KAAK,CAAC,YAAoB,EAAE,MAAoB;IACvD,IAAI,MAAM,EAAE,OAAO;QAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IAC9C,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;QAC7C,SAAS,IAAI,KAAW,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QACrG,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1D,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,QAAQ,CAAC,KAAa,EAAE,KAAa;IAC5C,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,EAAE,IAAI,KAAK,GAAG,CAAC,GAAG,MAAM;QAAE,MAAM,IAAI,SAAS,CAAC,iBAAiB,KAAK,sBAAsB,CAAC,CAAC;IACxI,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,WAAW,CAAC,OAAe,IAAgB,OAAO,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,6BAA6B,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC"}

package/dist/src/control-plane/remote-workers.js

@@ -0,0 +1,377 @@
+import { randomBytes, timingSafeEqual } from "node:crypto";
+import { readdir } from "node:fs/promises";
+import path from "node:path";
+import { HitchError, atomicWriteJSON, ensureDir, readJSON, sha256Bytes, statePaths, withFileLock } from "../foundation/index.js";
+import { validateResourceVector } from "./resources.js";
+const WORKER_ID = /^worker_[a-z0-9][a-z0-9_-]{0,62}$/;
+const LEASE_ID = /^lease_[a-f0-9]{32}$/;
+const DEFAULT_HEARTBEAT_TTL_MS = 45_000;
+export class RemoteWorkerRegistry {
+    root;
+    directory;
+    locks;
+    heartbeatTtlMs;
+    constructor(input) {
+        if (!input.root)
+            throw new TypeError("remote worker registry root is required");
+        const ttl = input.heartbeatTtlMs ?? DEFAULT_HEARTBEAT_TTL_MS;
+        if (!Number.isSafeInteger(ttl) || ttl < 1_000)
+            throw new TypeError("remote worker heartbeat TTL is invalid");
+        const paths = statePaths(input.root);
+        this.root = paths.root;
+        this.directory = paths.workers;
+        this.locks = paths.workerLocks;
+        this.heartbeatTtlMs = ttl;
+    }
+    async initialize() {
+        await Promise.all([ensureDir(this.directory), ensureDir(this.locks)]);
+    }
+    async register(value) {
+        const registration = parseRemoteWorkerRegistration(value);
+        const token = randomBytes(32).toString("hex");
+        const tokenHash = sha256Bytes(token);
+        const persisted = await withFileLock(this.locks, registration.worker_id, async () => {
+            const existing = await this.readPersisted(registration.worker_id);
+            if (existing && (existing.worker.provider !== registration.provider
+                || existing.worker.collision_domain_id !== registration.collision_domain_id)) {
+                throw new HitchError("remote worker identity cannot be rebound", { code: "worker_identity_conflict", exitCode: 2 });
+            }
+            const now = new Date().toISOString();
+            const next = persistedWorker(registration, tokenHash, existing, now);
+            await atomicWriteJSON(this.workerPath(registration.worker_id), next);
+            return next;
+        }, { timeoutCode: "worker_record_locked", timeoutExitCode: 12 });
+        return { worker: publicRecord(persisted), token };
+    }
+    async authenticate(workerId, token) {
+        if (!WORKER_ID.test(workerId) || !/^[a-f0-9]{64}$/.test(token))
+            return false;
+        const record = await this.readPersisted(workerId);
+        if (!record || record.revoked_at)
+            return false;
+        const supplied = Buffer.from(sha256Bytes(token));
+        const expected = Buffer.from(record.token_hash);
+        return supplied.length === expected.length && timingSafeEqual(supplied, expected);
+    }
+    async heartbeat(workerId, value) {
+        if (!WORKER_ID.test(workerId))
+            throw invalidWorker("remote worker id is invalid");
+        const heartbeat = parseRemoteWorkerHeartbeat(value);
+        const persisted = await withFileLock(this.locks, workerId, async () => {
+            const current = await this.requirePersisted(workerId);
+            if (current.revoked_at)
+                throw new HitchError("remote worker identity is revoked", { code: "worker_revoked", exitCode: 11 });
+            if (heartbeat.generation !== current.generation)
+                throw new HitchError("remote worker generation is stale", { code: "worker_generation_mismatch", exitCode: 12 });
+            assertWithin(heartbeat.allocated, current.worker.capacity.allocatable, "remote worker allocation exceeds allocatable capacity");
+            const next = {
+                ...current,
+                worker: {
+                    ...current.worker,
+                    status: heartbeat.health === "unavailable" ? "offline" : "ready",
+                    capacity: { ...current.worker.capacity, allocated: heartbeat.allocated },
+                },
+                provider_status: {
+                    ...current.provider_status,
+                    health: heartbeat.health,
+                    capacity: { ...current.provider_status.capacity, allocated: heartbeat.allocated },
+                    heartbeat_at: heartbeat.sent_at,
+                },
+                active_leases: heartbeat.active_leases,
+                heartbeat_at: heartbeat.sent_at,
+            };
+            await atomicWriteJSON(this.workerPath(workerId), next);
+            return next;
+        }, { timeoutCode: "worker_record_locked", timeoutExitCode: 12 });
+        return publicRecord(persisted);
+    }
+    async validateHeartbeatGeneration(workerId, generation) {
+        if (!WORKER_ID.test(workerId) || !Number.isSafeInteger(generation) || generation < 1)
+            throw invalidWorker("remote worker heartbeat identity is invalid");
+        const current = await this.requirePersisted(workerId);
+        if (current.revoked_at)
+            throw new HitchError("remote worker identity is revoked", { code: "worker_revoked", exitCode: 11 });
+        if (generation !== current.generation)
+            throw new HitchError("remote worker generation is stale", { code: "worker_generation_mismatch", exitCode: 12 });
+    }
+    async revoke(workerId) {
+        if (!WORKER_ID.test(workerId))
+            throw invalidWorker("remote worker id is invalid");
+        const persisted = await withFileLock(this.locks, workerId, async () => {
+            const current = await this.requirePersisted(workerId);
+            if (current.revoked_at)
+                return current;
+            const now = new Date().toISOString();
+            const next = {
+                ...current,
+                generation: current.generation + 1,
+                worker: { ...current.worker, status: "offline" },
+                provider_status: { ...current.provider_status, health: "unavailable", heartbeat_at: now },
+                active_leases: [],
+                revoked_at: now,
+                heartbeat_at: now,
+            };
+            await atomicWriteJSON(this.workerPath(workerId), next);
+            return next;
+        }, { timeoutCode: "worker_record_locked", timeoutExitCode: 12 });
+        return publicRecord(persisted);
+    }
+    async get(workerId, now = Date.now()) {
+        if (!WORKER_ID.test(workerId))
+            throw invalidWorker("remote worker id is invalid");
+        const record = await this.readPersisted(workerId);
+        return record ? publicRecord(withLiveness(record, now, this.heartbeatTtlMs)) : null;
+    }
+    async list(now = Date.now()) {
+        await ensureDir(this.directory);
+        const entries = await readdir(this.directory, { withFileTypes: true });
+        const records = [];
+        for (const entry of entries.sort((left, right) => Buffer.from(left.name).compare(Buffer.from(right.name)))) {
+            if (!entry.isFile() || !/^worker_[a-z0-9][a-z0-9_-]{0,62}\.json$/.test(entry.name))
+                continue;
+            const parsed = parsePersistedRemoteWorker(await readJSON(path.join(this.directory, entry.name)));
+            records.push(publicRecord(withLiveness(parsed, now, this.heartbeatTtlMs)));
+        }
+        return records;
+    }
+    workerPath(workerId) {
+        return path.join(this.directory, `${workerId}.json`);
+    }
+    readPersisted(workerId) {
+        return readJSON(this.workerPath(workerId), null).then((value) => value === null ? null : parsePersistedRemoteWorker(value));
+    }
+    async requirePersisted(workerId) {
+        const record = await this.readPersisted(workerId);
+        if (!record)
+            throw new HitchError(`remote worker not found: ${workerId}`, { code: "worker_not_found", exitCode: 3 });
+        return record;
+    }
+}
+export function parseRemoteWorkerRegistration(value) {
+    const record = exact(value, ["schema_version", "worker_id", "provider", "collision_domain_id", "platforms", "backends", "features", "task_membership", "capacity"], "remote worker registration");
+    if (record.schema_version !== "1" || typeof record.worker_id !== "string" || !WORKER_ID.test(record.worker_id)
+        || !validText(record.provider) || !validText(record.collision_domain_id))
+        throw invalidWorker("remote worker registration identity is invalid");
+    const platforms = stringSet(record.platforms, "remote worker platforms");
+    const memberships = stringSet(record.task_membership, "remote worker task membership");
+    if (platforms.length === 0 || memberships.length === 0 || memberships.some((entry) => entry !== "known" && entry !== "opaque"))
+        throw invalidWorker("remote worker capabilities are invalid");
+    const backends = parseBackends(record.backends);
+    const features = parseFeatures(record.features);
+    const capacity = parseRegistrationCapacity(record.capacity);
+    return {
+        schema_version: "1", worker_id: record.worker_id, provider: record.provider,
+        collision_domain_id: record.collision_domain_id, platforms, backends, features,
+        task_membership: memberships, capacity,
+    };
+}
+export function parseRemoteWorkerHeartbeat(value) {
+    const record = exact(value, ["schema_version", "generation", "health", "allocated", "active_leases", "sent_at"], "remote worker heartbeat");
+    if (record.schema_version !== "1" || !Number.isSafeInteger(record.generation) || record.generation < 1
+        || !new Set(["healthy", "degraded", "unavailable"]).has(String(record.health))
+        || typeof record.sent_at !== "string" || !Number.isFinite(Date.parse(record.sent_at))
+        || Math.abs(Date.now() - Date.parse(record.sent_at)) > 5 * 60_000 || !Array.isArray(record.active_leases)) {
+        throw invalidWorker("remote worker heartbeat is invalid");
+    }
+    const leases = parseHeartbeatLeases(record.active_leases);
+    return {
+        schema_version: "1", generation: record.generation,
+        health: record.health,
+        allocated: validateResourceVector(record.allocated, "remote worker allocated resources"),
+        active_leases: leases, sent_at: record.sent_at,
+    };
+}
+function parseHeartbeatLeases(value) {
+    if (!Array.isArray(value))
+        throw invalidWorker("remote worker heartbeat leases are invalid");
+    const leases = value.map((value) => {
+        const lease = exact(value, ["lease_id", "epoch"], "remote worker heartbeat lease");
+        if (typeof lease.lease_id !== "string" || !LEASE_ID.test(lease.lease_id)
+            || !Number.isSafeInteger(lease.epoch) || lease.epoch < 1)
+            throw invalidWorker("remote worker heartbeat lease is invalid");
+        return { lease_id: lease.lease_id, epoch: lease.epoch };
+    }).sort((left, right) => left.lease_id.localeCompare(right.lease_id));
+    if (new Set(leases.map((lease) => lease.lease_id)).size !== leases.length)
+        throw invalidWorker("remote worker heartbeat leases are duplicated");
+    return leases;
+}
+function persistedWorker(registration, tokenHash, existing, now) {
+    const allocated = zero();
+    const worker = {
+        schema_version: "1", worker_id: registration.worker_id, provider: registration.provider, status: "ready",
+        collision_domain_id: registration.collision_domain_id,
+        capabilities: {
+            backends: registration.backends.map((entry) => entry.id), platforms: registration.platforms,
+            task_membership: registration.task_membership,
+            isolated_same_task_attempts: registration.features.isolated_same_task_attempts, remote: true,
+        },
+        capacity: { ...registration.capacity, allocated },
+    };
+    const providerStatus = {
+        schema_version: "1", provider: registration.provider, worker_id: registration.worker_id,
+        collision_domain_id: registration.collision_domain_id, health: "healthy", platforms: registration.platforms,
+        backends: registration.backends, features: registration.features,
+        capacity: { total: registration.capacity.total, allocatable: registration.capacity.allocatable, allocated }, heartbeat_at: now,
+    };
+    return {
+        schema_version: "1", generation: (existing?.generation ?? 0) + 1, worker, provider_status: providerStatus,
+        active_leases: [], token_hash: tokenHash, registered_at: existing?.registered_at ?? now, heartbeat_at: now,
+    };
+}
+function parsePersistedRemoteWorker(value) {
+    const record = exact(value, ["schema_version", "generation", "worker", "provider_status", "active_leases", "token_hash", "registered_at", "heartbeat_at", "revoked_at"], "persisted remote worker");
+    if (record.schema_version !== "1" || !Number.isSafeInteger(record.generation) || record.generation < 1
+        || typeof record.token_hash !== "string" || !/^sha256:[a-f0-9]{64}$/.test(record.token_hash)
+        || !validTimestamp(record.registered_at) || !validTimestamp(record.heartbeat_at)
+        || record.revoked_at !== undefined && !validTimestamp(record.revoked_at))
+        throw invalidWorker("persisted remote worker identity is invalid");
+    const worker = parsePersistedWorker(record.worker);
+    const provider = parsePersistedProvider(record.provider_status);
+    const activeLeases = parseHeartbeatLeases(record.active_leases);
+    if (worker.worker_id !== provider.worker_id || worker.provider !== provider.provider || worker.collision_domain_id !== provider.collision_domain_id
+        || JSON.stringify(worker.capacity.allocated) !== JSON.stringify(provider.capacity.allocated))
+        throw invalidWorker("persisted remote worker views are inconsistent");
+    return {
+        schema_version: "1", generation: record.generation, worker, provider_status: provider,
+        active_leases: activeLeases, token_hash: record.token_hash,
+        registered_at: record.registered_at, heartbeat_at: record.heartbeat_at,
+        ...(record.revoked_at === undefined ? {} : { revoked_at: record.revoked_at }),
+    };
+}
+function parsePersistedWorker(value) {
+    const record = exact(value, ["schema_version", "worker_id", "provider", "status", "collision_domain_id", "capabilities", "capacity"], "persisted execution worker");
+    const capabilities = exact(record.capabilities, ["backends", "platforms", "task_membership", "isolated_same_task_attempts", "remote"], "persisted worker capabilities");
+    const capacity = exact(record.capacity, ["total", "reserved_for_system", "allocatable", "allocated"], "persisted worker capacity");
+    const registration = parseRemoteWorkerRegistration({
+        schema_version: record.schema_version, worker_id: record.worker_id, provider: record.provider,
+        collision_domain_id: record.collision_domain_id, platforms: capabilities.platforms,
+        backends: stringSet(capabilities.backends, "persisted worker backends").map((id) => ({ id, version: "unknown" })),
+        features: { docker: true, buildkit: false, model_proxy: false, isolated_same_task_attempts: capabilities.isolated_same_task_attempts },
+        task_membership: capabilities.task_membership,
+        capacity: { total: capacity.total, reserved_for_system: capacity.reserved_for_system, allocatable: capacity.allocatable },
+    });
+    if (!new Set(["ready", "draining", "offline"]).has(String(record.status)) || capabilities.remote !== true)
+        throw invalidWorker("persisted execution worker status is invalid");
+    const allocated = validateResourceVector(capacity.allocated, "persisted worker allocated resources");
+    assertWithin(allocated, registration.capacity.allocatable, "persisted worker allocation exceeds capacity");
+    return {
+        schema_version: "1", worker_id: registration.worker_id, provider: registration.provider,
+        status: record.status, collision_domain_id: registration.collision_domain_id,
+        capabilities: {
+            backends: stringSet(capabilities.backends, "persisted worker backends"), platforms: registration.platforms,
+            task_membership: registration.task_membership, isolated_same_task_attempts: registration.features.isolated_same_task_attempts, remote: true,
+        }, capacity: { ...registration.capacity, allocated },
+    };
+}
+function parsePersistedProvider(value) {
+    const record = exact(value, ["schema_version", "provider", "worker_id", "collision_domain_id", "health", "platforms", "backends", "features", "capacity", "heartbeat_at"], "persisted provider status");
+    const capacity = exact(record.capacity, ["total", "allocatable", "allocated"], "persisted provider capacity");
+    const total = validateResourceVector(capacity.total, "persisted provider total resources");
+    const allocatable = validateResourceVector(capacity.allocatable, "persisted provider allocatable resources");
+    assertWithin(allocatable, total, "persisted provider allocatable resources exceed total capacity");
+    const registration = parseRemoteWorkerRegistration({
+        schema_version: record.schema_version, worker_id: record.worker_id, provider: record.provider,
+        collision_domain_id: record.collision_domain_id, platforms: record.platforms, backends: record.backends,
+        features: record.features, task_membership: ["known"],
+        capacity: { total, allocatable, reserved_for_system: subtract(total, allocatable) },
+    });
+    const allocated = validateResourceVector(capacity.allocated, "persisted provider allocated resources");
+    assertWithin(allocated, registration.capacity.allocatable, "persisted provider allocation exceeds capacity");
+    if (!new Set(["healthy", "degraded", "unavailable"]).has(String(record.health)) || !validTimestamp(record.heartbeat_at))
+        throw invalidWorker("persisted provider status is invalid");
+    return {
+        schema_version: "1", provider: registration.provider, worker_id: registration.worker_id,
+        collision_domain_id: registration.collision_domain_id, health: record.health,
+        platforms: registration.platforms, backends: registration.backends, features: registration.features,
+        capacity: { total: registration.capacity.total, allocatable: registration.capacity.allocatable, allocated }, heartbeat_at: record.heartbeat_at,
+    };
+}
+function parseRegistrationCapacity(value) {
+    const capacity = exact(value, ["total", "reserved_for_system", "allocatable"], "remote worker capacity");
+    const total = validateResourceVector(capacity.total, "remote worker total capacity");
+    const reserved = validateResourceVector(capacity.reserved_for_system, "remote worker reserved capacity");
+    const allocatable = validateResourceVector(capacity.allocatable, "remote worker allocatable capacity");
+    for (const field of resourceFields())
+        if (resourceValue(reserved, field) + resourceValue(allocatable, field) !== resourceValue(total, field))
+            throw invalidWorker("remote worker capacity accounting is invalid");
+    return { total, reserved_for_system: reserved, allocatable };
+}
+function parseBackends(value) {
+    if (!Array.isArray(value) || value.length === 0)
+        throw invalidWorker("remote worker backends are invalid");
+    const result = value.map((entry) => {
+        const backend = exact(entry, ["id", "version"], "remote worker backend");
+        if (!validText(backend.id) || !validText(backend.version))
+            throw invalidWorker("remote worker backend is invalid");
+        return { id: backend.id, version: backend.version };
+    }).sort((left, right) => left.id.localeCompare(right.id));
+    if (new Set(result.map((entry) => entry.id)).size !== result.length)
+        throw invalidWorker("remote worker backends are duplicated");
+    return result;
+}
+function parseFeatures(value) {
+    const features = exact(value, ["docker", "buildkit", "model_proxy", "isolated_same_task_attempts"], "remote worker features");
+    if (Object.values(features).some((entry) => typeof entry !== "boolean"))
+        throw invalidWorker("remote worker features are invalid");
+    return features;
+}
+function withLiveness(record, now, ttl) {
+    if (record.revoked_at || now - Date.parse(record.heartbeat_at) <= ttl)
+        return record;
+    return { ...record, worker: { ...record.worker, status: "offline" }, provider_status: { ...record.provider_status, health: "unavailable" } };
+}
+function publicRecord(record) {
+    const { token_hash: _tokenHash, ...visible } = record;
+    return structuredClone(visible);
+}
+function exact(value, keys, label) {
+    if (!value || typeof value !== "object" || Array.isArray(value))
+        throw invalidWorker(`${label} must be an object`);
+    const record = value;
+    if (Object.keys(record).some((key) => !keys.includes(key)))
+        throw invalidWorker(`${label} has unknown fields`);
+    return record;
+}
+function stringSet(value, label) {
+    if (!Array.isArray(value) || value.some((entry) => !validText(entry)))
+        throw invalidWorker(`${label} are invalid`);
+    const result = [...value].sort();
+    if (new Set(result).size !== result.length)
+        throw invalidWorker(`${label} are duplicated`);
+    return result;
+}
+function assertWithin(value, limit, message) {
+    for (const field of resourceFields())
+        if (resourceValue(value, field) > resourceValue(limit, field))
+            throw invalidWorker(message);
+}
+function resourceFields() {
+    return ["cpu_millis", "memory_bytes", "container_slots", "build_slots", "gpu_count", "ephemeral_disk_bytes"];
+}
+function zero() {
+    return { cpu_millis: 0, memory_bytes: 0, container_slots: 0, build_slots: 0 };
+}
+function subtract(left, right) {
+    const result = {
+        cpu_millis: left.cpu_millis - right.cpu_millis,
+        memory_bytes: left.memory_bytes - right.memory_bytes,
+        container_slots: left.container_slots - right.container_slots,
+        build_slots: left.build_slots - right.build_slots,
+    };
+    if (left.gpu_count !== undefined || right.gpu_count !== undefined)
+        result.gpu_count = (left.gpu_count ?? 0) - (right.gpu_count ?? 0);
+    if (left.ephemeral_disk_bytes !== undefined || right.ephemeral_disk_bytes !== undefined)
+        result.ephemeral_disk_bytes = (left.ephemeral_disk_bytes ?? 0) - (right.ephemeral_disk_bytes ?? 0);
+    return result;
+}
+function resourceValue(resources, field) { return resources[field] ?? 0; }
+function validText(value) {
+    return typeof value === "string" && value.length > 0 && value.length <= 1_024 && !/[\0\r\n]/.test(value);
+}
+function validTimestamp(value) {
+    return typeof value === "string" && Number.isFinite(Date.parse(value));
+}
+function invalidWorker(message) {
+    return new HitchError(message, { code: "worker_protocol_invalid", exitCode: 2 });
+}
+//# sourceMappingURL=remote-workers.js.map