agent-hitch 0.2.9 → 0.2.11

package/dist/src/control-plane/model-capture-planning.js.map

@@ -1 +1 @@
-{"version":3,"file":"model-capture-planning.js","sourceRoot":"","sources":["../../../src/control-plane/model-capture-planning.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAElD,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAE5D,MAAM,UAAU,uBAAuB,CACrC,OAAoB,EACpB,SAAgC,EAChC,QAAmC;IAEnC,MAAM,SAAS,GAAG,OAAO,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACnD,MAAM,SAAS,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;IAChG,OAAO,gBAAgB,CAAC;QACtB,MAAM,EAAE,SAAS,CAAC,aAAa;QAC/B,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC,YAAY;QAC3C,QAAQ;KACT,CAAC,CAAC;AACL,CAAC"}
+{"version":3,"file":"model-capture-planning.js","sourceRoot":"","sources":["../../../src/control-plane/model-capture-planning.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAElD,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAC5D,OAAO,EAAE,8BAA8B,EAAE,MAAM,mBAAmB,CAAC;AACnE,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAEpD,MAAM,UAAU,uBAAuB,CACrC,OAAoB,EACpB,SAAgC,EAChC,QAAmC;IAEnC,MAAM,SAAS,GAAG,OAAO,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACnD,MAAM,SAAS,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;IAChG,MAAM,IAAI,GAAG,gBAAgB,CAAC;QAC5B,MAAM,EAAE,SAAS,CAAC,aAAa;QAC/B,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC,YAAY;QAC3C,QAAQ;KACT,CAAC,CAAC;IACH,IAAI,OAAO,CAAC,eAAe,IAAI,OAAO,CAAC,gBAAgB,EAAE,CAAC;QACxD,MAAM,MAAM,GAAG,SAAS,CAAC,QAAQ,KAAK,cAAc,CAAC;QACrD,MAAM,OAAO,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,2BAA2B,CAAC,CAAC,CAAC,oBAAoB,CAAC;QAC9F,IAAI,MAAM,IAAI,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,WAAW,IAAI,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,GAAG;eAC9E,OAAO,CAAC,eAAe,IAAI,CAAC,CAAC,OAAO,CAAC,eAAe,CAAC,UAAU,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,YAAY,CAAC,CAAC,EAAE,CAAC;YAChH,MAAM,IAAI,UAAU,CAAC,4BAA4B,OAAO,iCAAiC,EAAE;gBACzF,IAAI,EAAE,kCAAkC,EAAE,QAAQ,EAAE,EAAE;aACvD,CAAC,CAAC;QACL,CAAC;QACD,OAAO,8BAA8B,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;IACnF,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC"}

package/dist/src/control-plane/ordered-eval-control.js

@@ -0,0 +1,221 @@
+import path from "node:path";
+import { HitchError, atomicWriteJSON, readJSON, sha256Bytes, sha256JSON, statePaths, withFileLock } from "../foundation/index.js";
+import { newEvalId, validateEvalId } from "../evals/index.js";
+import { idempotencyIndexPath, validateIdempotencyKey } from "./eval-records.js";
+const authorities = new Map();
+function fail(message, code = "eval_control_invalid") {
+    throw new HitchError(message, { code, exitCode: 12 });
+}
+function object(value) {
+    if (!value || typeof value !== "object" || Array.isArray(value)) {
+        fail("eval control requires an object");
+    }
+    return value;
+}
+export function evalCommandPath(root, hash) {
+    return path.join(statePaths(root).indexes, "eval-commands", `${hash.slice(7)}.json`);
+}
+function bindingPath(root, id) {
+    return path.join(statePaths(root).indexes, "eval-command-bindings", `${id}.json`);
+}
+async function readCommandBinding(root, evalId) {
+    const binding = await readJSON(bindingPath(root, evalId), null);
+    const submission = await readJSON(path.join(statePaths(root).evals, evalId, "submission.json"), null);
+    return { binding, keyHash: binding?.key_hash ?? submission?.idempotency_key_hash };
+}
+export function parseOrderedEvalCommand(value) {
+    const input = object(value);
+    const fields = ["schema_version", "key", "subject_digest", "sequence", "action", "submission", "rerun"];
+    if (Object.keys(input).some(key => !fields.includes(key))
+        || input.schema_version !== "2" || typeof input.key !== "string"
+        || !/^sha256:[a-f0-9]{64}$/.test(String(input.subject_digest))
+        || !Number.isSafeInteger(input.sequence) || Number(input.sequence) < 0
+        || !["start", "pause"].includes(String(input.action))) {
+        fail("invalid ordered eval intent");
+    }
+    validateIdempotencyKey(input.key);
+    const hasSubmission = input.submission !== undefined;
+    const hasRerun = input.rerun !== undefined;
+    if ((hasSubmission && hasRerun) || (input.action === "pause" && (hasSubmission || hasRerun))) {
+        fail("pause cannot submit work");
+    }
+    if (hasSubmission) {
+        const submission = object(input.submission);
+        const request = object(submission.request);
+        const submissionFields = ["schema_version", "request", "execution", "idempotency_key"];
+        if (request.backend !== "harbor" || Object.keys(submission).some(key => !submissionFields.includes(key))) {
+            fail("controlled submission requires a canonical submission envelope");
+        }
+    }
+    if (hasRerun) {
+        const rerun = object(input.rerun);
+        const body = object(rerun.input);
+        if (Object.keys(rerun).some(key => !["eval_id", "input"].includes(key))
+            || !/^rerun_[a-f0-9]{32}$/.test(String(body.rerun_id))) {
+            fail("controlled rerun requires a stable identity");
+        }
+        validateEvalId(String(rerun.eval_id));
+    }
+    return input;
+}
+export async function readOrderedEvalIntent(file) {
+    const command = parseOrderedEvalCommand(await readJSON(file));
+    if (command.submission !== undefined || command.rerun !== undefined) {
+        fail("control file must contain only an intent");
+    }
+    return command;
+}
+export async function assertOrderedSubmission(root, keyHash, input) {
+    const file = evalCommandPath(root, keyHash);
+    if (authorities.get(file) !== input.authority) {
+        fail("ordered submission must hold its command lock", "eval_control_required");
+    }
+    const record = await readJSON(file, null);
+    if (!record || record.eval_id !== input.evalId || record.subject_digest !== input.subjectDigest
+        || record.sequence !== input.sequence || record.action !== "start") {
+        fail("a newer eval command fences submission", "eval_control_stale");
+    }
+}
+function validRerunRecord(id, value, commandSequence) {
+    return /^rerun_[a-f0-9]{32}$/.test(id) && !!value && /^sha256:[a-f0-9]{64}$/.test(value.digest)
+        && Number.isSafeInteger(value.sequence) && value.sequence >= 0 && value.sequence <= commandSequence
+        && (value.cancelled_by === undefined || (Number.isSafeInteger(value.cancelled_by)
+            && value.cancelled_by >= value.sequence && value.cancelled_by <= commandSequence));
+}
+function assertCommandIdentity(record, keyHash, subjectDigest) {
+    if (record.schema_version !== "2" || record.key_hash !== keyHash || record.subject_digest !== subjectDigest
+        || !Number.isSafeInteger(record.sequence) || record.sequence < 0
+        || !["start", "pause"].includes(record.action) || !record.reruns || Array.isArray(record.reruns)
+        || Object.entries(record.reruns).some(([id, value]) => !validRerunRecord(id, value, record.sequence))) {
+        fail("eval control identity changed", "eval_control_conflict");
+    }
+}
+/** Admission and all controlled mutations use the original idempotency lock. */
+export async function orderedEvalControl(root, value, evals, reruns) {
+    const command = parseOrderedEvalCommand(value);
+    const keyHash = sha256Bytes(command.key);
+    const file = evalCommandPath(root, keyHash);
+    return withFileLock(path.join(root, "locks", "eval-idempotency"), keyHash, async () => {
+        let record = await readJSON(file, null);
+        let index = await readJSON(idempotencyIndexPath(root, keyHash), null);
+        const evalId = validateEvalId(record?.eval_id ?? index?.eval_id ?? newEvalId());
+        return withFileLock(path.join(root, "locks", "eval-command-mutations"), evalId, async () => {
+            if (record) {
+                assertCommandIdentity(record, keyHash, command.subject_digest);
+                if (index && index.eval_id !== evalId) {
+                    fail("eval control identity changed", "eval_control_conflict");
+                }
+                if (command.sequence < record.sequence) {
+                    fail("a newer eval command fences this request", "eval_control_stale");
+                }
+                if (command.sequence === record.sequence && command.action !== record.action) {
+                    fail("eval control sequence cannot change meaning", "eval_control_conflict");
+                }
+                // A newer start cannot erase a pause whose scheduler writes were interrupted.
+                if (record.action === "pause" && command.action === "start") {
+                    await applyPause(file, record, !!index, evals, reruns);
+                }
+            }
+            record = {
+                schema_version: "2", key_hash: keyHash, subject_digest: command.subject_digest, eval_id: evalId,
+                sequence: command.sequence, action: command.action, reruns: record?.reruns ?? {},
+            };
+            await atomicWriteJSON(file, record);
+            await atomicWriteJSON(bindingPath(root, evalId), { schema_version: "2", key_hash: keyHash, eval_id: evalId });
+            if (command.submission !== undefined) {
+                const authority = Symbol("eval command");
+                authorities.set(file, authority);
+                try {
+                    await evals.submit(command.submission, {
+                        idempotencyKey: command.key,
+                        ordered: { evalId, subjectDigest: command.subject_digest, sequence: command.sequence, authority },
+                    });
+                }
+                finally {
+                    authorities.delete(file);
+                }
+                index = await readJSON(idempotencyIndexPath(root, keyHash), null);
+            }
+            let accepted = {};
+            if (command.rerun) {
+                if (!index || command.rerun.eval_id !== evalId) {
+                    fail("rerun must belong to the admitted eval", "eval_control_conflict");
+                }
+                const id = String(command.rerun.input.rerun_id);
+                const digest = sha256JSON(command.rerun.input);
+                const old = record.reruns[id];
+                if (old && old.digest !== digest) {
+                    fail("rerun identity already belongs to another request", "eval_control_conflict");
+                }
+                if (old?.cancelled_by !== undefined) {
+                    fail("this rerun identity was cancelled", "eval_rerun_cancelled");
+                }
+                record.reruns[id] ??= { digest, sequence: command.sequence };
+                await atomicWriteJSON(file, record);
+                const result = await reruns.submit(evalId, command.rerun.input);
+                accepted = { rerun_id: result.rerunId, rerun_type: result.rerunType };
+            }
+            if (command.action === "pause") {
+                await applyPause(file, record, !!index, evals, reruns);
+            }
+            let pendingReruns = false;
+            for (const id of Object.keys(record.reruns)) {
+                const status = await reruns.status(evalId, id);
+                pendingReruns ||= !!status && !["completed", "failed", "cancelled"].includes(String(status.state.status));
+            }
+            return {
+                schema_version: "2", eval_id: evalId, subject_digest: command.subject_digest, sequence: command.sequence,
+                action: command.action, submitted: !!index, pending_reruns: pendingReruns, ...accepted,
+            };
+        });
+    }, { timeoutCode: "idempotency_locked", timeoutExitCode: 12 });
+}
+async function applyPause(file, record, submitted, evals, reruns) {
+    for (const value of Object.values(record.reruns))
+        value.cancelled_by ??= record.sequence;
+    await atomicWriteJSON(file, record);
+    if (submitted)
+        await evals.cancel(record.eval_id);
+    for (const id of Object.keys(record.reruns))
+        await reruns.cancel(record.eval_id, id);
+}
+/** Consult the durable intent before recovery can enqueue or resume execution. */
+export async function orderedEvalCancelled(root, evalIdValue, rerunId) {
+    const evalId = validateEvalId(evalIdValue);
+    const { binding, keyHash } = await readCommandBinding(root, evalId);
+    if (!keyHash)
+        return false;
+    if (!/^sha256:[a-f0-9]{64}$/.test(keyHash)) {
+        fail("eval control key is invalid", "eval_control_conflict");
+    }
+    const record = await readJSON(evalCommandPath(root, keyHash), null);
+    if (!record) {
+        if (binding)
+            fail("eval control intent is missing", "eval_control_conflict");
+        return false;
+    }
+    if (record.schema_version !== "2" || record.key_hash !== keyHash || record.eval_id !== evalId
+        || !["start", "pause"].includes(record.action)
+        || !record.reruns || typeof record.reruns !== "object" || Array.isArray(record.reruns)) {
+        fail("eval control recovery identity changed", "eval_control_conflict");
+    }
+    if (!rerunId)
+        return record.action === "pause";
+    const rerun = record.reruns[rerunId];
+    return !!rerun && (record.action === "pause" || rerun.cancelled_by !== undefined);
+}
+/** Old daemon requests cannot mutate an eval after ordered control adopts it. */
+export async function withLegacyEvalMutation(root, evalIdValue, action) {
+    const evalId = validateEvalId(evalIdValue);
+    const { keyHash } = await readCommandBinding(root, evalId);
+    const locked = () => withFileLock(path.join(root, "locks", "eval-command-mutations"), evalId, async () => {
+        // Adoption may have occurred after the initial read, so check again under the lock.
+        if (await readJSON(bindingPath(root, evalId), null)
+            || (keyHash && await readJSON(evalCommandPath(root, keyHash), null))) {
+            fail("eval requires ordered control", "eval_control_required");
+        }
+        return action();
+    });
+    return keyHash ? withFileLock(path.join(root, "locks", "eval-idempotency"), keyHash, locked) : locked();
+}
+//# sourceMappingURL=ordered-eval-control.js.map

package/dist/src/control-plane/ordered-eval-control.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ordered-eval-control.js","sourceRoot":"","sources":["../../../src/control-plane/ordered-eval-control.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,UAAU,EAAE,eAAe,EAAE,QAAQ,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAClI,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAC9D,OAAO,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,MAAM,mBAAmB,CAAC;AAmCjF,MAAM,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;AAE9C,SAAS,IAAI,CAAC,OAAe,EAAE,IAAI,GAAG,sBAAsB;IAC1D,MAAM,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;AACxD,CAAC;AAED,SAAS,MAAM,CAAC,KAAc;IAC5B,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QAChE,IAAI,CAAC,iCAAiC,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,KAAa,CAAC;AACvB,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,IAAY,EAAE,IAAY;IACxD,OAAO,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,eAAe,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;AACvF,CAAC;AAED,SAAS,WAAW,CAAC,IAAY,EAAE,EAAU;IAC3C,OAAO,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,uBAAuB,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;AACpF,CAAC;AAED,KAAK,UAAU,kBAAkB,CAAC,IAAY,EAAE,MAAc;IAC5D,MAAM,OAAO,GAAG,MAAM,QAAQ,CAA8B,WAAW,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE,IAAI,CAAC,CAAC;IAC7F,MAAM,UAAU,GAAG,MAAM,QAAQ,CAC/B,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,iBAAiB,CAAC,EAAE,IAAI,CACnE,CAAC;IACF,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,IAAI,UAAU,EAAE,oBAAoB,EAAE,CAAC;AACrF,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,KAAc;IACpD,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC5B,MAAM,MAAM,GAAG,CAAC,gBAAgB,EAAE,KAAK,EAAE,gBAAgB,EAAE,UAAU,EAAE,QAAQ,EAAE,YAAY,EAAE,OAAO,CAAC,CAAC;IACxG,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;WACpD,KAAK,CAAC,cAAc,KAAK,GAAG,IAAI,OAAO,KAAK,CAAC,GAAG,KAAK,QAAQ;WAC7D,CAAC,uBAAuB,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;WAC3D,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC;WACnE,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;QACxD,IAAI,CAAC,6BAA6B,CAAC,CAAC;IACtC,CAAC;IACD,sBAAsB,CAAC,KAAK,CAAC,GAAa,CAAC,CAAC;IAC5C,MAAM,aAAa,GAAG,KAAK,CAAC,UAAU,KAAK,SAAS,CAAC;IACrD,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC;IAC3C,IAAI,CAAC,aAAa,IAAI,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,OAAO,IAAI,CAAC,aAAa,IAAI,QAAQ,CAAC,CAAC,EAAE,CAAC;QAC7F,IAAI,CAAC,0BAA0B,CAAC,CAAC;IACnC,CAAC;IACD,IAAI,aAAa,EAAE,CAAC;QAClB,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QAC5C,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QAC3C,MAAM,gBAAgB,GAAG,CAAC,gBAAgB,EAAE,SAAS,EAAE,WAAW,EAAE,iBAAiB,CAAC,CAAC;QACvF,IAAI,OAAO,CAAC,OAAO,KAAK,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,gBAAgB,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;YACzG,IAAI,CAAC,gEAAgE,CAAC,CAAC;QACzE,CAAC;IACH,CAAC;IACD,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAClC,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACjC,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;eAClE,CAAC,sBAAsB,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;YACzD,IAAI,CAAC,6CAA6C,CAAC,CAAC;QACtD,CAAC;QACD,cAAc,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;IACxC,CAAC;IACD,OAAO,KAAwC,CAAC;AAClD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,IAAY;IACtD,MAAM,OAAO,GAAG,uBAAuB,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;IAC9D,IAAI,OAAO,CAAC,UAAU,KAAK,SAAS,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QACpE,IAAI,CAAC,0CAA0C,CAAC,CAAC;IACnD,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,IAAY,EAAE,OAAe,EAAE,KAA4B;IACvG,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC5C,IAAI,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,SAAS,EAAE,CAAC;QAC9C,IAAI,CAAC,+CAA+C,EAAE,uBAAuB,CAAC,CAAC;IACjF,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAuB,IAAI,EAAE,IAAI,CAAC,CAAC;IAChE,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,CAAC,MAAM,IAAI,MAAM,CAAC,cAAc,KAAK,KAAK,CAAC,aAAa;WAC1F,MAAM,CAAC,QAAQ,KAAK,KAAK,CAAC,QAAQ,IAAI,MAAM,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;QACrE,IAAI,CAAC,wCAAwC,EAAE,oBAAoB,CAAC,CAAC;IACvE,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB,CAAC,EAAU,EAAE,KAAkB,EAAE,eAAuB;IAC/E,OAAO,sBAAsB,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,uBAAuB,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;WAC1F,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,QAAQ,IAAI,CAAC,IAAI,KAAK,CAAC,QAAQ,IAAI,eAAe;WAChG,CAAC,KAAK,CAAC,YAAY,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,YAAY,CAAC;eAC5E,KAAK,CAAC,YAAY,IAAI,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,YAAY,IAAI,eAAe,CAAC,CAAC,CAAC;AACzF,CAAC;AAED,SAAS,qBAAqB,CAAC,MAAqB,EAAE,OAAe,EAAE,aAAqB;IAC1F,IAAI,MAAM,CAAC,cAAc,KAAK,GAAG,IAAI,MAAM,CAAC,QAAQ,KAAK,OAAO,IAAI,MAAM,CAAC,cAAc,KAAK,aAAa;WACtG,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,MAAM,CAAC,QAAQ,GAAG,CAAC;WAC7D,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC;WAC7F,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,gBAAgB,CAAC,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;QACxG,IAAI,CAAC,+BAA+B,EAAE,uBAAuB,CAAC,CAAC;IACjE,CAAC;AACH,CAAC;AAED,gFAAgF;AAChF,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,IAAY,EAAE,KAAc,EAAE,KAAoB,EAAE,MAA0B;IAE9E,MAAM,OAAO,GAAG,uBAAuB,CAAC,KAAK,CAAC,CAAC;IAC/C,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACzC,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC5C,OAAO,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,kBAAkB,CAAC,EAAE,OAAO,EAAE,KAAK,IAAI,EAAE;QACpF,IAAI,MAAM,GAAG,MAAM,QAAQ,CAAuB,IAAI,EAAE,IAAI,CAAC,CAAC;QAC9D,IAAI,KAAK,GAAG,MAAM,QAAQ,CACxB,oBAAoB,CAAC,IAAI,EAAE,OAAO,CAAC,EAAE,IAAI,CAC1C,CAAC;QACF,MAAM,MAAM,GAAG,cAAc,CAAC,MAAM,EAAE,OAAO,IAAI,KAAK,EAAE,OAAO,IAAI,SAAS,EAAE,CAAC,CAAC;QAChF,OAAO,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,wBAAwB,CAAC,EAAE,MAAM,EAAE,KAAK,IAAI,EAAE;YACzF,IAAI,MAAM,EAAE,CAAC;gBACX,qBAAqB,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,cAAc,CAAC,CAAC;gBAC/D,IAAI,KAAK,IAAI,KAAK,CAAC,OAAO,KAAK,MAAM,EAAE,CAAC;oBACtC,IAAI,CAAC,+BAA+B,EAAE,uBAAuB,CAAC,CAAC;gBACjE,CAAC;gBACD,IAAI,OAAO,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC;oBACvC,IAAI,CAAC,0CAA0C,EAAE,oBAAoB,CAAC,CAAC;gBACzE,CAAC;gBACD,IAAI,OAAO,CAAC,QAAQ,KAAK,MAAM,CAAC,QAAQ,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC;oBAC7E,IAAI,CAAC,6CAA6C,EAAE,uBAAuB,CAAC,CAAC;gBAC/E,CAAC;gBACD,8EAA8E;gBAC9E,IAAI,MAAM,CAAC,MAAM,KAAK,OAAO,IAAI,OAAO,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;oBAC5D,MAAM,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;gBACzD,CAAC;YACH,CAAC;YACD,MAAM,GAAG;gBACP,cAAc,EAAE,GAAG,EAAE,QAAQ,EAAE,OAAO,EAAE,cAAc,EAAE,OAAO,CAAC,cAAc,EAAE,OAAO,EAAE,MAAM;gBAC/F,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,IAAI,EAAE;aACjF,CAAC;YACF,MAAM,eAAe,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YACpC,MAAM,eAAe,CAAC,WAAW,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE,EAAE,cAAc,EAAE,GAAG,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;YAC9G,IAAI,OAAO,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;gBACrC,MAAM,SAAS,GAAG,MAAM,CAAC,cAAc,CAAC,CAAC;gBACzC,WAAW,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;gBACjC,IAAI,CAAC;oBACH,MAAM,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,UAAoD,EAAE;wBAC/E,cAAc,EAAE,OAAO,CAAC,GAAG;wBAC3B,OAAO,EAAE,EAAE,MAAM,EAAE,aAAa,EAAE,OAAO,CAAC,cAAc,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,SAAS,EAAE;qBAClG,CAAC,CAAC;gBACL,CAAC;wBAAS,CAAC;oBACT,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBAC3B,CAAC;gBACD,KAAK,GAAG,MAAM,QAAQ,CAAC,oBAAoB,CAAC,IAAI,EAAE,OAAO,CAAC,EAAE,IAAI,CAAC,CAAC;YACpE,CAAC;YACD,IAAI,QAAQ,GAAS,EAAE,CAAC;YACxB,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;gBAClB,IAAI,CAAC,KAAK,IAAI,OAAO,CAAC,KAAK,CAAC,OAAO,KAAK,MAAM,EAAE,CAAC;oBAC/C,IAAI,CAAC,wCAAwC,EAAE,uBAAuB,CAAC,CAAC;gBAC1E,CAAC;gBACD,MAAM,EAAE,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;gBAChD,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBAC/C,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBAC9B,IAAI,GAAG,IAAI,GAAG,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;oBACjC,IAAI,CAAC,mDAAmD,EAAE,uBAAuB,CAAC,CAAC;gBACrF,CAAC;gBACD,IAAI,GAAG,EAAE,YAAY,KAAK,SAAS,EAAE,CAAC;oBACpC,IAAI,CAAC,mCAAmC,EAAE,sBAAsB,CAAC,CAAC;gBACpE,CAAC;gBACD,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC;gBAC7D,MAAM,eAAe,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;gBACpC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBAChE,QAAQ,GAAG,EAAE,QAAQ,EAAE,MAAM,CAAC,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC;YACxE,CAAC;YACD,IAAI,OAAO,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;gBAC/B,MAAM,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;YACzD,CAAC;YACD,IAAI,aAAa,GAAG,KAAK,CAAC;YAC1B,KAAK,MAAM,EAAE,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC5C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;gBAC/C,aAAa,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,WAAW,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;YAC5G,CAAC;YACD,OAAO;gBACL,cAAc,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,OAAO,CAAC,cAAc,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBACxG,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC,KAAK,EAAE,cAAc,EAAE,aAAa,EAAE,GAAG,QAAQ;aACvF,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC,EAAE,EAAE,WAAW,EAAE,oBAAoB,EAAE,eAAe,EAAE,EAAE,EAAE,CAAC,CAAC;AACjE,CAAC;AAED,KAAK,UAAU,UAAU,CACvB,IAAY,EAAE,MAAqB,EAAE,SAAkB,EAAE,KAAoB,EAAE,MAA0B;IAEzG,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC;QAAE,KAAK,CAAC,YAAY,KAAK,MAAM,CAAC,QAAQ,CAAC;IACzF,MAAM,eAAe,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACpC,IAAI,SAAS;QAAE,MAAM,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAClD,KAAK,MAAM,EAAE,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;QAAE,MAAM,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;AACvF,CAAC;AAED,kFAAkF;AAClF,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,IAAY,EAAE,WAAmB,EAAE,OAAgB;IAC5F,MAAM,MAAM,GAAG,cAAc,CAAC,WAAW,CAAC,CAAC;IAC3C,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,MAAM,kBAAkB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACpE,IAAI,CAAC,OAAO;QAAE,OAAO,KAAK,CAAC;IAC3B,IAAI,CAAC,uBAAuB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC3C,IAAI,CAAC,6BAA6B,EAAE,uBAAuB,CAAC,CAAC;IAC/D,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAuB,eAAe,CAAC,IAAI,EAAE,OAAO,CAAC,EAAE,IAAI,CAAC,CAAC;IAC1F,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,IAAI,OAAO;YAAE,IAAI,CAAC,gCAAgC,EAAE,uBAAuB,CAAC,CAAC;QAC7E,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IAAI,MAAM,CAAC,cAAc,KAAK,GAAG,IAAI,MAAM,CAAC,QAAQ,KAAK,OAAO,IAAI,MAAM,CAAC,OAAO,KAAK,MAAM;WACxF,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC;WAC3C,CAAC,MAAM,CAAC,MAAM,IAAI,OAAO,MAAM,CAAC,MAAM,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;QACzF,IAAI,CAAC,wCAAwC,EAAE,uBAAuB,CAAC,CAAC;IAC1E,CAAC;IACD,IAAI,CAAC,OAAO;QAAE,OAAO,MAAM,CAAC,MAAM,KAAK,OAAO,CAAC;IAC/C,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACrC,OAAO,CAAC,CAAC,KAAK,IAAI,CAAC,MAAM,CAAC,MAAM,KAAK,OAAO,IAAI,KAAK,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC;AACpF,CAAC;AAED,iFAAiF;AACjF,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAI,IAAY,EAAE,WAAmB,EAAE,MAAwB;IACzG,MAAM,MAAM,GAAG,cAAc,CAAC,WAAW,CAAC,CAAC;IAC3C,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,kBAAkB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC3D,MAAM,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,wBAAwB,CAAC,EAAE,MAAM,EAAE,KAAK,IAAI,EAAE;QACvG,oFAAoF;QACpF,IAAI,MAAM,QAAQ,CAAC,WAAW,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE,IAAI,CAAC;eAC9C,CAAC,OAAO,IAAI,MAAM,QAAQ,CAAC,eAAe,CAAC,IAAI,EAAE,OAAO,CAAC,EAAE,IAAI,CAAC,CAAC,EAAE,CAAC;YACvE,IAAI,CAAC,+BAA+B,EAAE,uBAAuB,CAAC,CAAC;QACjE,CAAC;QACD,OAAO,MAAM,EAAE,CAAC;IAClB,CAAC,CAAC,CAAC;IACH,OAAO,OAAO,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,kBAAkB,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;AAC1G,CAAC"}

package/dist/src/control-plane/remote-backend-result.js

@@ -0,0 +1,18 @@
+import path from "node:path";
+export function remoteBackendResult(offered, terminal, trial, backendDirectory, version = null, verifier) {
+    const directory = backendDirectory ?? path.join("remote", offered.work.work_id);
+    const succeeded = terminal.terminal?.status === "succeeded" && trial !== null;
+    return {
+        backend: {
+            name: "harbor", executable: `remote-worker:${offered.worker_id}`, version,
+            identity: `${offered.lease.provider}:${offered.worker_id}:${offered.generation}`,
+            config_path: path.join(directory, verifier ? "assessment.json" : "remote-offer.json"), result_path: succeeded ? path.join(directory, verifier ? "evidence/trial.json" : "remote-result.json") : null,
+            stdout_path: path.join(directory, verifier ? "evidence/verifier/test-stdout.txt" : "remote.stdout.log"), stderr_path: path.join(directory, verifier ? "evidence/verifier/test-stderr.txt" : "remote.stderr.log"),
+            process_exit_code: verifier ? verifier.backend.process_exit_code : succeeded ? 0 : 1, signal: (verifier?.backend.signal ?? null),
+            job_directory: path.join(directory, verifier ? "evidence" : "job"),
+        },
+        rawResult: trial ? { trial_results: [trial] } : null,
+        summary: trial ? { n_trials: 1, remote_worker: offered.worker_id } : null,
+    };
+}
+//# sourceMappingURL=remote-backend-result.js.map

package/dist/src/control-plane/remote-backend-result.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"remote-backend-result.js","sourceRoot":"","sources":["../../../src/control-plane/remote-backend-result.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAG7B,MAAM,UAAU,mBAAmB,CAAC,OAA0B,EAAE,QAA2B,EAAE,KAAqC,EAAE,gBAA+B,EAAE,UAAyB,IAAI,EAChM,QAA+D;IAC/D,MAAM,SAAS,GAAG,gBAAgB,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAChF,MAAM,SAAS,GAAG,QAAQ,CAAC,QAAQ,EAAE,MAAM,KAAK,WAAW,IAAI,KAAK,KAAK,IAAI,CAAC;IAC9E,OAAO;QACL,OAAO,EAAE;YACP,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,iBAAiB,OAAO,CAAC,SAAS,EAAE,EAAE,OAAO;YACzE,QAAQ,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,QAAQ,IAAI,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,UAAU,EAAE;YAChF,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,mBAAmB,CAAC,EAAE,WAAW,EAAE,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,IAAI;YACpM,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAC,mCAAmC,CAAC,CAAC,CAAC,mBAAmB,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAC,mCAAmC,CAAC,CAAC,CAAC,mBAAmB,CAAC;YAChN,iBAAiB,EAAE,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,QAAQ,EAAE,OAAO,CAAC,MAAM,IAAI,IAAI,CAA0B;YACzJ,aAAa,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC;SACnE;QACD,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI;QACpD,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,aAAa,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,IAAI;KAC1E,CAAC;AACJ,CAAC"}

package/dist/src/control-plane/remote-execution-lease.js

@@ -0,0 +1,15 @@
+import path from "node:path";
+import { parseExecutionLease } from "../evals/index.js";
+import { HitchError, readJSON, sha256JSON, statePaths } from "../foundation/index.js";
+/** A frozen offer is not a renewal. Read the current controller lease without changing its epoch or expiry. */
+export async function readRemoteExecutionLease(root, offer) {
+    const lease = parseExecutionLease(await readJSON(path.join(statePaths(root).evals, offer.lease.eval_id, "leases", `${offer.lease.lease_id}.json`)));
+    const fields = ["lease_id", "eval_id", "work_id", "worker_id", "provider", "collision_domain_id", "epoch", "reservation"];
+    if (offer.state !== "accepted" || !["offered", "accepted", "running"].includes(lease.state) || Date.parse(lease.expires_at) <= Date.now()
+        || fields.some(field => sha256JSON(lease[field]) !== sha256JSON(offer.lease[field]))
+        || sha256JSON(lease.resource_epochs ?? [lease.epoch]) !== sha256JSON(offer.lease.resource_epochs ?? [offer.lease.epoch])) {
+        throw new HitchError("remote execution lease is cancelled, expired or no longer active", { code: "remote_execution_lease_fenced", exitCode: 12 });
+    }
+    return lease;
+}
+//# sourceMappingURL=remote-execution-lease.js.map

package/dist/src/control-plane/remote-execution-lease.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"remote-execution-lease.js","sourceRoot":"","sources":["../../../src/control-plane/remote-execution-lease.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAEtF,+GAA+G;AAC/G,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAAC,IAAY,EAAE,KAAwB;IACnF,MAAM,KAAK,GAAG,mBAAmB,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,QAAQ,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC,QAAQ,OAAO,CAAC,CAAC,CAAC,CAAC;IACpJ,MAAM,MAAM,GAAG,CAAC,UAAU,EAAE,SAAS,EAAE,SAAS,EAAE,WAAW,EAAE,UAAU,EAAE,qBAAqB,EAAE,OAAO,EAAE,aAAa,CAAU,CAAC;IACnI,IAAI,KAAK,CAAC,KAAK,KAAK,UAAU,IAAI,CAAC,CAAC,SAAS,EAAE,UAAU,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE;WACpI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,KAAK,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;WACjF,UAAU,CAAC,KAAK,CAAC,eAAe,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,KAAK,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,eAAe,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;QAC3H,MAAM,IAAI,UAAU,CAAC,kEAAkE,EAAE,EAAE,IAAI,EAAE,+BAA+B,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;IACpJ,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}

package/dist/src/control-plane/remote-execution-observation.js

@@ -0,0 +1,108 @@
+import { randomBytes } from "node:crypto";
+import { HitchError, sha256JSON } from "../foundation/index.js";
+import { parseExecutionObservationReceipt } from "./execution-observation-contract.js";
+const problem = (code, message) => new HitchError(message, { code, exitCode: 12 });
+/** Read-only challenges live for one daemon request. Restart/timeout never reuses cached observations. */
+export class RemoteExecutionObservationCoordinator {
+    input;
+    pending = new Map();
+    closed = false;
+    timeoutMs;
+    constructor(input) {
+        this.input = input;
+        this.timeoutMs = input.timeoutMs ?? 35_000;
+        if (!/^[a-f0-9]{32}$/.test(input.instanceId) || !Number.isSafeInteger(this.timeoutMs) || this.timeoutMs < 20 || this.timeoutMs > 40_000)
+            throw new TypeError("invalid observation coordinator options");
+    }
+    async observe(provider, nonce, signal) {
+        signal.throwIfAborted();
+        if (this.closed)
+            throw problem("execution_observation_closed", "daemon observation service is closed");
+        if (!/^[a-f0-9]{32}$/.test(nonce))
+            throw problem("invalid_input", "invalid observation nonce");
+        const matches = (await this.input.registry.list()).filter(record => record.worker.provider === provider);
+        if (matches.length !== 1)
+            throw problem("execution_observation_unsupported", "select exactly one registered worker for provider observation");
+        const worker = matches[0];
+        this.assertAvailable(worker);
+        const workerId = worker.worker.worker_id;
+        if (this.pending.has(workerId))
+            throw problem("execution_observation_busy", "a fresh observation for this worker is already pending; retry");
+        const challenge = { schema_version: "2", request_id: randomBytes(16).toString("hex"), nonce,
+            daemon_instance_id: this.input.instanceId, worker_id: workerId, generation: worker.generation, provider,
+            collision_domain_id: worker.worker.collision_domain_id, expires_at: new Date(Date.now() + this.timeoutMs).toISOString() };
+        const receipt = await new Promise((resolve, reject) => {
+            const finish = (result, error) => {
+                if (this.pending.get(workerId) !== entry)
+                    return;
+                this.pending.delete(workerId);
+                clearTimeout(timer);
+                signal.removeEventListener("abort", abort);
+                if (error)
+                    reject(error);
+                else
+                    resolve(result);
+            };
+            const abort = () => finish(undefined, problem("execution_observation_cancelled", "observation requester disconnected"));
+            const timer = setTimeout(() => finish(undefined, problem("execution_observation_timeout", "worker did not return a fresh runtime/environment observation before its deadline")), this.timeoutMs);
+            const entry = { challenge, complete: result => finish(result), fail: error => finish(undefined, error) };
+            this.pending.set(workerId, entry);
+            signal.addEventListener("abort", abort, { once: true });
+            if (signal.aborted)
+                abort();
+            else if (this.closed)
+                entry.fail(problem("execution_observation_closed", "daemon observation service is closed"));
+        });
+        await this.assertGeneration(challenge);
+        return receipt;
+    }
+    async poll(workerId, generation) {
+        await this.input.registry.validateHeartbeatGeneration(workerId, generation);
+        const pending = this.pending.get(workerId);
+        if (!pending)
+            return null;
+        if (pending.challenge.generation !== generation) {
+            pending.fail(problem("worker_generation_mismatch", "worker generation changed during environment observation"));
+            return null;
+        }
+        if (Date.parse(pending.challenge.expires_at) <= Date.now()) {
+            pending.fail(problem("execution_observation_timeout", "worker observation deadline expired"));
+            return null;
+        }
+        return structuredClone(pending.challenge);
+    }
+    async submit(workerId, value) {
+        const receipt = parseExecutionObservationReceipt(value), challenge = receipt.challenge;
+        if (challenge.worker_id !== workerId)
+            throw problem("execution_observation_mismatch", "observation belongs to another worker");
+        await this.assertGeneration(challenge);
+        const pending = this.pending.get(workerId);
+        if (!pending || sha256JSON(pending.challenge) !== sha256JSON(challenge)
+            || challenge.daemon_instance_id !== this.input.instanceId || Date.parse(challenge.expires_at) <= Date.now()) {
+            throw problem("execution_observation_mismatch", "observation challenge is stale, expired or does not match this daemon request");
+        }
+        pending.complete(receipt);
+    }
+    close() {
+        this.closed = true;
+        for (const entry of this.pending.values())
+            entry.fail(problem("execution_observation_closed", "daemon stopped before the worker observation completed"));
+    }
+    async assertGeneration(challenge) {
+        await this.input.registry.validateHeartbeatGeneration(challenge.worker_id, challenge.generation);
+        const current = await this.input.registry.get(challenge.worker_id);
+        if (!current || current.generation !== challenge.generation || current.worker.provider !== challenge.provider
+            || current.worker.collision_domain_id !== challenge.collision_domain_id)
+            throw problem("execution_observation_mismatch", "worker observation identity changed");
+        this.assertAvailable(current);
+    }
+    assertAvailable(worker) {
+        if (worker.revoked_at || worker.worker.status !== "ready" || worker.provider_status.health !== "healthy") {
+            throw problem("execution_provider_unavailable", "worker must be healthy and online for environment observation");
+        }
+        if (worker.provider_status.features.execution_observation !== "2") {
+            throw problem("execution_observation_unsupported", "worker must advertise execution_observation 2 and run a supporting Hitch binary");
+        }
+    }
+}
+//# sourceMappingURL=remote-execution-observation.js.map

package/dist/src/control-plane/remote-execution-observation.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"remote-execution-observation.js","sourceRoot":"","sources":["../../../src/control-plane/remote-execution-observation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAE1C,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAEhE,OAAO,EAAE,gCAAgC,EAAE,MAAM,qCAAqC,CAAC;AAOvF,MAAM,OAAO,GAAG,CAAC,IAAY,EAAE,OAAe,EAAE,EAAE,CAAC,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;AAEnG,0GAA0G;AAC1G,MAAM,OAAO,qCAAqC;IAInB;IAHZ,OAAO,GAAG,IAAI,GAAG,EAA8B,CAAC;IACzD,MAAM,GAAG,KAAK,CAAC;IACN,SAAS,CAAS;IACnC,YAA6B,KAAiF;QAAjF,UAAK,GAAL,KAAK,CAA4E;QAC5G,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC,SAAS,IAAI,MAAM,CAAC;QAC3C,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,SAAS,GAAG,EAAE,IAAI,IAAI,CAAC,SAAS,GAAG,MAAM;YAAE,MAAM,IAAI,SAAS,CAAC,yCAAyC,CAAC,CAAC;IAC1M,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,QAAgB,EAAE,KAAa,EAAE,MAAmB;QAChE,MAAM,CAAC,cAAc,EAAE,CAAC;QACxB,IAAI,IAAI,CAAC,MAAM;YAAE,MAAM,OAAO,CAAC,8BAA8B,EAAE,sCAAsC,CAAC,CAAC;QACvG,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC;YAAE,MAAM,OAAO,CAAC,eAAe,EAAE,2BAA2B,CAAC,CAAC;QAC/F,MAAM,OAAO,GAAG,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC;QACzG,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM,OAAO,CAAC,mCAAmC,EAAE,+DAA+D,CAAC,CAAC;QAC9I,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAE,CAAC;QAC3B,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;QAC7B,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC;QACzC,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;YAAE,MAAM,OAAO,CAAC,4BAA4B,EAAE,+DAA+D,CAAC,CAAC;QAC7I,MAAM,SAAS,GAA0C,EAAE,cAAc,EAAE,GAAG,EAAE,UAAU,EAAE,WAAW,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,KAAK;YAChI,kBAAkB,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,QAAQ;YACvG,mBAAmB,EAAE,MAAM,CAAC,MAAM,CAAC,mBAAmB,EAAE,UAAU,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC;QAC5H,MAAM,OAAO,GAAG,MAAM,IAAI,OAAO,CAAsC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACzF,MAAM,MAAM,GAAG,CAAC,MAA4C,EAAE,KAAa,EAAE,EAAE;gBAC7E,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,KAAK;oBAAE,OAAO;gBACjD,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;gBAAC,YAAY,CAAC,KAAK,CAAC,CAAC;gBAAC,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;gBAC/F,IAAI,KAAK;oBAAE,MAAM,CAAC,KAAK,CAAC,CAAC;;oBAAM,OAAO,CAAC,MAAO,CAAC,CAAC;YAClD,CAAC,CAAC;YACF,MAAM,KAAK,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,SAAS,EAAE,OAAO,CAAC,iCAAiC,EAAE,oCAAoC,CAAC,CAAC,CAAC;YACxH,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,SAAS,EAAE,OAAO,CAAC,+BAA+B,EAAE,mFAAmF,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;YACjM,MAAM,KAAK,GAAuB,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE,CAAC;YAC7H,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;YAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YAC3F,IAAI,MAAM,CAAC,OAAO;gBAAE,KAAK,EAAE,CAAC;iBACvB,IAAI,IAAI,CAAC,MAAM;gBAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,8BAA8B,EAAE,sCAAsC,CAAC,CAAC,CAAC;QACpH,CAAC,CAAC,CAAC;QACH,MAAM,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;QACvC,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,QAAgB,EAAE,UAAkB;QAC7C,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,2BAA2B,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QAC5E,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC3C,IAAI,CAAC,OAAO;YAAE,OAAO,IAAI,CAAC;QAC1B,IAAI,OAAO,CAAC,SAAS,CAAC,UAAU,KAAK,UAAU,EAAE,CAAC;YAChD,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,4BAA4B,EAAE,0DAA0D,CAAC,CAAC,CAAC;YAChH,OAAO,IAAI,CAAC;QACd,CAAC;QACD,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;YAC3D,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,+BAA+B,EAAE,qCAAqC,CAAC,CAAC,CAAC;YAC9F,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO,eAAe,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAC5C,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,QAAgB,EAAE,KAAc;QAC3C,MAAM,OAAO,GAAG,gCAAgC,CAAC,KAAK,CAAC,EAAE,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QACvF,IAAI,SAAS,CAAC,SAAS,KAAK,QAAQ;YAAE,MAAM,OAAO,CAAC,gCAAgC,EAAE,uCAAuC,CAAC,CAAC;QAC/H,MAAM,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;QACvC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC3C,IAAI,CAAC,OAAO,IAAI,UAAU,CAAC,OAAO,CAAC,SAAS,CAAC,KAAK,UAAU,CAAC,SAAS,CAAC;eAClE,SAAS,CAAC,kBAAkB,KAAK,IAAI,CAAC,KAAK,CAAC,UAAU,IAAI,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;YAC9G,MAAM,OAAO,CAAC,gCAAgC,EAAE,+EAA+E,CAAC,CAAC;QACnI,CAAC;QACD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IAC5B,CAAC;IAED,KAAK;QACH,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;YAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,8BAA8B,EAAE,wDAAwD,CAAC,CAAC,CAAC;IAC3J,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAAC,SAAgD;QAC7E,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,2BAA2B,CAAC,SAAS,CAAC,SAAS,EAAE,SAAS,CAAC,UAAU,CAAC,CAAC;QACjG,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;QACnE,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,KAAK,SAAS,CAAC,UAAU,IAAI,OAAO,CAAC,MAAM,CAAC,QAAQ,KAAK,SAAS,CAAC,QAAQ;eACxG,OAAO,CAAC,MAAM,CAAC,mBAAmB,KAAK,SAAS,CAAC,mBAAmB;YAAE,MAAM,OAAO,CAAC,gCAAgC,EAAE,qCAAqC,CAAC,CAAC;QAClK,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;IAChC,CAAC;IACO,eAAe,CAAC,MAAkC;QACxD,IAAI,MAAM,CAAC,UAAU,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,KAAK,OAAO,IAAI,MAAM,CAAC,eAAe,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YACzG,MAAM,OAAO,CAAC,gCAAgC,EAAE,+DAA+D,CAAC,CAAC;QACnH,CAAC;QACD,IAAI,MAAM,CAAC,eAAe,CAAC,QAAQ,CAAC,qBAAqB,KAAK,GAAG,EAAE,CAAC;YAClE,MAAM,OAAO,CAAC,mCAAmC,EAAE,iFAAiF,CAAC,CAAC;QACxI,CAAC;IACH,CAAC;CACF"}

package/dist/src/control-plane/remote-model-routes.js

@@ -0,0 +1,107 @@
+import path from "node:path";
+import { HitchError, atomicWriteJSON, readJSON, sha256JSON, statePaths, withFileLock } from "../foundation/index.js";
+import { callRemoteModel, remoteModelBinding } from "../model-access/index.js";
+import { parseExecutionLease } from "../evals/index.js";
+export { parseRemoteModelBinding, remoteModelBinding } from "../model-access/index.js";
+/** Private controller state, never part of an input artifact or worker credential envelope. */
+export class RemoteModelRoutes {
+    root;
+    managedReady;
+    constructor(root) {
+        this.root = root;
+    }
+    deferManagedRecovery() {
+        if (this.managedReady)
+            throw new TypeError("managed model recovery is already pending");
+        let resolve;
+        const promise = new Promise(done => { resolve = done; });
+        this.managedReady = { promise, resolve };
+    }
+    finishManagedRecovery() { this.managedReady?.resolve(); this.managedReady = undefined; }
+    async authorize(offer) {
+        const lease = parseExecutionLease(await readJSON(path.join(statePaths(this.root).evals, offer.lease.eval_id, "leases", `${offer.lease.lease_id}.json`)));
+        if (offer.state !== "accepted" || !["offered", "accepted", "running"].includes(lease.state)
+            || lease.worker_id !== offer.worker_id || lease.epoch !== offer.lease.epoch || lease.work_id !== offer.work.work_id
+            || lease.eval_id !== offer.lease.eval_id || lease.provider !== offer.lease.provider
+            || lease.collision_domain_id !== offer.lease.collision_domain_id || Date.parse(lease.expires_at) <= Date.now()) {
+            throw fenced("remote model lease is cancelled, expired or no longer active");
+        }
+        return offer;
+    }
+    directory(offer) {
+        return path.join(statePaths(this.root).workerProtocol, "model-routes", offer.lease.lease_id);
+    }
+    identity(offer) {
+        return { worker_id: offer.worker_id, generation: offer.generation, offer_id: offer.offer_id,
+            lease_id: offer.lease.lease_id, epoch: offer.lease.epoch, work_id: offer.work.work_id, eval_id: offer.lease.eval_id };
+    }
+    async prepare(offer, target) {
+        const value = { schema_version: "2", identity: this.identity(offer), model_binding: remoteModelBinding(target), target };
+        await withFileLock(path.join(this.directory(offer), "locks"), "route", async () => {
+            const file = path.join(this.directory(offer), "route.json");
+            const old = await readJSON(file, null);
+            if (old && sha256JSON(old) !== sha256JSON(value))
+                throw fenced("remote model route identity changed");
+            if (!old)
+                await atomicWriteJSON(file, value);
+        });
+    }
+    async route(offer) {
+        const value = await readJSON(path.join(this.directory(offer), "route.json"));
+        if (sha256JSON(value.identity) !== sha256JSON(this.identity(offer))
+            || sha256JSON(value.model_binding) !== sha256JSON(remoteModelBinding(value.target)))
+            throw fenced("remote model route differs from its worker lease");
+        return value;
+    }
+    async recoveryTarget(offer) {
+        if (!await readJSON(path.join(this.directory(offer), "route.json"), null))
+            return null;
+        return structuredClone((await this.route(offer)).target);
+    }
+    async boundRun(offer) {
+        const route = await this.route(offer);
+        const bound = await readJSON(path.join(this.directory(offer), "run.json"), null);
+        if (!bound || !/^run_[a-f0-9]{32}$/.test(bound.run_id) || bound.model_binding_digest !== sha256JSON(route.model_binding))
+            throw fenced("remote model has no matching canonical run binding");
+        const ack = await readJSON(path.join(this.directory(offer), "bound.json"), null);
+        if (!ack || sha256JSON(ack) !== sha256JSON(bound))
+            throw fenced("remote model run binding has not been acknowledged");
+        return { runId: bound.run_id, binding: remoteModelBinding(route.target) };
+    }
+    async call(offer, runId, operation, body, signal) {
+        if (!/^run_[a-f0-9]{32}$/.test(runId))
+            throw fenced("invalid remote canonical run ID");
+        const route = await this.route(offer);
+        if (route.target.kind === "managed-inference" && this.managedReady) {
+            await waitForRecovery(this.managedReady.promise, signal);
+            await this.authorize(offer);
+        }
+        await withFileLock(path.join(this.directory(offer), "locks"), "binding", async () => {
+            const file = path.join(this.directory(offer), "run.json"), next = { run_id: runId, model_binding_digest: sha256JSON(route.model_binding) };
+            const old = await readJSON(file, null);
+            if (old && sha256JSON(old) !== sha256JSON(next) || !old && operation !== "bind")
+                throw fenced("remote lease cannot change its canonical run or generate before binding");
+            if (!old)
+                await atomicWriteJSON(file, next);
+        });
+        if (operation === "generate")
+            await this.boundRun(offer);
+        const result = await callRemoteModel(route.target, runId, operation, body, signal);
+        if (operation === "bind") {
+            signal.throwIfAborted();
+            await atomicWriteJSON(path.join(this.directory(offer), "bound.json"), { run_id: runId, model_binding_digest: sha256JSON(route.model_binding) });
+        }
+        return result;
+    }
+}
+async function waitForRecovery(ready, signal) {
+    signal.throwIfAborted();
+    await new Promise((resolve, reject) => {
+        const abort = () => reject(signal.reason);
+        signal.addEventListener("abort", abort, { once: true });
+        ready.then(resolve, reject).finally(() => signal.removeEventListener("abort", abort)).catch(() => { });
+    });
+    signal.throwIfAborted();
+}
+function fenced(message) { return new HitchError(message, { code: "remote_model_fenced", exitCode: 12 }); }
+//# sourceMappingURL=remote-model-routes.js.map

package/dist/src/control-plane/remote-model-routes.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"remote-model-routes.js","sourceRoot":"","sources":["../../../src/control-plane/remote-model-routes.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,UAAU,EAAE,eAAe,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACrH,OAAO,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAE/E,OAAO,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AAExD,OAAO,EAAE,uBAAuB,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAGvF,+FAA+F;AAC/F,MAAM,OAAO,iBAAiB;IAEC;IADrB,YAAY,CAA0D;IAC9E,YAA6B,IAAY;QAAZ,SAAI,GAAJ,IAAI,CAAQ;IAAG,CAAC;IAC7C,oBAAoB;QAClB,IAAI,IAAI,CAAC,YAAY;YAAE,MAAM,IAAI,SAAS,CAAC,2CAA2C,CAAC,CAAC;QACxF,IAAI,OAAoB,CAAC;QACzB,MAAM,OAAO,GAAG,IAAI,OAAO,CAAO,IAAI,CAAC,EAAE,GAAG,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/D,IAAI,CAAC,YAAY,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;IAC3C,CAAC;IACD,qBAAqB,KAAW,IAAI,CAAC,YAAY,EAAE,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,YAAY,GAAG,SAAS,CAAC,CAAC,CAAC;IAC9F,KAAK,CAAC,SAAS,CAAC,KAAwB;QACtC,MAAM,KAAK,GAAG,mBAAmB,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,QAAQ,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC,QAAQ,OAAO,CAAC,CAAC,CAAC,CAAC;QACzJ,IAAI,KAAK,CAAC,KAAK,KAAK,UAAU,IAAI,CAAC,CAAC,SAAS,EAAE,UAAU,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC;eACtF,KAAK,CAAC,SAAS,KAAK,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,KAAK,KAAK,KAAK,CAAC,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,OAAO,KAAK,KAAK,CAAC,IAAI,CAAC,OAAO;eAChH,KAAK,CAAC,OAAO,KAAK,KAAK,CAAC,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,QAAQ,KAAK,KAAK,CAAC,KAAK,CAAC,QAAQ;eAChF,KAAK,CAAC,mBAAmB,KAAK,KAAK,CAAC,KAAK,CAAC,mBAAmB,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;YACjH,MAAM,MAAM,CAAC,8DAA8D,CAAC,CAAC;QAC/E,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IACO,SAAS,CAAC,KAAwB;QACxC,OAAO,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,cAAc,EAAE,cAAc,EAAE,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC/F,CAAC;IACO,QAAQ,CAAC,KAAwB;QACvC,OAAO,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,UAAU,EAAE,KAAK,CAAC,UAAU,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACzF,QAAQ,EAAE,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;IAC1H,CAAC;IACD,KAAK,CAAC,OAAO,CAAC,KAAwB,EAAE,MAA2B;QACjE,MAAM,KAAK,GAAG,EAAE,cAAc,EAAE,GAAG,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,aAAa,EAAE,kBAAkB,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;QACzH,MAAM,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC,EAAE,OAAO,EAAE,KAAK,IAAI,EAAE;YAChF,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,YAAY,CAAC,CAAC;YAC5D,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAiB,IAAI,EAAE,IAAI,CAAC,CAAC;YACvD,IAAI,GAAG,IAAI,UAAU,CAAC,GAAG,CAAC,KAAK,UAAU,CAAC,KAAK,CAAC;gBAAE,MAAM,MAAM,CAAC,qCAAqC,CAAC,CAAC;YACtG,IAAI,CAAC,GAAG;gBAAE,MAAM,eAAe,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC/C,CAAC,CAAC,CAAC;IACL,CAAC;IACO,KAAK,CAAC,KAAK,CAAC,KAAwB;QAC1C,MAAM,KAAK,GAAG,MAAM,QAAQ,CAA6E,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,YAAY,CAAC,CAAC,CAAC;QACzJ,IAAI,UAAU,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;eAC9D,UAAU,CAAC,KAAK,CAAC,aAAa,CAAC,KAAK,UAAU,CAAC,kBAAkB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YAAE,MAAM,MAAM,CAAC,kDAAkD,CAAC,CAAC;QACxJ,OAAO,KAAK,CAAC;IACf,CAAC;IACD,KAAK,CAAC,cAAc,CAAC,KAAwB;QAC3C,IAAI,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,YAAY,CAAC,EAAE,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC;QACvF,OAAO,eAAe,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAC3D,CAAC;IACD,KAAK,CAAC,QAAQ,CAAC,KAAwB;QACrC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACtC,MAAM,KAAK,GAAG,MAAM,QAAQ,CAA0D,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,UAAU,CAAC,EAAE,IAAI,CAAC,CAAC;QAC1I,IAAI,CAAC,KAAK,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,oBAAoB,KAAK,UAAU,CAAC,KAAK,CAAC,aAAa,CAAC;YAAE,MAAM,MAAM,CAAC,oDAAoD,CAAC,CAAC;QAC7L,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAiB,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,YAAY,CAAC,EAAE,IAAI,CAAC,CAAC;QACjG,IAAI,CAAC,GAAG,IAAI,UAAU,CAAC,GAAG,CAAC,KAAK,UAAU,CAAC,KAAK,CAAC;YAAE,MAAM,MAAM,CAAC,oDAAoD,CAAC,CAAC;QACtH,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,kBAAkB,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;IAC5E,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,KAAwB,EAAE,KAAa,EAAE,SAA8B,EAAE,IAAa,EAAE,MAAmB;QACpH,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,KAAK,CAAC;YAAE,MAAM,MAAM,CAAC,iCAAiC,CAAC,CAAC;QACvF,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACtC,IAAI,KAAK,CAAC,MAAM,CAAC,IAAI,KAAK,mBAAmB,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACnE,MAAM,eAAe,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YACzD,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAC9B,CAAC;QACD,MAAM,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC,EAAE,SAAS,EAAE,KAAK,IAAI,EAAE;YAClF,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,UAAU,CAAC,EAAE,IAAI,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,oBAAoB,EAAE,UAAU,CAAC,KAAK,CAAC,aAAa,CAAC,EAAE,CAAC;YAC3I,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAiB,IAAI,EAAE,IAAI,CAAC,CAAC;YACvD,IAAI,GAAG,IAAI,UAAU,CAAC,GAAG,CAAC,KAAK,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,SAAS,KAAK,MAAM;gBAAE,MAAM,MAAM,CAAC,yEAAyE,CAAC,CAAC;YACzK,IAAI,CAAC,GAAG;gBAAE,MAAM,eAAe,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAC9C,CAAC,CAAC,CAAC;QACH,IAAI,SAAS,KAAK,UAAU;YAAE,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACzD,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;QACnF,IAAI,SAAS,KAAK,MAAM,EAAE,CAAC;YACzB,MAAM,CAAC,cAAc,EAAE,CAAC;YACxB,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,YAAY,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,oBAAoB,EAAE,UAAU,CAAC,KAAK,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;QAClJ,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AACD,KAAK,UAAU,eAAe,CAAC,KAAoB,EAAE,MAAmB;IACtE,MAAM,CAAC,cAAc,EAAE,CAAC;IACxB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC1C,MAAM,KAAK,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC1C,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QACxD,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACxG,CAAC,CAAC,CAAC;IACH,MAAM,CAAC,cAAc,EAAE,CAAC;AAC1B,CAAC;AACD,SAAS,MAAM,CAAC,OAAe,IAAI,OAAO,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,qBAAqB,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC"}

package/dist/src/control-plane/remote-observation-reporter.js

@@ -0,0 +1,37 @@
+import { sha256JSON } from "../foundation/index.js";
+/** Version probes run outside the work/heartbeat loop and never acquire a task allocation. */
+export class RemoteObservationReporter {
+    client;
+    source;
+    onError;
+    stopController = new AbortController();
+    pending;
+    constructor(client, source, onError) {
+        this.client = client;
+        this.source = source;
+        this.onError = onError;
+    }
+    tick() {
+        if (this.pending || this.stopController.signal.aborted)
+            return;
+        this.pending = this.report().catch(error => {
+            if (!this.stopController.signal.aborted)
+                this.onError(error);
+        }).finally(() => { this.pending = undefined; });
+    }
+    async stop() { this.stopController.abort(); await this.pending; }
+    async report() {
+        const signal = AbortSignal.any([this.stopController.signal, AbortSignal.timeout(40_000)]);
+        const challenge = await this.client.pollExecutionObservation(signal);
+        if (!challenge)
+            return;
+        const remaining = Date.parse(challenge.expires_at) - Date.now();
+        if (remaining <= 0)
+            return;
+        const deadline = AbortSignal.any([signal, AbortSignal.timeout(Math.min(remaining, 35_000))]);
+        const { runtime, environment } = await this.source.observe(deadline);
+        deadline.throwIfAborted();
+        await this.client.submitExecutionObservation({ schema_version: "2", challenge, runtime, environment, environment_digest: sha256JSON(environment) }, deadline);
+    }
+}
+//# sourceMappingURL=remote-observation-reporter.js.map

package/dist/src/control-plane/remote-observation-reporter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"remote-observation-reporter.js","sourceRoot":"","sources":["../../../src/control-plane/remote-observation-reporter.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAGpD,8FAA8F;AAC9F,MAAM,OAAO,yBAAyB;IAGP;IAAiD;IAC3D;IAHF,cAAc,GAAG,IAAI,eAAe,EAAE,CAAC;IAChD,OAAO,CAA4B;IAC3C,YAA6B,MAA8B,EAAmB,MAAoC,EAC/F,OAAiC;QADvB,WAAM,GAAN,MAAM,CAAwB;QAAmB,WAAM,GAAN,MAAM,CAA8B;QAC/F,YAAO,GAAP,OAAO,CAA0B;IAAG,CAAC;IAExD,IAAI;QACF,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,OAAO;YAAE,OAAO;QAC/D,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;YACzC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,OAAO;gBAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAC/D,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,OAAO,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;IAClD,CAAC;IAED,KAAK,CAAC,IAAI,KAAoB,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;IAExE,KAAK,CAAC,MAAM;QAClB,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC1F,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,wBAAwB,CAAC,MAAM,CAAC,CAAC;QACrE,IAAI,CAAC,SAAS;YAAE,OAAO;QACvB,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAChE,IAAI,SAAS,IAAI,CAAC;YAAE,OAAO;QAC3B,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7F,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACrE,QAAQ,CAAC,cAAc,EAAE,CAAC;QAC1B,MAAM,IAAI,CAAC,MAAM,CAAC,0BAA0B,CAAC,EAAE,cAAc,EAAE,GAAG,EAAE,SAAS,EAAE,OAAO,EAAE,WAAW,EAAE,kBAAkB,EAAE,UAAU,CAAC,WAAW,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;IAChK,CAAC;CACF"}