agent-hitch 0.2.8 → 0.2.10

package/dist/src/control-plane/inference-manager.js

@@ -0,0 +1,357 @@
+import { randomUUID } from "node:crypto";
+import path from "node:path";
+import { HitchError, atomicWriteJSON, ensureDir, readJSON, sha256JSON, statePaths, withFileLock } from "../foundation/index.js";
+import { prepareLocalInference, SGLangServiceSupervisor } from "../inference/index.js";
+import { LocalModelGateway } from "../model-access/index.js";
+import { sealManagedGateway } from "./managed-gateway-receipt.js";
+export class LocalInferenceManager {
+    root;
+    resources;
+    supervisor;
+    preflight;
+    startGateway;
+    onEvent;
+    services = new Map();
+    serviceById = new Map();
+    gatewayPending = new Map();
+    resourceLeases = new Map();
+    recovery;
+    recovered = new Map();
+    recoveryPollIntervalMs;
+    recoveryTimer;
+    recoveryChecking;
+    closed = false;
+    constructor(options) {
+        this.root = options.root;
+        this.resources = options.resources;
+        this.onEvent = options.onEvent;
+        this.recovery = options.recovery;
+        this.recoveryPollIntervalMs = options.recoveryPollIntervalMs ?? 1_000;
+        if (!Number.isSafeInteger(this.recoveryPollIntervalMs) || this.recoveryPollIntervalMs < 1)
+            throw new TypeError("invalid inference recovery polling interval");
+        this.supervisor = options.supervisor ?? new SGLangServiceSupervisor({ root: options.root, ...(options.onEvent ? { onEvent: options.onEvent } : {}) });
+        this.preflight = options.preflight ?? prepareLocalInference;
+        this.startGateway = options.startGateway ?? LocalModelGateway.start;
+        this.supervisor.subscribeTerminal(async (record, released) => {
+            const service = this.serviceById.get(record.service_id);
+            if (service) {
+                service.invalidated = true;
+                await service.gateway.close();
+                if (released)
+                    await this.cleanupManagedService(service);
+            }
+            else if (released) {
+                const key = `${record.inference_id}:${record.isolation_key}`;
+                this.resourceLeases.get(key)?.release();
+                this.resourceLeases.delete(key);
+            }
+        });
+    }
+    async initialize() {
+        const plans = await this.recovery?.select(await this.supervisor.list()) ?? [];
+        const claims = new Map();
+        for (const plan of plans)
+            claims.set(plan.record.service_id, [...claims.get(plan.record.service_id) ?? [], plan.ownerId]);
+        const attached = await this.supervisor.recover([...claims].map(([service_id, owner_ids]) => ({ service_id, owner_ids })));
+        for (const restored of attached) {
+            const selected = plans.filter(plan => plan.record.service_id === restored.record.service_id);
+            if (!selected.length || selected.some(plan => plan.port !== selected[0].port || sha256JSON(plan.lock) !== sha256JSON(restored.lock))) {
+                throw new HitchError("recovered model service has conflicting gateway evidence", { code: "inference_recovery_ambiguous", exitCode: 12 });
+            }
+            const first = restored.owners.get(selected[0].ownerId);
+            const gateway = await this.startGateway({ upstreamBaseUrl: first.base_url, engineToken: first.engine_token, wireModel: first.wire_model,
+                lock: restored.lock, bindHost: "127.0.0.1", listenPort: selected[0].port });
+            const key = `${restored.record.inference_id}:${restored.record.isolation_key}`;
+            const managed = { key, serviceId: restored.record.service_id, gateway, refs: selected.length, lock: restored.lock };
+            this.services.set(key, managed);
+            this.serviceById.set(managed.serviceId, managed);
+            for (const plan of selected) {
+                const service = restored.owners.get(plan.ownerId);
+                let owner;
+                const registration = gateway.restore(plan.ownerId, plan.target.binding, plan.target.credential, async () => service.isReady() && (owner.claimed || await plan.state() === "active"));
+                let released = false;
+                const lease = { binding: registration.binding, credential: registration.credential, lock: restored.lock,
+                    service_id: service.service_id, service_epoch: service.epoch,
+                    release: async () => {
+                        if (released)
+                            return;
+                        registration.revoke();
+                        await service.release();
+                        managed.refs = Math.max(0, managed.refs - 1);
+                        released = true;
+                    } };
+                owner = { plan, service, registration, lease, claimed: false };
+                this.recovered.set(`${key}:${plan.ownerId}`, owner);
+            }
+            this.onEvent?.({ type: "inference.gateway.reattached", service_id: managed.serviceId, inference_id: restored.record.inference_id, epoch: restored.record.epoch });
+        }
+        if (this.recovered.size) {
+            this.recoveryTimer = setInterval(() => {
+                this.recoveryChecking ??= this.reconcileRecovered().catch(() => {
+                    this.onEvent?.({ type: "inference.recovery.unresolved", code: "inference_recovery_ambiguous" });
+                }).finally(() => { this.recoveryChecking = undefined; });
+            }, this.recoveryPollIntervalMs);
+            this.recoveryTimer.unref?.();
+        }
+    }
+    async reconcileRecovered() {
+        for (const [key, owner] of this.recovered) {
+            const state = owner.service.isReady() ? await owner.plan.state() : "finished";
+            if (owner.claimed || this.recovered.get(key) !== owner)
+                continue;
+            if (state !== "finished")
+                continue;
+            owner.retiring ??= owner.lease.release().then(() => { this.recovered.delete(key); });
+            await owner.retiring;
+        }
+        if (!this.recovered.size && this.recoveryTimer) {
+            clearInterval(this.recoveryTimer);
+            delete this.recoveryTimer;
+        }
+    }
+    async acquire(input) {
+        if (this.closed)
+            throw new HitchError("local inference manager is closed", { code: "inference_route_unavailable", exitCode: 12 });
+        const prepared = await this.prepareInputs(input);
+        return (await this.acquirePrepared(input, prepared, true)).lease;
+    }
+    prepareInputs(input) {
+        return this.preflight({
+            root: this.root,
+            selection: input.selection,
+            harnessRef: input.harness_ref,
+            reusableLocks: [...this.services.values()].filter((s) => !s.invalidated).map((s) => s.lock),
+            ...(input.signal ? { runtime: { signal: input.signal } } : {}),
+            ...(input.on_event ? { onProgress: (message) => input.on_event?.({ type: "inference.preparing", message }) } : {}),
+        });
+    }
+    async acquirePrepared(input, prepared, persistEvidence) {
+        if (this.closed)
+            throw new HitchError("local inference manager is closed", { code: "inference_route_unavailable", exitCode: 12 });
+        if (input.signal?.aborted)
+            throw new HitchError("inference acquisition cancelled", { code: "cancelled", exitCode: 9 });
+        const cacheScopeOwner = !prepared.lock.model_node && prepared.lock.execution.prefix_cache.mode === "disabled"
+            ? "prefix-cache-disabled"
+            : input.cache_scope_owner;
+        const isolationKey = sha256JSON({ inference_id: prepared.lock.inference_id, cache_scope_owner: cacheScopeOwner });
+        const serviceKey = `${prepared.lock.inference_id}:${isolationKey}`;
+        const recoveredKey = `${serviceKey}:${input.run_id}`;
+        await this.recovered.get(recoveredKey)?.retiring;
+        if (this.closed || input.signal?.aborted)
+            throw new HitchError("inference acquisition interrupted during recovery", { code: "inference_route_unavailable", exitCode: 12 });
+        const recovered = this.recovered.get(recoveredKey);
+        if (recovered) {
+            if (!recovered.service.isReady() || recovered.plan.cacheScopeOwner !== input.cache_scope_owner
+                || sha256JSON(recovered.plan.lock) !== sha256JSON(prepared.lock)
+                || sha256JSON(recovered.plan.evidenceOwner) !== sha256JSON(input.evidence_owner ?? null)) {
+                throw new HitchError("resumed inference owner differs from its original service", { code: "inference_recovery_ambiguous", exitCode: 12 });
+            }
+            recovered.claimed = true;
+            this.recovered.delete(recoveredKey);
+            return { lease: recovered.lease, observation: recovered.service.observation ?? null };
+        }
+        let allocation = this.resourceLeases.get(serviceKey);
+        // Model-node process capacity/ownership is enforced by that node's shared
+        // training/evaluation ledger, never by the controller host's GPU inventory.
+        if (!allocation && this.resources && !prepared.lock.model_node) {
+            if (!this.resources.canEverFit(prepared.lock.resources)) {
+                const missing = Object.entries(prepared.lock.resources).filter(([key, value]) => value > (this.resources.capacity[key] ?? 0))
+                    .map(([key, value]) => `${key} requires ${value}`);
+                throw new HitchError(`local inference resources exceed daemon capacity (${missing.join(", ")}); restart the daemon with sufficient --capacity-* limits`, { code: "inference_capacity_exceeded", exitCode: 12 });
+            }
+            allocation = this.resources.tryAcquire(serviceKey, "inference", prepared.lock.resources) ?? undefined;
+            if (!allocation)
+                throw new HitchError("local inference resources are currently unavailable", { code: "inference_capacity_exceeded", exitCode: 12 });
+            this.resourceLeases.set(serviceKey, allocation);
+        }
+        // Only a supervisor terminal event confirms that the engine no longer owns
+        // these resources. An acquisition failure alone is not such confirmation.
+        const serviceLease = await this.supervisor.acquire({
+            lock: prepared.lock,
+            model: prepared.model,
+            runtime: prepared.runtime,
+            isolationKey,
+            ownerId: input.run_id,
+            ...(input.signal ? { signal: input.signal } : {}),
+        });
+        const managed = await this.ensureGateway(serviceKey, serviceLease, prepared.lock, allocation, input);
+        managed.refs += 1;
+        let registration;
+        try {
+            if (!serviceLease.isReady() || input.signal?.aborted)
+                throw new HitchError("inference service is no longer available", { code: "inference_route_unavailable", exitCode: 12 });
+            registration = managed.gateway.register(input.run_id);
+            if (prepared.lock.model_node)
+                await sealManagedGateway(this.root, { serviceId: serviceLease.service_id, epoch: serviceLease.epoch,
+                    inferenceId: prepared.lock.inference_id, isolationKey, ownerId: input.run_id, cacheScopeOwner: input.cache_scope_owner,
+                    evidenceOwner: input.evidence_owner, binding: registration.binding, credential: registration.credential });
+            if (persistEvidence)
+                await writeInferenceEvidence(this.root, input.run_id, input.evidence_owner, {
+                    lock: prepared.lock,
+                    model: prepared.model,
+                    runtime: prepared.runtime,
+                    service: { service_id: serviceLease.service_id, epoch: serviceLease.epoch, isolation_key: isolationKey },
+                    doctor: prepared.doctor ?? null,
+                    model_node_observation: prepared.node_observation ?? null,
+                    observation: serviceLease.observation ?? null,
+                });
+            input.on_event?.({
+                type: "inference.ready", run_id: input.run_id, service_id: serviceLease.service_id,
+                inference_id: prepared.lock.inference_id, backend: prepared.lock.execution.platform.backend,
+            });
+        }
+        catch (error) {
+            registration?.revoke();
+            managed.refs = Math.max(0, managed.refs - 1);
+            await serviceLease.release();
+            throw error;
+        }
+        let released = false;
+        const lease = {
+            binding: registration.binding,
+            credential: registration.credential,
+            lock: prepared.lock,
+            service_id: serviceLease.service_id,
+            service_epoch: serviceLease.epoch,
+            release: async () => {
+                if (released)
+                    return;
+                released = true;
+                registration.revoke();
+                managed.refs = Math.max(0, managed.refs - 1);
+                await serviceLease.release();
+            },
+        };
+        return { lease, observation: serviceLease.observation ?? null };
+    }
+    async prepare(selection, signal, onEvent) {
+        if (this.closed)
+            throw new HitchError("local inference manager is closed", { code: "inference_route_unavailable", exitCode: 12 });
+        const id = `run_${randomUUID().replaceAll("-", "")}`;
+        const input = {
+            run_id: id, harness_ref: "model-call", selection, cache_scope_owner: id,
+            ...(signal ? { signal } : {}), ...(onEvent ? { on_event: onEvent } : {}),
+        };
+        const prepared = await this.prepareInputs(input);
+        const { lease, observation } = await this.acquirePrepared(input, prepared, false);
+        try {
+            const result = { schema_version: "1", ...prepared, observation };
+            await atomicWriteJSON(path.join(statePaths(this.root).inferenceLocks, prepared.lock.inference_id.slice(7), "validation.json"), result);
+            return result;
+        }
+        finally {
+            await lease.release();
+            try {
+                await this.stop(lease.service_id);
+            }
+            catch (error) {
+                if (error.code !== "inference_in_use")
+                    throw error;
+            }
+        }
+    }
+    async list() { return this.supervisor.list(); }
+    async stop(serviceId, force = false) {
+        await this.supervisor.stop(serviceId, force);
+    }
+    async close() {
+        if (this.closed)
+            return;
+        this.closed = true;
+        if (this.recoveryTimer)
+            clearInterval(this.recoveryTimer);
+        await this.recoveryChecking;
+        await this.supervisor.close();
+    }
+    async ensureGateway(key, lease, lock, resources, input) {
+        const existing = this.services.get(key);
+        if (existing && existing.serviceId === lease.service_id && !existing.invalidated)
+            return existing;
+        let pending = this.gatewayPending.get(lease.service_id);
+        if (!pending) {
+            pending = (async () => {
+                const gateway = await this.startGateway({
+                    upstreamBaseUrl: lease.base_url,
+                    engineToken: lease.engine_token,
+                    wireModel: lease.wire_model,
+                    lock,
+                    ...(input.on_event ? { onRequest: input.on_event } : {}),
+                });
+                if (!lease.isReady()) {
+                    await gateway.close();
+                    throw new HitchError("inference service failed while opening its gateway", { code: "inference_route_unavailable", exitCode: 12 });
+                }
+                const managed = { key, serviceId: lease.service_id, gateway, lock, refs: 0, ...(resources ? { resources } : {}) };
+                this.services.set(key, managed);
+                this.serviceById.set(lease.service_id, managed);
+                return managed;
+            })();
+            this.gatewayPending.set(lease.service_id, pending);
+            pending.finally(() => { if (this.gatewayPending.get(lease.service_id) === pending)
+                this.gatewayPending.delete(lease.service_id); }).catch(() => { });
+        }
+        try {
+            return await pending;
+        }
+        catch (error) {
+            await lease.release();
+            try {
+                await this.supervisor.stop(lease.service_id);
+            }
+            catch { /* Other owners or an ambiguous stop retain the reservation. */ }
+            throw error;
+        }
+    }
+    async cleanupManagedService(service) {
+        if (this.services.get(service.key) !== service)
+            return;
+        this.services.delete(service.key);
+        this.serviceById.delete(service.serviceId);
+        if (this.resourceLeases.get(service.key) === service.resources)
+            this.resourceLeases.delete(service.key);
+        await service.gateway.close().catch(() => { });
+        service.resources?.release();
+    }
+}
+async function writeInferenceEvidence(root, runId, owner, evidence) {
+    if (!/^run_[a-f0-9]{32}$/.test(runId))
+        throw new TypeError("inference evidence run ID is invalid");
+    if (owner && (!/^eval_[a-f0-9]{32}$/.test(owner.eval_id)
+        || owner.rerun_id !== undefined && !/^rerun_[a-f0-9]{32}$/.test(owner.rerun_id))) {
+        throw new TypeError("inference evidence eval identity is invalid");
+    }
+    const directory = await ensureDir(owner
+        ? owner.rerun_id
+            ? path.join(statePaths(root).evals, owner.eval_id, "reruns", owner.rerun_id, "inference")
+            : path.join(statePaths(root).evals, owner.eval_id, "inference")
+        : path.join(statePaths(root).runs, runId, "inference"));
+    const execution = {
+        schema_version: evidence.model_node_observation ? "2" : "1", run_id: runId,
+        ...(owner ? { eval_id: owner.eval_id, ...(owner.rerun_id ? { rerun_id: owner.rerun_id } : {}) } : {}),
+        service: evidence.service, doctor: evidence.doctor, observation: evidence.observation,
+        ...(evidence.model_node_observation ? { model_node_observation: evidence.model_node_observation } : {}),
+        prepared_at: new Date().toISOString(),
+    };
+    const files = [["lock.json", evidence.lock], ["model.manifest.json", evidence.model],
+        ["runtime.manifest.json", evidence.runtime], ["execution.json", execution]];
+    await withFileLock(statePaths(root).inferenceOperationLocks, sha256JSON({ inference_evidence: directory }), async () => {
+        const previous = await readJSON(path.join(directory, "execution.json"), null);
+        if (execution.schema_version === "2" && previous && sha256JSON(previous.service) === sha256JSON(execution.service)) {
+            // Recovery can finish the worker and retire its gateway before the eval
+            // scheduler reacquires the same service to finalize. Retain the original
+            // evidence, including its time, instead of making that service look new.
+            if (typeof previous.prepared_at !== "string" || !Number.isFinite(Date.parse(previous.prepared_at))
+                || sha256JSON(previous) !== sha256JSON({ ...execution, prepared_at: previous.prepared_at })) {
+                throw new HitchError("original managed inference execution evidence changed", { code: "inference_recovery_ambiguous", exitCode: 12 });
+            }
+            for (const [name, value] of files.slice(0, -1)) {
+                if (sha256JSON(await readJSON(path.join(directory, name), null)) !== sha256JSON(value)) {
+                    throw new HitchError("original managed inference input evidence changed", { code: "inference_recovery_ambiguous", exitCode: 12 });
+                }
+            }
+            return;
+        }
+        await Promise.all(files.map(([name, value]) => atomicWriteJSON(path.join(directory, name), value)));
+    });
+}
+//# sourceMappingURL=inference-manager.js.map

package/dist/src/control-plane/inference-manager.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"inference-manager.js","sourceRoot":"","sources":["../../../src/control-plane/inference-manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,IAAI,MAAM,WAAW,CAAC;AAQ7B,OAAO,EAAE,UAAU,EAAE,eAAe,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAChI,OAAO,EAAE,qBAAqB,EAAE,uBAAuB,EAAE,MAAM,uBAAuB,CAAC;AAEvF,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAI7D,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAgClE,MAAM,OAAO,qBAAqB;IACf,IAAI,CAAS;IACb,SAAS,CAA6B;IACtC,UAAU,CAA0B;IACpC,SAAS,CAAwF;IACjG,YAAY,CAAoE;IAChF,OAAO,CAAyD;IAChE,QAAQ,GAAG,IAAI,GAAG,EAA0B,CAAC;IAC7C,WAAW,GAAG,IAAI,GAAG,EAA0B,CAAC;IAChD,cAAc,GAAG,IAAI,GAAG,EAAmC,CAAC;IAC5D,cAAc,GAAG,IAAI,GAAG,EAAyB,CAAC;IAClD,QAAQ,CAAqC;IAC7C,SAAS,GAAG,IAAI,GAAG,EAA0B,CAAC;IAC9C,sBAAsB,CAAS;IACxC,aAAa,CAAkB;IAC/B,gBAAgB,CAA4B;IAC5C,MAAM,GAAG,KAAK,CAAC;IAEvB,YAAY,OAAqC;QAC/C,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QACnC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;QAC/B,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;QACjC,IAAI,CAAC,sBAAsB,GAAG,OAAO,CAAC,sBAAsB,IAAI,KAAK,CAAC;QACtE,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,sBAAsB,CAAC,IAAI,IAAI,CAAC,sBAAsB,GAAG,CAAC;YAAE,MAAM,IAAI,SAAS,CAAC,6CAA6C,CAAC,CAAC;QAC9J,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,IAAI,uBAAuB,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QACtJ,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,qBAAqB,CAAC;QAC5D,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,iBAAiB,CAAC,KAAK,CAAC;QACpE,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE;YAC3D,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YACxD,IAAI,OAAO,EAAE,CAAC;gBACZ,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC;gBAC3B,MAAM,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;gBAC9B,IAAI,QAAQ;oBAAE,MAAM,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC;YAC1D,CAAC;iBAAM,IAAI,QAAQ,EAAE,CAAC;gBACpB,MAAM,GAAG,GAAG,GAAG,MAAM,CAAC,YAAY,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;gBAC7D,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,OAAO,EAAE,CAAC;gBACxC,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAClC,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,UAAU;QACd,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QAC9E,MAAM,MAAM,GAAG,IAAI,GAAG,EAAoB,CAAC;QAC3C,KAAK,MAAM,IAAI,IAAI,KAAK;YAAE,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;QAC1H,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,EAAE,SAAS,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC;QAC1H,KAAK,MAAM,QAAQ,IAAI,QAAQ,EAAE,CAAC;YAChC,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,KAAK,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YAC7F,IAAI,CAAC,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAE,CAAC,IAAI,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;gBACtI,MAAM,IAAI,UAAU,CAAC,0DAA0D,EAAE,EAAE,IAAI,EAAE,8BAA8B,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;YAC3I,CAAC;YACD,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAE,CAAC,OAAO,CAAE,CAAC;YACzD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,EAAE,eAAe,EAAE,KAAK,CAAC,QAAQ,EAAE,WAAW,EAAE,KAAK,CAAC,YAAY,EAAE,SAAS,EAAE,KAAK,CAAC,UAAU;gBACrI,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,CAAC,CAAC;YAC/E,MAAM,GAAG,GAAG,GAAG,QAAQ,CAAC,MAAM,CAAC,YAAY,IAAI,QAAQ,CAAC,MAAM,CAAC,aAAa,EAAE,CAAC;YAC/E,MAAM,OAAO,GAAmB,EAAE,GAAG,EAAE,SAAS,EAAE,QAAQ,CAAC,MAAM,CAAC,UAAU,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,CAAC,MAAM,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,CAAC;YACpI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YAClF,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;gBAC5B,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAE,CAAC;gBACnD,IAAI,KAAqB,CAAC;gBAC1B,MAAM,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU,EAC5F,KAAK,IAAI,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,IAAI,MAAM,IAAI,CAAC,KAAK,EAAE,KAAK,QAAQ,CAAC,CAAC,CAAC;gBACvF,IAAI,QAAQ,GAAG,KAAK,CAAC;gBACrB,MAAM,KAAK,GAA4B,EAAE,OAAO,EAAE,YAAY,CAAC,OAAO,EAAE,UAAU,EAAE,YAAY,CAAC,UAAU,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI;oBAC9H,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,aAAa,EAAE,OAAO,CAAC,KAAK;oBAC5D,OAAO,EAAE,KAAK,IAAI,EAAE;wBAClB,IAAI,QAAQ;4BAAE,OAAO;wBACrB,YAAY,CAAC,MAAM,EAAE,CAAC;wBACtB,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;wBACxB,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;wBAAC,QAAQ,GAAG,IAAI,CAAC;oBAChE,CAAC,EAAE,CAAC;gBACN,KAAK,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE,YAAY,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;gBAC/D,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,GAAG,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE,KAAK,CAAC,CAAC;YACtD,CAAC;YACD,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,8BAA8B,EAAE,UAAU,EAAE,OAAO,CAAC,SAAS,EAAE,YAAY,EAAE,QAAQ,CAAC,MAAM,CAAC,YAAY,EAAE,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;QACpK,CAAC;QACD,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;YACxB,IAAI,CAAC,aAAa,GAAG,WAAW,CAAC,GAAG,EAAE;gBACpC,IAAI,CAAC,gBAAgB,KAAK,IAAI,CAAC,kBAAkB,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE;oBAC7D,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,+BAA+B,EAAE,IAAI,EAAE,8BAA8B,EAAE,CAAC,CAAC;gBAClG,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,gBAAgB,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;YAC3D,CAAC,EAAE,IAAI,CAAC,sBAAsB,CAAC,CAAC;YAChC,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,EAAE,CAAC;QAC/B,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,kBAAkB;QAC9B,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YAC1C,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC;YAC9E,IAAI,KAAK,CAAC,OAAO,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,KAAK;gBAAE,SAAS;YACjE,IAAI,KAAK,KAAK,UAAU;gBAAE,SAAS;YACnC,KAAK,CAAC,QAAQ,KAAK,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACrF,MAAM,KAAK,CAAC,QAAQ,CAAC;QACvB,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YAAC,aAAa,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YAAC,OAAO,IAAI,CAAC,aAAa,CAAC;QAAC,CAAC;IACnH,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,KAAqC;QACjD,IAAI,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,UAAU,CAAC,mCAAmC,EAAE,EAAE,IAAI,EAAE,6BAA6B,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;QAClI,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;QACjD,OAAO,CAAC,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC;IACnE,CAAC;IAEO,aAAa,CAAC,KAAqC;QACzD,OAAO,IAAI,CAAC,SAAS,CAAC;YACpB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,UAAU,EAAE,KAAK,CAAC,WAAW;YAC7B,aAAa,EAAE,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;YAC3F,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC9D,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACnH,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,eAAe,CAAC,KAAqC,EAAE,QAAyC,EAAE,eAAwB;QACtI,IAAI,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,UAAU,CAAC,mCAAmC,EAAE,EAAE,IAAI,EAAE,6BAA6B,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;QAClI,IAAI,KAAK,CAAC,MAAM,EAAE,OAAO;YAAE,MAAM,IAAI,UAAU,CAAC,iCAAiC,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;QACvH,MAAM,eAAe,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,IAAI,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,IAAI,KAAK,UAAU;YAC3G,CAAC,CAAC,uBAAuB;YACzB,CAAC,CAAC,KAAK,CAAC,iBAAiB,CAAC;QAC5B,MAAM,YAAY,GAAG,UAAU,CAAC,EAAE,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,iBAAiB,EAAE,eAAe,EAAE,CAAC,CAAC;QAClH,MAAM,UAAU,GAAG,GAAG,QAAQ,CAAC,IAAI,CAAC,YAAY,IAAI,YAAY,EAAE,CAAC;QACnE,MAAM,YAAY,GAAG,GAAG,UAAU,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QACrD,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,QAAQ,CAAC;QACjD,IAAI,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,EAAE,OAAO;YAAE,MAAM,IAAI,UAAU,CAAC,mDAAmD,EAAE,EAAE,IAAI,EAAE,6BAA6B,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;QAC3K,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QACnD,IAAI,SAAS,EAAE,CAAC;YACd,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,SAAS,CAAC,IAAI,CAAC,eAAe,KAAK,KAAK,CAAC,iBAAiB;mBACzF,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC;mBAC7D,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,UAAU,CAAC,KAAK,CAAC,cAAc,IAAI,IAAI,CAAC,EAAE,CAAC;gBAC3F,MAAM,IAAI,UAAU,CAAC,2DAA2D,EAAE,EAAE,IAAI,EAAE,8BAA8B,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;YAC5I,CAAC;YACD,SAAS,CAAC,OAAO,GAAG,IAAI,CAAC;YAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;YAC9D,OAAO,EAAE,KAAK,EAAE,SAAS,CAAC,KAAK,EAAE,WAAW,EAAE,SAAS,CAAC,OAAO,CAAC,WAAW,IAAI,IAAI,EAAE,CAAC;QACxF,CAAC;QACD,IAAI,UAAU,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACrD,0EAA0E;QAC1E,4EAA4E;QAC5E,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,SAAS,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;YAC/D,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;gBACxD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,GAAG,CAAC,IAAI,CAAC,SAAU,CAAC,QAAQ,CAAC,GAA2C,CAAC,IAAI,CAAC,CAAC,CAAC;qBACnK,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,aAAa,KAAK,EAAE,CAAC,CAAC;gBACrD,MAAM,IAAI,UAAU,CAAC,qDAAqD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,2DAA2D,EAAE,EAAE,IAAI,EAAE,6BAA6B,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;YAClN,CAAC;YACD,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,UAAU,EAAE,WAAW,EAAE,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,SAAS,CAAC;YACtG,IAAI,CAAC,UAAU;gBAAE,MAAM,IAAI,UAAU,CAAC,qDAAqD,EAAE,EAAE,IAAI,EAAE,6BAA6B,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;YACpJ,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;QAClD,CAAC;QACD,2EAA2E;QAC3E,0EAA0E;QAC1E,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC;YACjD,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,OAAO,EAAE,QAAQ,CAAC,OAAO;YACzB,YAAY;YACZ,OAAO,EAAE,KAAK,CAAC,MAAM;YACrB,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAClD,CAAC,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,EAAE,YAAY,EAAE,QAAQ,CAAC,IAAI,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC;QACrG,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC;QAClB,IAAI,YAAY,CAAC;QACjB,IAAI,CAAC;YACH,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,IAAI,KAAK,CAAC,MAAM,EAAE,OAAO;gBAAE,MAAM,IAAI,UAAU,CAAC,0CAA0C,EAAE,EAAE,IAAI,EAAE,6BAA6B,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;YAC9K,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YACtD,IAAI,QAAQ,CAAC,IAAI,CAAC,UAAU;gBAAE,MAAM,kBAAkB,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,YAAY,CAAC,UAAU,EAAE,KAAK,EAAE,YAAY,CAAC,KAAK;oBAC/H,WAAW,EAAE,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,YAAY,EAAE,OAAO,EAAE,KAAK,CAAC,MAAM,EAAE,eAAe,EAAE,KAAK,CAAC,iBAAiB;oBACtH,aAAa,EAAE,KAAK,CAAC,cAAc,EAAE,OAAO,EAAE,YAAY,CAAC,OAAO,EAAE,UAAU,EAAE,YAAY,CAAC,UAAU,EAAE,CAAC,CAAC;YAC7G,IAAI,eAAe;gBAAE,MAAM,sBAAsB,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,cAAc,EAAE;oBAC/F,IAAI,EAAE,QAAQ,CAAC,IAAI;oBACnB,KAAK,EAAE,QAAQ,CAAC,KAAK;oBACrB,OAAO,EAAE,QAAQ,CAAC,OAAO;oBACzB,OAAO,EAAE,EAAE,UAAU,EAAE,YAAY,CAAC,UAAU,EAAE,KAAK,EAAE,YAAY,CAAC,KAAK,EAAE,aAAa,EAAE,YAAY,EAAE;oBACxG,MAAM,EAAE,QAAQ,CAAC,MAAM,IAAI,IAAI;oBAC/B,sBAAsB,EAAE,QAAQ,CAAC,gBAAgB,IAAI,IAAI;oBACzD,WAAW,EAAE,YAAY,CAAC,WAAW,IAAI,IAAI;iBAC9C,CAAC,CAAC;YACH,KAAK,CAAC,QAAQ,EAAE,CAAC;gBACf,IAAI,EAAE,iBAAiB,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,UAAU,EAAE,YAAY,CAAC,UAAU;gBAClF,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,OAAO,EAAE,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,OAAO;aAC5F,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,YAAY,EAAE,MAAM,EAAE,CAAC;YACvB,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;YAC7C,MAAM,YAAY,CAAC,OAAO,EAAE,CAAC;YAC7B,MAAM,KAAK,CAAC;QACd,CAAC;QACD,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,MAAM,KAAK,GAA4B;YACrC,OAAO,EAAE,YAAY,CAAC,OAAO;YAC7B,UAAU,EAAE,YAAY,CAAC,UAAU;YACnC,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,UAAU,EAAE,YAAY,CAAC,UAAU;YACnC,aAAa,EAAE,YAAY,CAAC,KAAK;YACjC,OAAO,EAAE,KAAK,IAAI,EAAE;gBAClB,IAAI,QAAQ;oBAAE,OAAO;gBACrB,QAAQ,GAAG,IAAI,CAAC;gBAChB,YAAY,CAAC,MAAM,EAAE,CAAC;gBACtB,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;gBAC7C,MAAM,YAAY,CAAC,OAAO,EAAE,CAAC;YAC/B,CAAC;SACF,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,YAAY,CAAC,WAAW,IAAI,IAAI,EAAE,CAAC;IAClE,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,SAAoC,EAAE,MAAoB,EAAE,OAAkD;QAC1H,IAAI,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,UAAU,CAAC,mCAAmC,EAAE,EAAE,IAAI,EAAE,6BAA6B,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;QAClI,MAAM,EAAE,GAAG,OAAO,UAAU,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,CAAC;QACrD,MAAM,KAAK,GAAmC;YAC5C,MAAM,EAAE,EAAE,EAAE,WAAW,EAAE,YAAY,EAAE,SAAS,EAAE,iBAAiB,EAAE,EAAE;YACvE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACzE,CAAC;QACF,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;QACjD,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;QAClF,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,EAAE,cAAc,EAAE,GAAG,EAAE,GAAG,QAAQ,EAAE,WAAW,EAAE,CAAC;YACjE,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,cAAc,EAAE,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,iBAAiB,CAAC,EAAE,MAAM,CAAC,CAAC;YACvI,OAAO,MAAM,CAAC;QAChB,CAAC;gBAAS,CAAC;YACT,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;YACtB,IAAI,CAAC;gBAAC,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YAAC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACxD,IAAK,KAA2B,CAAC,IAAI,KAAK,kBAAkB;oBAAE,MAAM,KAAK,CAAC;YAC5E,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,IAAI,KAAK,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAE/C,KAAK,CAAC,IAAI,CAAC,SAAkB,EAAE,KAAK,GAAG,KAAK;QAC1C,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IAC/C,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO;QACxB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,IAAI,CAAC,aAAa;YAAE,aAAa,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC1D,MAAM,IAAI,CAAC,gBAAgB,CAAC;QAC5B,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;IAChC,CAAC;IAEO,KAAK,CAAC,aAAa,CACzB,GAAW,EACX,KAAyB,EACzB,IAAqC,EACrC,SAAoC,EACpC,KAAqC;QAErC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACxC,IAAI,QAAQ,IAAI,QAAQ,CAAC,SAAS,KAAK,KAAK,CAAC,UAAU,IAAI,CAAC,QAAQ,CAAC,WAAW;YAAE,OAAO,QAAQ,CAAC;QAClG,IAAI,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QACxD,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,GAAG,CAAC,KAAK,IAAI,EAAE;gBACpB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC;oBACtC,eAAe,EAAE,KAAK,CAAC,QAAQ;oBAC/B,WAAW,EAAE,KAAK,CAAC,YAAY;oBAC/B,SAAS,EAAE,KAAK,CAAC,UAAU;oBAC3B,IAAI;oBACJ,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBACzD,CAAC,CAAC;gBACH,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;oBACrB,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;oBACtB,MAAM,IAAI,UAAU,CAAC,oDAAoD,EAAE,EAAE,IAAI,EAAE,6BAA6B,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;gBACpI,CAAC;gBACD,MAAM,OAAO,GAAmB,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,CAAC,UAAU,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;gBAClI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;gBAChC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;gBAChD,OAAO,OAAO,CAAC;YACjB,CAAC,CAAC,EAAE,CAAC;YACL,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YACnD,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,IAAI,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,OAAO;gBAAE,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACtJ,CAAC;QACD,IAAI,CAAC;YAAC,OAAO,MAAM,OAAO,CAAC;QAAC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAC3C,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;YACtB,IAAI,CAAC;gBAAC,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YAAC,CAAC;YAAC,MAAM,CAAC,CAAC,+DAA+D,CAAC,CAAC;YAC/H,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,qBAAqB,CAAC,OAAuB;QACzD,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,OAAO;YAAE,OAAO;QACvD,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAClC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAC3C,IAAI,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,OAAO,CAAC,SAAS;YAAE,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACxG,MAAM,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAC9C,OAAO,CAAC,SAAS,EAAE,OAAO,EAAE,CAAC;IAC/B,CAAC;CACF;AAED,KAAK,UAAU,sBAAsB,CACnC,IAAY,EACZ,KAAa,EACb,KAAuD,EACvD,QAAiC;IAEjC,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,sCAAsC,CAAC,CAAC;IACnG,IAAI,KAAK,IAAI,CAAC,CAAC,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC;WACnD,KAAK,CAAC,QAAQ,KAAK,SAAS,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;QACnF,MAAM,IAAI,SAAS,CAAC,6CAA6C,CAAC,CAAC;IACrE,CAAC;IACD,MAAM,SAAS,GAAG,MAAM,SAAS,CAAC,KAAK;QACrC,CAAC,CAAC,KAAK,CAAC,QAAQ;YACd,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,WAAW,CAAC;YACzF,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE,WAAW,CAAC;QACjE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,WAAW,CAAC,CAAC,CAAC;IAC1D,MAAM,SAAS,GAAG;QACd,cAAc,EAAE,QAAQ,CAAC,sBAAsB,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK;QAC1E,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACrG,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW;QACrF,GAAG,CAAC,QAAQ,CAAC,sBAAsB,CAAC,CAAC,CAAC,EAAE,sBAAsB,EAAE,QAAQ,CAAC,sBAAsB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvG,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACxC,CAAC;IACF,MAAM,KAAK,GAA6B,CAAC,CAAC,WAAW,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,qBAAqB,EAAE,QAAQ,CAAC,KAAK,CAAC;QAC5G,CAAC,uBAAuB,EAAE,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,gBAAgB,EAAE,SAAS,CAAC,CAAC,CAAC;IAC9E,MAAM,YAAY,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,uBAAuB,EAAE,UAAU,CAAC,EAAE,kBAAkB,EAAE,SAAS,EAAE,CAAC,EAAE,KAAK,IAAI,EAAE;QACrH,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAiC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,gBAAgB,CAAC,EAAE,IAAI,CAAC,CAAC;QAC9G,IAAI,SAAS,CAAC,cAAc,KAAK,GAAG,IAAI,QAAQ,IAAI,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,UAAU,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;YACnH,wEAAwE;YACxE,yEAAyE;YACzE,yEAAyE;YACzE,IAAI,OAAO,QAAQ,CAAC,WAAW,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;mBAC7F,UAAU,CAAC,QAAQ,CAAC,KAAK,UAAU,CAAC,EAAE,GAAG,SAAS,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;gBAC9F,MAAM,IAAI,UAAU,CAAC,uDAAuD,EAAE,EAAE,IAAI,EAAE,8BAA8B,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;YACxI,CAAC;YACD,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC/C,IAAI,UAAU,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,CAAC,KAAK,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;oBACvF,MAAM,IAAI,UAAU,CAAC,mDAAmD,EAAE,EAAE,IAAI,EAAE,8BAA8B,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;gBACpI,CAAC;YACH,CAAC;YACD,OAAO;QACT,CAAC;QACD,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;IACtG,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/src/control-plane/managed-gateway-receipt.js

@@ -0,0 +1,26 @@
+import path from "node:path";
+import { HitchError, atomicWriteJSON, readJSON, sha256JSON, statePaths } from "../foundation/index.js";
+function receipt(input) {
+    return { schema_version: "2", service_id: input.serviceId, epoch: input.epoch, inference_id: input.inferenceId,
+        isolation_key: input.isolationKey, owner_id: input.ownerId,
+        scope_digest: sha256JSON({ cache_scope_owner: input.cacheScopeOwner, evidence_owner: input.evidenceOwner ?? null }),
+        target_digest: sha256JSON({ binding: input.binding, credential: input.credential }) };
+}
+function file(root, input) {
+    if (!/^inference_[a-f0-9]{32}$/.test(input.serviceId) || !/^run_[a-f0-9]{32}$/.test(input.ownerId))
+        throw ambiguous();
+    return path.join(statePaths(root).inferenceServices, input.serviceId, "gateways", receipt(input).target_digest.slice(7) + ".json");
+}
+export async function sealManagedGateway(root, input) {
+    const expected = receipt(input), previous = await readJSON(file(root, input), null);
+    if (previous && sha256JSON(previous) !== sha256JSON(expected))
+        throw ambiguous();
+    if (!previous)
+        await atomicWriteJSON(file(root, input), expected);
+}
+export async function verifyManagedGateway(root, input) {
+    if (sha256JSON(await readJSON(file(root, input), null)) !== sha256JSON(receipt(input)))
+        throw ambiguous();
+}
+function ambiguous() { return new HitchError("original private gateway receipt is missing or changed", { code: "inference_recovery_ambiguous", exitCode: 12 }); }
+//# sourceMappingURL=managed-gateway-receipt.js.map

package/dist/src/control-plane/managed-gateway-receipt.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"managed-gateway-receipt.js","sourceRoot":"","sources":["../../../src/control-plane/managed-gateway-receipt.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,UAAU,EAAE,eAAe,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAOvG,SAAS,OAAO,CAAC,KAAiC;IAChD,OAAO,EAAE,cAAc,EAAE,GAAG,EAAE,UAAU,EAAE,KAAK,CAAC,SAAS,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,YAAY,EAAE,KAAK,CAAC,WAAW;QAC5G,aAAa,EAAE,KAAK,CAAC,YAAY,EAAE,QAAQ,EAAE,KAAK,CAAC,OAAO;QAC1D,YAAY,EAAE,UAAU,CAAC,EAAE,iBAAiB,EAAE,KAAK,CAAC,eAAe,EAAE,cAAc,EAAE,KAAK,CAAC,aAAa,IAAI,IAAI,EAAE,CAAC;QACnH,aAAa,EAAE,UAAU,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,UAAU,EAAE,KAAK,CAAC,UAAU,EAAE,CAAC,EAAE,CAAC;AAC1F,CAAC;AACD,SAAS,IAAI,CAAC,IAAY,EAAE,KAAiC;IAC3D,IAAI,CAAC,0BAA0B,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC;QAAE,MAAM,SAAS,EAAE,CAAC;IACtH,OAAO,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,iBAAiB,EAAE,KAAK,CAAC,SAAS,EAAE,UAAU,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC;AACrI,CAAC;AACD,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,IAAY,EAAE,KAAiC;IACtF,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE,QAAQ,GAAG,MAAM,QAAQ,CAAiB,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,IAAI,CAAC,CAAC;IACpG,IAAI,QAAQ,IAAI,UAAU,CAAC,QAAQ,CAAC,KAAK,UAAU,CAAC,QAAQ,CAAC;QAAE,MAAM,SAAS,EAAE,CAAC;IACjF,IAAI,CAAC,QAAQ;QAAE,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,QAAQ,CAAC,CAAC;AACpE,CAAC;AACD,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,IAAY,EAAE,KAAiC;IACxF,IAAI,UAAU,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,IAAI,CAAC,CAAC,KAAK,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAAE,MAAM,SAAS,EAAE,CAAC;AAC5G,CAAC;AACD,SAAS,SAAS,KAAK,OAAO,IAAI,UAAU,CAAC,wDAAwD,EAAE,EAAE,IAAI,EAAE,8BAA8B,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC"}

package/dist/src/control-plane/managed-service-recovery.js

@@ -0,0 +1,158 @@
+import { readdir } from "node:fs/promises";
+import path from "node:path";
+import { parseExecutionLease, parsePhysicalExecution, readExecutionLeases } from "../evals/index.js";
+import { HitchError, readJSON, sha256JSON, statePaths } from "../foundation/index.js";
+import { validateInferenceLockShape } from "../inference/index.js";
+import { remoteModelBinding } from "../model-access/index.js";
+import { parseEvalControl } from "./eval-records.js";
+import { RemoteWorkInputStore } from "./remote-work-inputs.js";
+import { verifyManagedGateway } from "./managed-gateway-receipt.js";
+import { orderedEvalCancelled } from "./ordered-eval-control.js";
+/** Private routes already seal the original gateway credentials before dispatch. */
+export class ManagedServiceRecovery {
+    root;
+    registry;
+    protocol;
+    constructor(root, registry, protocol) {
+        this.root = root;
+        this.registry = registry;
+        this.protocol = protocol;
+    }
+    async select(records) {
+        const groups = new Map();
+        const evals = await readdir(statePaths(this.root).evals, { withFileTypes: true }).catch(error => {
+            if (error.code === "ENOENT")
+                return [];
+            throw error;
+        });
+        const inputs = new RemoteWorkInputStore(this.root);
+        for (const directory of evals) {
+            if (!directory.isDirectory() || !/^eval_[a-f0-9]{32}$/.test(directory.name))
+                continue;
+            const evalDirectory = path.join(statePaths(this.root).evals, directory.name);
+            for (const lease of await readExecutionLeases(evalDirectory)) {
+                if (lease.provider === "local-docker" || !["offered", "accepted", "running"].includes(lease.state) || Date.parse(lease.expires_at) <= Date.now())
+                    continue;
+                const offer = await this.protocol.findOfferForLease(lease.worker_id, lease.lease_id);
+                if (!offer || offer.state !== "accepted")
+                    continue;
+                const worker = await this.registry.get(offer.worker_id);
+                if (!worker || worker.revoked_at || worker.generation !== offer.generation)
+                    continue;
+                const ref = offer.inputs?.find(item => item.kind === "work-spec");
+                if (!ref)
+                    continue;
+                const spec = await readJSON((await inputs.verify(ref)).path);
+                if (spec.model_binding?.kind !== "managed-inference")
+                    continue;
+                const target = await this.protocol.modelRoutes.recoveryTarget(offer);
+                if (!target || target.kind !== "managed-inference" || target.binding.kind !== "managed-node"
+                    || spec.schema_version !== "2" || sha256JSON(spec.work) !== sha256JSON(offer.work)
+                    || sha256JSON(spec.model_binding) !== sha256JSON(remoteModelBinding(target)))
+                    throw ambiguous("managed route differs from its sealed work input");
+                const url = new URL(target.binding.base_url), match = url.pathname.match(/^\/runs\/(run_[a-f0-9]{32})\/v1\/$/);
+                if (!match || url.protocol !== "http:" || url.hostname !== "127.0.0.1" || !url.port || url.username || url.password || url.search || url.hash
+                    || !/^[a-f0-9]{64}$/.test(target.credential))
+                    throw ambiguous("original private gateway address or credential is invalid");
+                const ownerId = match[1];
+                const physical = spec.physical_execution ? parsePhysicalExecution(spec.physical_execution) : undefined;
+                if (physical?.kind === "verifier-only")
+                    throw ambiguous("verifier work cannot recover a model gateway");
+                const rerunId = physical && physical.kind !== "physical-infrastructure-retry" ? physical.rerun_id : undefined;
+                if (ownerId !== `run_${(rerunId ?? lease.eval_id).slice((rerunId ? "rerun_" : "eval_").length)}`)
+                    throw ambiguous("gateway owner differs from its original eval or rerun");
+                const cacheScopeOwner = rerunId ? `${lease.eval_id}:${rerunId}` : lease.eval_id;
+                const evidenceOwner = { kind: "eval", eval_id: lease.eval_id, ...(rerunId ? { rerun_id: rerunId } : {}) };
+                if (await this.cancelled(evidenceOwner))
+                    continue;
+                const candidates = records.filter(record => record.inference_id === target.binding.inference_id && record.lease_owner_ids.includes(ownerId)
+                    && sha256JSON(record.model_node ?? null) === sha256JSON(target.binding.model_node));
+                if (candidates.length !== 1 || candidates[0].state !== "ready")
+                    throw ambiguous("accepted model work has no unique original ready service");
+                const record = candidates[0];
+                await verifyManagedGateway(this.root, { serviceId: record.service_id, epoch: record.epoch, inferenceId: record.inference_id,
+                    isolationKey: record.isolation_key, ownerId, cacheScopeOwner, evidenceOwner, binding: target.binding, credential: target.credential });
+                const scopeDirectory = rerunId ? path.join(evalDirectory, "reruns", rerunId) : evalDirectory;
+                const lock = validateInferenceLockShape(await readJSON(path.join(scopeDirectory, "inference", "lock.json")));
+                const execution = await readJSON(path.join(scopeDirectory, "inference", "execution.json"));
+                if (record.inference_id !== lock.inference_id || lock.model_id !== target.modelId
+                    || lock.protocol.api !== target.binding.api || lock.generation.max_output_tokens !== target.maxOutputTokens
+                    || sha256JSON(lock.model_node ?? null) !== sha256JSON(record.model_node)
+                    || record.isolation_key !== sha256JSON({ inference_id: lock.inference_id, cache_scope_owner: cacheScopeOwner })
+                    || execution.schema_version !== "2" || execution.run_id !== ownerId || execution.eval_id !== lease.eval_id || execution.rerun_id !== rerunId
+                    || sha256JSON(execution.service) !== sha256JSON({ service_id: record.service_id, epoch: record.epoch, isolation_key: record.isolation_key })) {
+                    throw ambiguous("original inference execution evidence differs from the active gateway");
+                }
+                const key = `${record.service_id}:${ownerId}`;
+                const prior = groups.get(key);
+                if (prior) {
+                    if (sha256JSON(prior.plan.target) !== sha256JSON(target))
+                        throw ambiguous("one model owner has conflicting private gateways");
+                    prior.seeds.push(offer);
+                }
+                else {
+                    const seeds = [offer];
+                    const plan = { record, ownerId, lock, cacheScopeOwner, evidenceOwner, target, port: Number(url.port),
+                        state: () => this.state(seeds, target, evidenceOwner) };
+                    groups.set(key, { plan, seeds });
+                }
+            }
+        }
+        const plans = [];
+        for (const { plan } of groups.values())
+            if (await plan.state() !== "finished")
+                plans.push(plan);
+        return plans;
+    }
+    async cancelled(owner) {
+        if (await orderedEvalCancelled(this.root, owner.eval_id, owner.rerun_id))
+            return true;
+        const directory = path.join(statePaths(this.root).evals, owner.eval_id);
+        if (owner.rerun_id) {
+            const state = await readJSON(path.join(directory, "reruns", owner.rerun_id, "state.json"));
+            if (state.schema_version !== "1" || state.eval_id !== owner.eval_id || state.rerun_id !== owner.rerun_id
+                || !["queued", "running", "completed", "failed", "cancelled"].includes(String(state.status)))
+                throw ambiguous("rerun state identity changed");
+            const marker = await readJSON(path.join(directory, "reruns", owner.rerun_id, "cancellation.json"), null);
+            if (marker && (marker.schema_version !== "1" || marker.eval_id !== owner.eval_id || marker.rerun_id !== owner.rerun_id))
+                throw ambiguous("rerun cancellation identity changed");
+            return !!marker || ["completed", "failed", "cancelled"].includes(String(state.status));
+        }
+        const control = parseEvalControl(await readJSON(path.join(directory, "control.json")));
+        if (control.eval_id !== owner.eval_id)
+            throw ambiguous("eval control identity changed");
+        return !!control.cancel_requested_at || ["cancelling", "cancelled", "failed", "succeeded"].includes(control.state);
+    }
+    async state(seeds, target, owner) {
+        if (await this.cancelled(owner))
+            return "finished";
+        let waiting = false;
+        for (const seed of seeds) {
+            const offer = await this.protocol.getOffer(seed.worker_id, seed.offer_id);
+            if (!offer)
+                throw ambiguous("accepted model offer disappeared during recovery");
+            if (sha256JSON(identity(offer)) !== sha256JSON(identity(seed)))
+                throw ambiguous("accepted model offer identity changed");
+            if (offer.state !== "accepted")
+                continue;
+            const lease = parseExecutionLease(await readJSON(path.join(statePaths(this.root).evals, owner.eval_id, "leases", `${offer.lease.lease_id}.json`)));
+            if (lease.epoch !== offer.lease.epoch || !["offered", "accepted", "running"].includes(lease.state) || Date.parse(lease.expires_at) <= Date.now())
+                continue;
+            const worker = await this.registry.get(offer.worker_id);
+            if (!worker || worker.revoked_at || worker.generation !== offer.generation)
+                continue;
+            await this.protocol.modelRoutes.authorize(offer);
+            if (sha256JSON(await this.protocol.modelRoutes.recoveryTarget(offer)) !== sha256JSON(target))
+                throw ambiguous("private model route changed during recovery");
+            if (worker.worker.status === "ready")
+                return "active";
+            waiting = true;
+        }
+        return waiting ? "waiting" : "finished";
+    }
+}
+function identity(offer) {
+    return { worker: offer.worker_id, generation: offer.generation, offer: offer.offer_id, nonce: offer.nonce, lease: offer.lease, work: offer.work, inputs: offer.inputs };
+}
+function ambiguous(message) { return new HitchError(message, { code: "inference_recovery_ambiguous", exitCode: 12 }); }
+//# sourceMappingURL=managed-service-recovery.js.map

package/dist/src/control-plane/managed-service-recovery.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"managed-service-recovery.js","sourceRoot":"","sources":["../../../src/control-plane/managed-service-recovery.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,mBAAmB,EAAE,sBAAsB,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AACrG,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACtF,OAAO,EAAE,0BAA0B,EAAE,MAAM,uBAAuB,CAAC;AACnE,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAE9D,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAGrD,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AAejE,oFAAoF;AACpF,MAAM,OAAO,sBAAsB;IACZ;IAAuB;IAAyC;IAArF,YAAqB,IAAY,EAAW,QAA8B,EAAW,QAA8B;QAA9F,SAAI,GAAJ,IAAI,CAAQ;QAAW,aAAQ,GAAR,QAAQ,CAAsB;QAAW,aAAQ,GAAR,QAAQ,CAAsB;IAAG,CAAC;IAEvH,KAAK,CAAC,MAAM,CAAC,OAAmC;QAC9C,MAAM,MAAM,GAAG,IAAI,GAAG,EAA4E,CAAC;QACnG,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;YAC9F,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;gBAAE,OAAO,EAAE,CAAC;YAAC,MAAM,KAAK,CAAC;QACjF,CAAC,CAAC,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnD,KAAK,MAAM,SAAS,IAAI,KAAK,EAAE,CAAC;YAC9B,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;gBAAE,SAAS;YACtF,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC;YAC7E,KAAK,MAAM,KAAK,IAAI,MAAM,mBAAmB,CAAC,aAAa,CAAC,EAAE,CAAC;gBAC7D,IAAI,KAAK,CAAC,QAAQ,KAAK,cAAc,IAAI,CAAC,CAAC,SAAS,EAAE,UAAU,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE;oBAAE,SAAS;gBAC3J,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;gBACrF,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,KAAK,UAAU;oBAAE,SAAS;gBACnD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;gBACxD,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,UAAU,IAAI,MAAM,CAAC,UAAU,KAAK,KAAK,CAAC,UAAU;oBAAE,SAAS;gBACrF,MAAM,GAAG,GAAG,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC;gBAAC,IAAI,CAAC,GAAG;oBAAE,SAAS;gBACtF,MAAM,IAAI,GAAG,MAAM,QAAQ,CAA0B,CAAC,MAAM,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBACtF,IAAK,IAAI,CAAC,aAAgD,EAAE,IAAI,KAAK,mBAAmB;oBAAE,SAAS;gBACnG,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;gBACrE,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,IAAI,KAAK,mBAAmB,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,KAAK,cAAc;uBACvF,IAAI,CAAC,cAAc,KAAK,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC;uBAC/E,UAAU,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,UAAU,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;oBAAE,MAAM,SAAS,CAAC,kDAAkD,CAAC,CAAC;gBACpJ,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,KAAK,GAAG,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,oCAAoC,CAAC,CAAC;gBAC/G,IAAI,CAAC,KAAK,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,WAAW,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,IAAI;uBACxI,CAAC,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC;oBAAE,MAAM,SAAS,CAAC,2DAA2D,CAAC,CAAC;gBAC7H,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC;gBAC1B,MAAM,QAAQ,GAAG,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,sBAAsB,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;gBACvG,IAAI,QAAQ,EAAE,IAAI,KAAK,eAAe;oBAAE,MAAM,SAAS,CAAC,8CAA8C,CAAC,CAAC;gBACxG,MAAM,OAAO,GAAG,QAAQ,IAAI,QAAQ,CAAC,IAAI,KAAK,+BAA+B,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC;gBAC9G,IAAI,OAAO,KAAK,OAAO,CAAC,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,EAAE;oBAAE,MAAM,SAAS,CAAC,uDAAuD,CAAC,CAAC;gBAC3K,MAAM,eAAe,GAAG,OAAO,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,OAAO,IAAI,OAAO,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC;gBAChF,MAAM,aAAa,GAAG,EAAE,IAAI,EAAE,MAAe,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;gBACnH,IAAI,MAAM,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC;oBAAE,SAAS;gBAClD,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,YAAY,KAAK,MAAM,CAAC,OAAO,CAAC,YAAY,IAAI,MAAM,CAAC,eAAe,CAAC,QAAQ,CAAC,OAAO,CAAC;uBACtI,UAAU,CAAC,MAAM,CAAC,UAAU,IAAI,IAAI,CAAC,KAAK,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;gBACtF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,IAAI,UAAU,CAAC,CAAC,CAAE,CAAC,KAAK,KAAK,OAAO;oBAAE,MAAM,SAAS,CAAC,0DAA0D,CAAC,CAAC;gBAC7I,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,CAAE,CAAC;gBAC9B,MAAM,oBAAoB,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,MAAM,CAAC,UAAU,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,WAAW,EAAE,MAAM,CAAC,YAAY;oBACzH,YAAY,EAAE,MAAM,CAAC,aAAa,EAAE,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;gBACzI,MAAM,cAAc,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC;gBAC7F,MAAM,IAAI,GAAG,0BAA0B,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,WAAW,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC;gBAC7G,MAAM,SAAS,GAAG,MAAM,QAAQ,CAA0B,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,WAAW,EAAE,gBAAgB,CAAC,CAAC,CAAC;gBACpH,IAAI,MAAM,CAAC,YAAY,KAAK,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,QAAQ,KAAK,MAAM,CAAC,OAAO;uBAC5E,IAAI,CAAC,QAAQ,CAAC,GAAG,KAAK,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,KAAK,MAAM,CAAC,eAAe;uBACxG,UAAU,CAAC,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,KAAK,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC;uBACrE,MAAM,CAAC,aAAa,KAAK,UAAU,CAAC,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY,EAAE,iBAAiB,EAAE,eAAe,EAAE,CAAC;uBAC5G,SAAS,CAAC,cAAc,KAAK,GAAG,IAAI,SAAS,CAAC,MAAM,KAAK,OAAO,IAAI,SAAS,CAAC,OAAO,KAAK,KAAK,CAAC,OAAO,IAAI,SAAS,CAAC,QAAQ,KAAK,OAAO;uBACzI,UAAU,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,UAAU,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,aAAa,EAAE,MAAM,CAAC,aAAa,EAAE,CAAC,EAAE,CAAC;oBAC/I,MAAM,SAAS,CAAC,uEAAuE,CAAC,CAAC;gBAC3F,CAAC;gBACD,MAAM,GAAG,GAAG,GAAG,MAAM,CAAC,UAAU,IAAI,OAAO,EAAE,CAAC;gBAC9C,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAC9B,IAAI,KAAK,EAAE,CAAC;oBACV,IAAI,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,UAAU,CAAC,MAAM,CAAC;wBAAE,MAAM,SAAS,CAAC,kDAAkD,CAAC,CAAC;oBAC9H,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC1B,CAAC;qBAAM,CAAC;oBACN,MAAM,KAAK,GAAG,CAAC,KAAK,CAAC,CAAC;oBACtB,MAAM,IAAI,GAA+B,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC;wBAC9H,KAAK,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,aAAa,CAAC,EAAE,CAAC;oBAC1D,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;gBACnC,CAAC;YACH,CAAC;QACH,CAAC;QACD,MAAM,KAAK,GAAiC,EAAE,CAAC;QAC/C,KAAK,MAAM,EAAE,IAAI,EAAE,IAAI,MAAM,CAAC,MAAM,EAAE;YAAE,IAAI,MAAM,IAAI,CAAC,KAAK,EAAE,KAAK,UAAU;gBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChG,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,KAAkD;QACxE,IAAI,MAAM,oBAAoB,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,QAAQ,CAAC;YAAE,OAAO,IAAI,CAAC;QACtF,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;QACxE,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;YACnB,MAAM,KAAK,GAAG,MAAM,QAAQ,CAA0B,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC,CAAC;YACpH,IAAI,KAAK,CAAC,cAAc,KAAK,GAAG,IAAI,KAAK,CAAC,OAAO,KAAK,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,QAAQ,KAAK,KAAK,CAAC,QAAQ;mBACnG,CAAC,CAAC,QAAQ,EAAE,SAAS,EAAE,WAAW,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBAAE,MAAM,SAAS,CAAC,8BAA8B,CAAC,CAAC;YAChJ,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAiC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,mBAAmB,CAAC,EAAE,IAAI,CAAC,CAAC;YACzI,IAAI,MAAM,IAAI,CAAC,MAAM,CAAC,cAAc,KAAK,GAAG,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,CAAC,OAAO,IAAI,MAAM,CAAC,QAAQ,KAAK,KAAK,CAAC,QAAQ,CAAC;gBAAE,MAAM,SAAS,CAAC,qCAAqC,CAAC,CAAC;YAChL,OAAO,CAAC,CAAC,MAAM,IAAI,CAAC,WAAW,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;QACzF,CAAC;QACD,MAAM,OAAO,GAAG,gBAAgB,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC,CAAC,CAAC;QACvF,IAAI,OAAO,CAAC,OAAO,KAAK,KAAK,CAAC,OAAO;YAAE,MAAM,SAAS,CAAC,+BAA+B,CAAC,CAAC;QACxF,OAAO,CAAC,CAAC,OAAO,CAAC,mBAAmB,IAAI,CAAC,YAAY,EAAE,WAAW,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IACrH,CAAC;IAEO,KAAK,CAAC,KAAK,CAAC,KAA0B,EAAE,MAAc,EAAE,KAAkD;QAChH,IAAI,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC;YAAE,OAAO,UAAU,CAAC;QACnD,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC1E,IAAI,CAAC,KAAK;gBAAE,MAAM,SAAS,CAAC,kDAAkD,CAAC,CAAC;YAChF,IAAI,UAAU,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,KAAK,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;gBAAE,MAAM,SAAS,CAAC,uCAAuC,CAAC,CAAC;YACzH,IAAI,KAAK,CAAC,KAAK,KAAK,UAAU;gBAAE,SAAS;YACzC,MAAM,KAAK,GAAG,mBAAmB,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE,QAAQ,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC,QAAQ,OAAO,CAAC,CAAC,CAAC,CAAC;YACnJ,IAAI,KAAK,CAAC,KAAK,KAAK,KAAK,CAAC,KAAK,CAAC,KAAK,IAAI,CAAC,CAAC,SAAS,EAAE,UAAU,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE;gBAAE,SAAS;YAC3J,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;YACxD,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,UAAU,IAAI,MAAM,CAAC,UAAU,KAAK,KAAK,CAAC,UAAU;gBAAE,SAAS;YACrF,MAAM,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YACjD,IAAI,UAAU,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,KAAK,UAAU,CAAC,MAAM,CAAC;gBAAE,MAAM,SAAS,CAAC,6CAA6C,CAAC,CAAC;YAC7J,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,KAAK,OAAO;gBAAE,OAAO,QAAQ,CAAC;YACtD,OAAO,GAAG,IAAI,CAAC;QACjB,CAAC;QACD,OAAO,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC;IAC1C,CAAC;CACF;AACD,SAAS,QAAQ,CAAC,KAAwB;IACxC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,SAAS,EAAE,UAAU,EAAE,KAAK,CAAC,UAAU,EAAE,KAAK,EAAE,KAAK,CAAC,QAAQ,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC;AAC1K,CAAC;AACD,SAAS,SAAS,CAAC,OAAe,IAAI,OAAO,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,8BAA8B,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC"}

package/dist/src/control-plane/model-capture-planning.js

@@ -1,12 +1,26 @@
 import { getAdapter } from "../adapters/index.js";
 import { planModelCapture } from "../model-access/index.js";
+import { forceLocalInferenceCapturePlan } from "../evals/index.js";
+import { HitchError } from "../foundation/index.js";
 export function modelCapturePlanForEval(request, execution, provider) {
     const separator = request.harness_ref.indexOf("@");
     const harnessId = separator < 0 ? request.harness_ref : request.harness_ref.slice(0, separator);
-    return planModelCapture({
+    const plan = planModelCapture({
         policy: execution.model_capture,
         adapter: getAdapter(harnessId).requirements,
         provider,
     });
+    if (request.local_inference || request.training_binding) {
+        const remote = execution.provider !== "local-docker";
+        const feature = request.training_binding ? "training_external_binding" : "managed_model_node";
+        if (remote && (!provider.features.model_proxy || provider.features[feature] !== "2"
+            || request.local_inference && (!request.local_inference.model_node || !request.local_inference.inference_id))) {
+            throw new HitchError(`remote provider requires ${feature}=2 and an immutable model route`, {
+                code: "remote_model_binding_unsupported", exitCode: 10,
+            });
+        }
+        return forceLocalInferenceCapturePlan(plan, remote ? "in-sandbox" : "host-side");
+    }
+    return plan;
 }
 //# sourceMappingURL=model-capture-planning.js.map