agent-hitch 0.2.6 → 0.2.8

package/dist/src/control-plane/remote-result-transport.js

@@ -0,0 +1,175 @@
+import { createHash } from "node:crypto";
+import { lstat, mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { HitchError, atomicWriteJSON, ensureDir, readJSON } from "../foundation/index.js";
+import { importEvalTrialRun, parseExecutionEvidence } from "../evals/index.js";
+const MAX_ENVELOPE_BYTES = 128 * 1024 * 1024;
+const MAX_FILES = 100_000;
+export async function encodeRemoteResultEnvelope(input) {
+    validateTransportIdentity(input);
+    const files = await encodeTree(input.bundleDirectory);
+    const envelope = {
+        schema_version: "1", eval_id: input.evalId, work_id: input.workId,
+        lease_id: input.leaseId, lease_epoch: input.leaseEpoch, trial: input.trial, files,
+    };
+    const encoded = Buffer.from(`${JSON.stringify(envelope)}\n`);
+    if (encoded.length > MAX_ENVELOPE_BYTES)
+        throw transportError("remote result envelope exceeds its size limit");
+    return encoded;
+}
+export async function importRemoteResultEnvelope(input) {
+    const info = await lstat(input.artifactPath);
+    if (!info.isFile() || info.isSymbolicLink() || info.nlink !== 1 || info.size > MAX_ENVELOPE_BYTES)
+        throw transportError("remote result artifact is not a safe bounded file");
+    const envelope = parseRemoteResultEnvelope(JSON.parse(await readFile(input.artifactPath, "utf8")));
+    assertEnvelopeIdentity(envelope, input.work, input.lease);
+    const trialId = safeTrialId(envelope.trial.trial_name);
+    if (envelope.trial.task_name !== input.work.task_ids[0])
+        throw transportError("remote result task identity does not match its work item");
+    const backendDirectory = path.join(input.evalDirectory, "harbor", "work-items", input.work.work_id, `epoch-${String(input.lease.epoch).padStart(6, "0")}`);
+    const trialDirectory = path.join(backendDirectory, "job", trialId);
+    const bundleDirectory = path.join(trialDirectory, "hitch-run-bundle");
+    if (input.signal?.aborted)
+        throw input.signal.reason ?? new DOMException("The operation was aborted", "AbortError");
+    await rm(backendDirectory, { recursive: true, force: true });
+    try {
+        await materializeEnvelope(envelope, bundleDirectory);
+        const execution = parseExecutionEvidence(await readJSON(path.join(bundleDirectory, "execution.json")));
+        if (execution.provider !== input.lease.provider || execution.worker_id !== input.lease.worker_id
+            || execution.collision_domain_id !== input.lease.collision_domain_id || execution.eval_id !== input.lease.eval_id
+            || execution.work_id !== input.lease.work_id || execution.lease_id !== input.lease.lease_id
+            || execution.lease_epoch !== input.lease.epoch || execution.task_id !== input.work.task_ids[0]
+            || JSON.stringify(execution.reservation) !== JSON.stringify(input.work.reservation)) {
+            throw transportError("remote result execution evidence does not match its lease");
+        }
+        await writeTransportCompletionMarker(bundleDirectory, envelope, trialId);
+        const ref = await importEvalTrialRun({
+            root: input.root,
+            evalId: input.lease.eval_id,
+            evalDirectory: input.evalDirectory,
+            harborJobDirectory: path.join(backendDirectory, "job"),
+            expectedAttempt: input.work.logical_attempt,
+            request: input.request,
+            resolvedRevision: input.resolvedRevision,
+            benchmarkId: input.request.benchmark_id,
+            benchmarkRevision: input.request.benchmark_revision,
+            ...(input.publicationMode ? { publicationMode: input.publicationMode } : {}),
+            ...(input.runtimeId ? { runtimeId: input.runtimeId } : {}),
+            executionEvidence: execution,
+            ...(input.environmentImages ? { environmentImages: input.environmentImages } : {}),
+            ...(input.modelCapturePlan ? { modelCapturePlan: input.modelCapturePlan } : {}),
+            ...(input.signal ? { signal: input.signal } : {}),
+            requireCompleteMarker: true,
+        }, envelope.trial);
+        return { ref, trial: envelope.trial, backendDirectory };
+    }
+    catch (error) {
+        await rm(backendDirectory, { recursive: true, force: true });
+        throw error;
+    }
+}
+async function writeTransportCompletionMarker(bundleDirectory, envelope, trialId) {
+    const manifest = await readJSON(path.join(bundleDirectory, "manifest.json"));
+    if (typeof manifest.run_id !== "string" || !/^run_[a-f0-9]{32}$/.test(manifest.run_id)) {
+        throw transportError("remote result bundle manifest has an invalid run identity");
+    }
+    await atomicWriteJSON(path.join(bundleDirectory, "bundle.complete.json"), {
+        schema_version: "1", run_id: manifest.run_id, eval_id: envelope.eval_id, trial_id: trialId,
+    });
+}
+export function parseRemoteResultEnvelope(value) {
+    const record = exact(value, ["schema_version", "eval_id", "work_id", "lease_id", "lease_epoch", "trial", "files"], "remote result envelope");
+    validateTransportIdentity({ evalId: record.eval_id, workId: record.work_id, leaseId: record.lease_id, leaseEpoch: record.lease_epoch });
+    if (record.schema_version !== "1" || !record.trial || typeof record.trial !== "object" || Array.isArray(record.trial)
+        || !Array.isArray(record.files) || record.files.length < 1 || record.files.length > MAX_FILES)
+        throw transportError("remote result envelope is invalid");
+    const files = record.files.map(parseFile);
+    const sorted = [...files].sort((left, right) => Buffer.from(left.path).compare(Buffer.from(right.path)));
+    if (JSON.stringify(sorted) !== JSON.stringify(files) || new Set(files.map((file) => file.path)).size !== files.length)
+        throw transportError("remote result files are not canonical and unique");
+    return {
+        schema_version: "1", eval_id: record.eval_id, work_id: record.work_id,
+        lease_id: record.lease_id, lease_epoch: record.lease_epoch,
+        trial: record.trial, files,
+    };
+}
+async function encodeTree(root, relative = "") {
+    const directory = relative ? path.join(root, ...relative.split("/")) : root;
+    const entries = await readdir(directory, { withFileTypes: true });
+    const files = [];
+    for (const entry of entries.sort((left, right) => Buffer.from(left.name).compare(Buffer.from(right.name)))) {
+        const child = relative ? `${relative}/${entry.name}` : entry.name;
+        safeRelativePath(child);
+        const target = path.join(root, ...child.split("/"));
+        const info = await lstat(target);
+        if (info.isDirectory())
+            files.push(...await encodeTree(root, child));
+        else if (info.isFile() && !info.isSymbolicLink() && info.nlink === 1) {
+            const content = await readFile(target);
+            files.push({ path: child, size: content.length, sha256: digest(content), content_base64: content.toString("base64") });
+        }
+        else
+            throw transportError(`remote result bundle contains an unsafe entry: ${child}`);
+        if (files.length > MAX_FILES)
+            throw transportError("remote result bundle has too many files");
+    }
+    return files.sort((left, right) => Buffer.from(left.path).compare(Buffer.from(right.path)));
+}
+async function materializeEnvelope(envelope, destination) {
+    await mkdir(destination, { recursive: true, mode: 0o700 });
+    let total = 0;
+    for (const file of envelope.files) {
+        const content = Buffer.from(file.content_base64, "base64");
+        total += content.length;
+        if (content.length !== file.size || digest(content) !== file.sha256 || total > MAX_ENVELOPE_BYTES)
+            throw transportError(`remote result file integrity failed: ${file.path}`);
+        const target = path.join(destination, ...file.path.split("/"));
+        await ensureDir(path.dirname(target));
+        await writeFile(target, content, { flag: "wx", mode: 0o600 });
+    }
+}
+function parseFile(value) {
+    const record = exact(value, ["path", "size", "sha256", "content_base64"], "remote result file");
+    const relative = safeRelativePath(record.path);
+    if (!Number.isSafeInteger(record.size) || record.size < 0 || record.size > MAX_ENVELOPE_BYTES
+        || typeof record.sha256 !== "string" || !/^sha256:[a-f0-9]{64}$/.test(record.sha256)
+        || typeof record.content_base64 !== "string" || !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(record.content_base64)) {
+        throw transportError("remote result file is invalid");
+    }
+    return { path: relative, size: record.size, sha256: record.sha256, content_base64: record.content_base64 };
+}
+function assertEnvelopeIdentity(envelope, work, lease) {
+    if (envelope.eval_id !== work.eval_id || envelope.eval_id !== lease.eval_id || envelope.work_id !== work.work_id
+        || envelope.work_id !== lease.work_id || envelope.lease_id !== lease.lease_id || envelope.lease_epoch !== lease.epoch
+        || work.logical_attempt === null || work.task_ids.length !== 1)
+        throw transportError("remote result envelope identity does not match its lease");
+}
+function validateTransportIdentity(value) {
+    if (typeof value.evalId !== "string" || !/^eval_[a-f0-9]{32}$/.test(value.evalId)
+        || typeof value.workId !== "string" || !/^work_[a-f0-9]{32}$/.test(value.workId)
+        || typeof value.leaseId !== "string" || !/^lease_[a-f0-9]{32}$/.test(value.leaseId)
+        || !Number.isSafeInteger(value.leaseEpoch) || value.leaseEpoch < 1)
+        throw transportError("remote result transport identity is invalid");
+}
+function safeRelativePath(value) {
+    if (typeof value !== "string" || !value || value !== value.normalize("NFC") || value.includes("\\") || path.posix.isAbsolute(value)
+        || value.split("/").some((part) => !part || part === "." || part === ".."))
+        throw transportError("remote result path is unsafe");
+    return value;
+}
+function safeTrialId(value) {
+    if (typeof value !== "string" || !/^[A-Za-z0-9][A-Za-z0-9._-]{0,255}$/.test(value))
+        throw transportError("remote result trial id is unsafe");
+    return value;
+}
+function exact(value, keys, label) {
+    if (!value || typeof value !== "object" || Array.isArray(value))
+        throw transportError(`${label} must be an object`);
+    const record = value;
+    if (Object.keys(record).some((key) => !keys.includes(key)))
+        throw transportError(`${label} has unknown fields`);
+    return record;
+}
+function digest(value) { return `sha256:${createHash("sha256").update(value).digest("hex")}`; }
+function transportError(message) { return new HitchError(message, { code: "remote_result_invalid", exitCode: 12 }); }
+//# sourceMappingURL=remote-result-transport.js.map

package/dist/src/control-plane/remote-result-transport.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"remote-result-transport.js","sourceRoot":"","sources":["../../../src/control-plane/remote-result-transport.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAClF,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,UAAU,EAAE,eAAe,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAC1F,OAAO,EAAE,kBAAkB,EAAE,sBAAsB,EAAE,MAAM,mBAAmB,CAAC;AAG/E,MAAM,kBAAkB,GAAG,GAAG,GAAG,IAAI,GAAG,IAAI,CAAC;AAC7C,MAAM,SAAS,GAAG,OAAO,CAAC;AAmB1B,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAAC,KAOhD;IACC,yBAAyB,CAAC,KAAK,CAAC,CAAC;IACjC,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;IACtD,MAAM,QAAQ,GAA2B;QACvC,cAAc,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,MAAM;QACjE,QAAQ,EAAE,KAAK,CAAC,OAAO,EAAE,WAAW,EAAE,KAAK,CAAC,UAAU,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK;KAClF,CAAC;IACF,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC7D,IAAI,OAAO,CAAC,MAAM,GAAG,kBAAkB;QAAE,MAAM,cAAc,CAAC,+CAA+C,CAAC,CAAC;IAC/G,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAAC,KAahD;IACC,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;IAC7C,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,IAAI,CAAC,cAAc,EAAE,IAAI,IAAI,CAAC,KAAK,KAAK,CAAC,IAAI,IAAI,CAAC,IAAI,GAAG,kBAAkB;QAAE,MAAM,cAAc,CAAC,mDAAmD,CAAC,CAAC;IAC7K,MAAM,QAAQ,GAAG,yBAAyB,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,KAAK,CAAC,YAAY,EAAE,MAAM,CAAC,CAAY,CAAC,CAAC;IAC9G,sBAAsB,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;IAC1D,MAAM,OAAO,GAAG,WAAW,CAAC,QAAQ,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IACvD,IAAI,QAAQ,CAAC,KAAK,CAAC,SAAS,KAAK,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;QAAE,MAAM,cAAc,CAAC,0DAA0D,CAAC,CAAC;IAC1I,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,QAAQ,EAAE,YAAY,EAAE,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IAC3J,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;IACnE,MAAM,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,kBAAkB,CAAC,CAAC;IACtE,IAAI,KAAK,CAAC,MAAM,EAAE,OAAO;QAAE,MAAM,KAAK,CAAC,MAAM,CAAC,MAAM,IAAI,IAAI,YAAY,CAAC,2BAA2B,EAAE,YAAY,CAAC,CAAC;IACpH,MAAM,EAAE,CAAC,gBAAgB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAC7D,IAAI,CAAC;QACH,MAAM,mBAAmB,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC;QACrD,MAAM,SAAS,GAAG,sBAAsB,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,gBAAgB,CAAC,CAAC,CAAC,CAAC;QACvG,IAAI,SAAS,CAAC,QAAQ,KAAK,KAAK,CAAC,KAAK,CAAC,QAAQ,IAAI,SAAS,CAAC,SAAS,KAAK,KAAK,CAAC,KAAK,CAAC,SAAS;eAC3F,SAAS,CAAC,mBAAmB,KAAK,KAAK,CAAC,KAAK,CAAC,mBAAmB,IAAI,SAAS,CAAC,OAAO,KAAK,KAAK,CAAC,KAAK,CAAC,OAAO;eAC9G,SAAS,CAAC,OAAO,KAAK,KAAK,CAAC,KAAK,CAAC,OAAO,IAAI,SAAS,CAAC,QAAQ,KAAK,KAAK,CAAC,KAAK,CAAC,QAAQ;eACxF,SAAS,CAAC,WAAW,KAAK,KAAK,CAAC,KAAK,CAAC,KAAK,IAAI,SAAS,CAAC,OAAO,KAAK,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;eAC3F,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,WAAW,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;YACtF,MAAM,cAAc,CAAC,2DAA2D,CAAC,CAAC;QACpF,CAAC;QACD,MAAM,8BAA8B,CAAC,eAAe,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;QACzE,MAAM,GAAG,GAAG,MAAM,kBAAkB,CAAC;YACnC,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,OAAO;YAC3B,aAAa,EAAE,KAAK,CAAC,aAAa;YAClC,kBAAkB,EAAE,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,KAAK,CAAC;YACtD,eAAe,EAAE,KAAK,CAAC,IAAI,CAAC,eAAyB;YACrD,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;YACxC,WAAW,EAAE,KAAK,CAAC,OAAO,CAAC,YAAY;YACvC,iBAAiB,EAAE,KAAK,CAAC,OAAO,CAAC,kBAAkB;YACnD,GAAG,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,KAAK,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5E,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC1D,iBAAiB,EAAE,SAAS;YAC5B,GAAG,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,KAAK,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAClF,GAAG,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/E,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACjD,qBAAqB,EAAE,IAAI;SAC5B,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;QACnB,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,gBAAgB,EAAE,CAAC;IAC1D,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,EAAE,CAAC,gBAAgB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC7D,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,KAAK,UAAU,8BAA8B,CAAC,eAAuB,EAAE,QAAgC,EAAE,OAAe;IACtH,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAA0B,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,eAAe,CAAC,CAAC,CAAC;IACtG,IAAI,OAAO,QAAQ,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QACvF,MAAM,cAAc,CAAC,2DAA2D,CAAC,CAAC;IACpF,CAAC;IACD,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,sBAAsB,CAAC,EAAE;QACxE,cAAc,EAAE,GAAG,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO;KAC3F,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,yBAAyB,CAAC,KAAc;IACtD,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC,gBAAgB,EAAE,SAAS,EAAE,SAAS,EAAE,UAAU,EAAE,aAAa,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,wBAAwB,CAAC,CAAC;IAC7I,yBAAyB,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,QAAQ,EAAE,UAAU,EAAE,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC;IACxI,IAAI,MAAM,CAAC,cAAc,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC;WAChH,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,SAAS;QAAE,MAAM,cAAc,CAAC,mCAAmC,CAAC,CAAC;IAC3J,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IAC1C,MAAM,MAAM,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACzG,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,MAAM;QAAE,MAAM,cAAc,CAAC,kDAAkD,CAAC,CAAC;IAChM,OAAO;QACL,cAAc,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,OAAiB,EAAE,OAAO,EAAE,MAAM,CAAC,OAAiB;QACzF,QAAQ,EAAE,MAAM,CAAC,QAAkB,EAAE,WAAW,EAAE,MAAM,CAAC,WAAqB;QAC9E,KAAK,EAAE,MAAM,CAAC,KAAgC,EAAE,KAAK;KACtD,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,IAAY,EAAE,QAAQ,GAAG,EAAE;IACnD,MAAM,SAAS,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC5E,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,SAAS,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IAClE,MAAM,KAAK,GAAyB,EAAE,CAAC;IACvC,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3G,MAAM,KAAK,GAAG,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC;QAClE,gBAAgB,CAAC,KAAK,CAAC,CAAC;QACxB,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;QACpD,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,CAAC;QACjC,IAAI,IAAI,CAAC,WAAW,EAAE;YAAE,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,UAAU,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;aAChE,IAAI,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,IAAI,IAAI,CAAC,KAAK,KAAK,CAAC,EAAE,CAAC;YACrE,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,CAAC;YACvC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,EAAE,cAAc,EAAE,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QACzH,CAAC;;YAAM,MAAM,cAAc,CAAC,kDAAkD,KAAK,EAAE,CAAC,CAAC;QACvF,IAAI,KAAK,CAAC,MAAM,GAAG,SAAS;YAAE,MAAM,cAAc,CAAC,yCAAyC,CAAC,CAAC;IAChG,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAC9F,CAAC;AAED,KAAK,UAAU,mBAAmB,CAAC,QAAgC,EAAE,WAAmB;IACtF,MAAM,KAAK,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAC3D,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QAClC,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,QAAQ,CAAC,CAAC;QAC3D,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC;QACxB,IAAI,OAAO,CAAC,MAAM,KAAK,IAAI,CAAC,IAAI,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC,MAAM,IAAI,KAAK,GAAG,kBAAkB;YAAE,MAAM,cAAc,CAAC,wCAAwC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QAC7K,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;QAC/D,MAAM,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QACtC,MAAM,SAAS,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAChE,CAAC;AACH,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAC/B,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,gBAAgB,CAAC,EAAE,oBAAoB,CAAC,CAAC;IAChG,MAAM,QAAQ,GAAG,gBAAgB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC/C,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,IAAK,MAAM,CAAC,IAAe,GAAG,CAAC,IAAK,MAAM,CAAC,IAAe,GAAG,kBAAkB;WAChH,OAAO,MAAM,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,uBAAuB,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;WACjF,OAAO,MAAM,CAAC,cAAc,KAAK,QAAQ,IAAI,CAAC,kEAAkE,CAAC,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,CAAC;QAClJ,MAAM,cAAc,CAAC,+BAA+B,CAAC,CAAC;IACxD,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,CAAC,IAAc,EAAE,MAAM,EAAE,MAAM,CAAC,MAAgB,EAAE,cAAc,EAAE,MAAM,CAAC,cAAc,EAAE,CAAC;AACjI,CAAC;AAED,SAAS,sBAAsB,CAAC,QAAgC,EAAE,IAAuB,EAAE,KAAuB;IAChH,IAAI,QAAQ,CAAC,OAAO,KAAK,IAAI,CAAC,OAAO,IAAI,QAAQ,CAAC,OAAO,KAAK,KAAK,CAAC,OAAO,IAAI,QAAQ,CAAC,OAAO,KAAK,IAAI,CAAC,OAAO;WAC3G,QAAQ,CAAC,OAAO,KAAK,KAAK,CAAC,OAAO,IAAI,QAAQ,CAAC,QAAQ,KAAK,KAAK,CAAC,QAAQ,IAAI,QAAQ,CAAC,WAAW,KAAK,KAAK,CAAC,KAAK;WAClH,IAAI,CAAC,eAAe,KAAK,IAAI,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,cAAc,CAAC,0DAA0D,CAAC,CAAC;AACrJ,CAAC;AAED,SAAS,yBAAyB,CAAC,KAAkF;IACnH,IAAI,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;WAC5E,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;WAC7E,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC;WAChF,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,UAAU,CAAC,IAAK,KAAK,CAAC,UAAqB,GAAG,CAAC;QAAE,MAAM,cAAc,CAAC,6CAA6C,CAAC,CAAC;AACxJ,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAc;IACtC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,IAAI,KAAK,KAAK,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC;WAC9H,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,IAAI,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,IAAI,CAAC;QAAE,MAAM,cAAc,CAAC,8BAA8B,CAAC,CAAC;IACnI,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,WAAW,CAAC,KAAc;IACjC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,oCAAoC,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,MAAM,cAAc,CAAC,kCAAkC,CAAC,CAAC;IAC7I,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,KAAK,CAAC,KAAc,EAAE,IAAc,EAAE,KAAa;IAC1D,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,cAAc,CAAC,GAAG,KAAK,oBAAoB,CAAC,CAAC;IACpH,MAAM,MAAM,GAAG,KAAgC,CAAC;IAChD,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QAAE,MAAM,cAAc,CAAC,GAAG,KAAK,qBAAqB,CAAC,CAAC;IAChH,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,MAAM,CAAC,KAAa,IAAY,OAAO,UAAU,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;AAC/G,SAAS,cAAc,CAAC,OAAe,IAAgB,OAAO,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,uBAAuB,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC"}

package/dist/src/control-plane/remote-work-coordinator.js

@@ -0,0 +1,333 @@
+import path from "node:path";
+import { HitchError } from "../foundation/index.js";
+import { DEFAULT_EXECUTION_LEASE_HEARTBEAT_MS, DEFAULT_EXECUTION_LEASE_TTL_MS, createExecutionLease, markExecutionLeaseLost, releaseExecutionLease } from "../evals/index.js";
+import { evalTaskCollisionKey } from "./eval-records.js";
+import { importRemoteResultEnvelope } from "./remote-result-transport.js";
+import { DEFAULT_REMOTE_WORKER_RECONNECT_TIMEOUT_MS, recoverRemoteWorkerEvalLeases } from "./remote-work-recovery.js";
+import { prepareRemoteWorkInputs } from "./remote-work-inputs.js";
+import { subtractResourceVectors } from "./resources.js";
+export class RemoteWorkCoordinator {
+    execute;
+    root;
+    registry;
+    protocol;
+    collisions;
+    pollIntervalMs;
+    releaseTimeoutMs;
+    reconnectTimeoutMs;
+    constructor(input) {
+        this.root = input.root;
+        this.registry = input.registry;
+        this.protocol = input.protocol;
+        this.collisions = input.collisions;
+        this.pollIntervalMs = boundedInterval(input.pollIntervalMs ?? 100, "remote work poll interval");
+        this.releaseTimeoutMs = boundedInterval(input.releaseTimeoutMs ?? 10_000, "remote work release timeout");
+        this.reconnectTimeoutMs = boundedReconnectTimeout(input.reconnectTimeoutMs ?? DEFAULT_REMOTE_WORKER_RECONNECT_TIMEOUT_MS);
+        this.execute = (execution) => this.executeWork(execution);
+    }
+    async providerStatus(provider) {
+        const candidates = await this.providerWorkers(provider);
+        if (candidates.length === 0)
+            return null;
+        const first = candidates.sort(workerOrder)[0];
+        return first.provider_status;
+    }
+    async providerStatuses(provider) {
+        return (await this.providerWorkers(provider)).sort(workerOrder).map((worker) => worker.provider_status);
+    }
+    async canEverFit(provider, reservation) {
+        return (await this.providerWorkers(provider)).some((worker) => fits(reservation, worker.worker.capacity.allocatable));
+    }
+    recoverEvalLeases(input) {
+        return recoverRemoteWorkerEvalLeases({
+            ...input, root: this.root, registry: this.registry, protocol: this.protocol,
+            pollIntervalMs: this.pollIntervalMs, releaseTimeoutMs: this.releaseTimeoutMs, reconnectTimeoutMs: this.reconnectTimeoutMs,
+        });
+    }
+    async providerWorkers(provider) {
+        return (await this.registry.list()).filter((record) => record.worker.status === "ready" && !record.revoked_at && record.worker.provider === provider);
+    }
+    async executeWork(input) {
+        const credentialNames = this.protocol.credentialNamesFor(input.request.pass_env);
+        const inputs = await prepareRemoteWorkInputs({
+            root: input.root, request: input.request, plan: input.plan, work: input.workItem,
+            resolvedRevision: input.resolvedRevision, preparedArtifact: input.preparedArtifact,
+            runtimeDirectory: input.runtimeDirectory, runtimeId: input.runtimeId,
+            credentialNames,
+        });
+        const dispatch = await this.dispatch(input, inputs, credentialNames);
+        const { worker, lease, offer, collision } = dispatch;
+        let accepted = false;
+        let terminal = false;
+        let released = false;
+        let terminalOffer;
+        await input.onLeaseState(lease.leaseId, "running");
+        input.emit({ type: "eval.work.leased", work_id: input.workItem.work_id, lease_id: lease.leaseId, worker_id: worker.worker.worker_id, reservation: input.workItem.reservation });
+        input.emit({ type: "lease.offered", work_id: input.workItem.work_id, lease_id: lease.leaseId, worker_id: worker.worker.worker_id, offer_id: offer.offer_id });
+        try {
+            const acceptedOffer = await this.waitFor(input, offer, (current) => current.state !== "offered");
+            if (!acceptedOrLater(acceptedOffer))
+                throw new HitchError(`remote worker did not accept work: ${acceptedOffer.state}`, { code: "worker_rejected", exitCode: 10 });
+            accepted = true;
+            await lease.accept();
+            await lease.markRunning();
+            input.emit({ type: "lease.accepted", work_id: input.workItem.work_id, lease_id: lease.leaseId, worker_id: worker.worker.worker_id, offer_id: offer.offer_id });
+            input.emit({ type: "eval.work.started", work_id: input.workItem.work_id, lease_id: lease.leaseId, worker_id: worker.worker.worker_id });
+            const completed = await this.withHeartbeat(lease, async () => this.waitFor(input, acceptedOffer, (current) => current.state === "completed" || current.state === "release-requested" || current.state === "released"));
+            terminal = true;
+            terminalOffer = completed;
+            if (!completed.terminal)
+                throw ambiguous("remote worker completed without terminal evidence");
+            const artifacts = completed.terminal.artifacts.filter((artifact) => artifact.kind === "result-bundle");
+            if (completed.terminal.status !== "succeeded") {
+                await this.finishRelease(input, completed, lease.current());
+                released = true;
+                return { leaseId: lease.leaseId, refs: [], run: remoteBackendResult(worker, offer, completed, null, null) };
+            }
+            if (artifacts.length !== 1)
+                throw ambiguous("remote worker success requires exactly one result bundle");
+            const artifact = artifacts[0];
+            const imported = await importRemoteResultEnvelope({
+                root: input.root,
+                evalDirectory: input.evalDirectory,
+                request: input.request,
+                resolvedRevision: input.resolvedRevision,
+                work: input.workItem,
+                lease: lease.current(),
+                artifactPath: this.protocol.artifactPath(worker.worker.worker_id, lease.leaseId, artifact.digest),
+                runtimeId: input.runtimeId,
+                ...(input.signal ? { signal: input.signal } : {}),
+                ...(input.environmentImages ? { environmentImages: input.environmentImages } : {}),
+                ...(input.modelCapturePlan ? { modelCapturePlan: input.modelCapturePlan } : {}),
+                ...(input.publicationMode ? { publicationMode: input.publicationMode } : {}),
+            });
+            await input.publish(imported.ref);
+            input.emit({ type: "eval.work.completed", work_id: input.workItem.work_id, lease_id: lease.leaseId, worker_id: worker.worker.worker_id, run_id: imported.ref.run_id });
+            await this.finishRelease(input, completed, lease.current());
+            released = true;
+            return {
+                leaseId: lease.leaseId,
+                refs: [imported.ref],
+                run: remoteBackendResult(worker, offer, completed, imported.trial, imported.backendDirectory),
+            };
+        }
+        catch (error) {
+            input.emit({ type: "eval.work.lost", work_id: input.workItem.work_id, lease_id: lease.leaseId, worker_id: worker.worker.worker_id, code: error.code || "remote_work_failed" });
+            if (terminalOffer && !released) {
+                await this.finishRelease(input, terminalOffer, lease.current()).catch(() => undefined);
+            }
+            if (input.signal?.aborted)
+                await this.protocol.requestCancel(worker.worker.worker_id, offer.offer_id).catch(() => undefined);
+            if (!accepted)
+                await this.withdrawOrFenceAcceptedRace(input.evalDirectory, offer, lease);
+            else if (!terminal)
+                await markExecutionLeaseLost({ evalDirectory: input.evalDirectory, leaseId: lease.leaseId, expectedEpoch: lease.current().epoch }).catch(() => undefined);
+            throw error;
+        }
+        finally {
+            collision.release();
+            await input.onLeaseState(lease.leaseId, "terminal");
+        }
+    }
+    async dispatch(input, inputs, credentialNames) {
+        for (;;) {
+            if (input.signal?.aborted)
+                throw cancelled();
+            const registered = (await this.registry.list()).filter((worker) => !worker.revoked_at && worker.worker.provider === input.workItem.provider);
+            const capable = registered.filter((worker) => supports(worker, input.workItem, input.preparedArtifact.platform, input.modelCapturePlan));
+            if (registered.length > 0 && capable.length === 0) {
+                throw new HitchError(`no remote worker supports ${input.workItem.backend} on ${input.preparedArtifact.platform}`, {
+                    code: "execution_provider_unavailable", exitCode: 10,
+                });
+            }
+            if (capable.length > 0 && !capable.some((worker) => fits(input.workItem.reservation, worker.worker.capacity.allocatable))) {
+                throw new HitchError("remote work item exceeds every compatible worker capacity", {
+                    code: "resource_request_unsatisfiable", exitCode: 10,
+                });
+            }
+            const workers = capable.filter((worker) => compatible(worker, input.workItem));
+            for (const worker of workers.sort(workerOrder)) {
+                const collisionKey = evalTaskCollisionKey(input.request, input.workItem.task_ids[0], worker.worker.collision_domain_id);
+                const collision = this.collisions.tryAcquire(`${input.evalId}:${input.workItem.work_id}`, [collisionKey]);
+                if (!collision)
+                    continue;
+                const lease = await createExecutionLease({
+                    evalDirectory: input.evalDirectory, evalId: input.evalId, workId: input.workItem.work_id,
+                    worker: {
+                        workerId: worker.worker.worker_id,
+                        provider: worker.worker.provider,
+                        collisionDomainId: worker.worker.collision_domain_id,
+                    },
+                    reservation: input.workItem.reservation,
+                    ttlMs: DEFAULT_EXECUTION_LEASE_TTL_MS,
+                    initialState: "offered",
+                });
+                try {
+                    const offer = await this.protocol.createOffer(worker.worker.worker_id, lease.current(), input.workItem, inputs, credentialNames);
+                    return { worker, lease, offer, collision };
+                }
+                catch (error) {
+                    await lease.release().catch(() => undefined);
+                    collision.release();
+                    if (error.code !== "worker_rejected" && error.code !== "worker_unavailable")
+                        throw error;
+                }
+            }
+            await delay(this.pollIntervalMs, input.signal);
+        }
+    }
+    async waitFor(input, initial, ready) {
+        let offer = initial;
+        let unavailableSince;
+        let reportedUnavailable = false;
+        for (;;) {
+            if (ready(offer))
+                return offer;
+            if (input.signal?.aborted)
+                throw cancelled();
+            await delay(this.pollIntervalMs, input.signal);
+            offer = await this.protocol.getOffer(offer.worker_id, offer.offer_id) ?? (() => { throw ambiguous("remote work offer disappeared"); })();
+            if (offer.state === "rejected" || offer.state === "expired")
+                return offer;
+            if (ready(offer))
+                return offer;
+            const worker = await this.registry.get(offer.worker_id);
+            if (worker?.revoked_at)
+                throw ambiguous(`remote worker was revoked: ${offer.worker_id}`);
+            if (workerAvailableForOffer(worker, offer)) {
+                if (reportedUnavailable)
+                    input.emit({ type: "worker.reconnected", worker_id: offer.worker_id, lease_id: offer.lease.lease_id, lease_epoch: offer.lease.epoch });
+                unavailableSince = undefined;
+                reportedUnavailable = false;
+            }
+            else {
+                unavailableSince ??= Date.now();
+                if (!reportedUnavailable)
+                    input.emit({ type: "worker.heartbeat_missed", worker_id: offer.worker_id, lease_id: offer.lease.lease_id, lease_epoch: offer.lease.epoch });
+                reportedUnavailable = true;
+                if (Date.now() - unavailableSince >= this.reconnectTimeoutMs)
+                    throw ambiguous(`remote worker did not reconnect: ${offer.worker_id}`);
+            }
+        }
+    }
+    async withHeartbeat(lease, operation) {
+        let failure;
+        let tail = Promise.resolve();
+        const timer = setInterval(() => {
+            tail = tail.then(async () => {
+                await lease.heartbeat();
+            }).catch((error) => { failure ??= error; });
+        }, DEFAULT_EXECUTION_LEASE_HEARTBEAT_MS);
+        timer.unref();
+        try {
+            const result = await operation();
+            await tail;
+            if (failure !== undefined)
+                throw failure;
+            return result;
+        }
+        finally {
+            clearInterval(timer);
+            await tail;
+        }
+    }
+    async withdrawOrFenceAcceptedRace(evalDirectory, offer, lease) {
+        const withdrawn = await this.protocol.withdrawUnacceptedOffer(offer.worker_id, offer.offer_id).catch(async () => {
+            await markExecutionLeaseLost({ evalDirectory, leaseId: lease.leaseId, expectedEpoch: lease.current().epoch }).catch(() => undefined);
+            return null;
+        });
+        if (!withdrawn)
+            return;
+        if (!acceptedOrLater(withdrawn)) {
+            await lease.release().catch(() => undefined);
+            return;
+        }
+        await lease.accept().catch(() => undefined);
+        await this.protocol.requestCancel(offer.worker_id, offer.offer_id).catch(() => undefined);
+        await markExecutionLeaseLost({ evalDirectory, leaseId: lease.leaseId, expectedEpoch: lease.current().epoch }).catch(() => undefined);
+    }
+    async finishRelease(input, completed, lease) {
+        input.emit({ type: "sandbox.cleanup.started", work_id: lease.work_id, lease_id: lease.lease_id, worker_id: lease.worker_id });
+        let offer = await this.protocol.requestRelease(completed.worker_id, completed.offer_id);
+        const deadline = Date.now() + this.releaseTimeoutMs;
+        while (offer.state !== "released" && Date.now() < deadline && !input.signal?.aborted) {
+            await delay(this.pollIntervalMs, input.signal);
+            offer = await this.protocol.getOffer(offer.worker_id, offer.offer_id) ?? offer;
+        }
+        if (offer.state === "released") {
+            await this.releaseLease(input, lease);
+            input.emit({ type: "lease.released", work_id: lease.work_id, lease_id: lease.lease_id, worker_id: lease.worker_id });
+            input.emit({ type: "sandbox.cleanup.completed", work_id: lease.work_id, lease_id: lease.lease_id, worker_id: lease.worker_id, residual_resources: 0 });
+            return;
+        }
+        await markExecutionLeaseLost({ evalDirectory: input.evalDirectory, leaseId: lease.lease_id, expectedEpoch: lease.epoch });
+        input.emit({ type: "sandbox.cleanup.failed", work_id: lease.work_id, lease_id: lease.lease_id, worker_id: lease.worker_id, code: "worker_release_timeout" });
+    }
+    async releaseLease(input, lease) {
+        await releaseExecutionLease({ evalDirectory: input.evalDirectory, leaseId: lease.lease_id, expectedEpoch: lease.epoch });
+    }
+}
+function supports(worker, work, platform, capture) {
+    return worker.worker.capabilities.backends.includes(work.backend) && worker.worker.capabilities.platforms.includes(platform)
+        && (capture?.effective_mode !== "proxy" && capture?.effective_mode !== "hybrid" || capture.topology === "in-sandbox" && worker.provider_status.features.model_proxy);
+}
+function compatible(worker, work) {
+    return worker.worker.status === "ready" && fits(work.reservation, subtractResourceVectors(worker.worker.capacity.allocatable, worker.worker.capacity.allocated));
+}
+function remoteBackendResult(worker, offered, terminal, trial, backendDirectory) {
+    const directory = backendDirectory ?? path.join("remote", offered.work.work_id);
+    const succeeded = terminal.terminal?.status === "succeeded" && trial !== null;
+    return {
+        backend: {
+            name: "harbor", executable: `remote-worker:${worker.worker.worker_id}`,
+            version: worker.provider_status.backends.find((backend) => backend.id === "harbor")?.version ?? null,
+            identity: `${worker.worker.provider}:${worker.worker.worker_id}:${worker.generation}`,
+            config_path: path.join(directory, "remote-offer.json"),
+            result_path: succeeded ? path.join(directory, "remote-result.json") : null,
+            stdout_path: path.join(directory, "remote.stdout.log"), stderr_path: path.join(directory, "remote.stderr.log"),
+            process_exit_code: succeeded ? 0 : 1, signal: null, job_directory: path.join(directory, "job"),
+        },
+        rawResult: trial ? { trial_results: [trial] } : null,
+        summary: trial ? { n_trials: 1, remote_worker: worker.worker.worker_id } : null,
+    };
+}
+function workerOrder(left, right) {
+    return utilization(left.worker.capacity.allocated, left.worker.capacity.allocatable) - utilization(right.worker.capacity.allocated, right.worker.capacity.allocatable)
+        || left.worker.worker_id.localeCompare(right.worker.worker_id);
+}
+function utilization(allocated, allocatable) {
+    return Math.max(...fields().map((field) => resourceValue(allocatable, field) === 0
+        ? resourceValue(allocated, field) === 0 ? 0 : Infinity
+        : resourceValue(allocated, field) / resourceValue(allocatable, field)));
+}
+function fits(requested, capacity) { return fields().every((field) => resourceValue(requested, field) <= resourceValue(capacity, field)); }
+function acceptedOrLater(offer) {
+    return new Set(["accepted", "cancel-requested", "completed", "release-requested", "released"]).has(offer.state)
+        && typeof offer.accepted_at === "string" && typeof offer.accept_receipt_digest === "string";
+}
+function workerAvailableForOffer(worker, offer) {
+    if (worker?.worker.status !== "ready")
+        return false;
+    if (offer.state === "offered")
+        return true;
+    return worker.active_leases.some((lease) => lease.lease_id === offer.lease.lease_id && lease.epoch === offer.lease.epoch);
+}
+function fields() { return ["cpu_millis", "memory_bytes", "container_slots", "build_slots", "gpu_count", "ephemeral_disk_bytes"]; }
+function resourceValue(resources, field) { return resources[field] ?? 0; }
+function boundedInterval(value, label) { if (!Number.isSafeInteger(value) || value < 1 || value > 60_000)
+    throw new TypeError(`${label} is invalid`); return value; }
+function boundedReconnectTimeout(value) { if (!Number.isSafeInteger(value) || value < 1 || value > 5 * 60_000)
+    throw new TypeError("remote worker reconnect timeout is invalid"); return value; }
+function ambiguous(message) { return new HitchError(message, { code: "execution_state_ambiguous", exitCode: 12 }); }
+function cancelled() { return new HitchError("remote work was cancelled", { code: "cancelled", exitCode: 9 }); }
+function delay(milliseconds, signal) {
+    return new Promise((resolve, reject) => {
+        const timer = setTimeout(done, milliseconds);
+        const abort = () => done(cancelled());
+        function done(error) { clearTimeout(timer); signal?.removeEventListener("abort", abort); error ? reject(error) : resolve(); }
+        signal?.addEventListener("abort", abort, { once: true });
+        if (signal?.aborted)
+            abort();
+    });
+}
+//# sourceMappingURL=remote-work-coordinator.js.map

package/dist/src/control-plane/remote-work-coordinator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"remote-work-coordinator.js","sourceRoot":"","sources":["../../../src/control-plane/remote-work-coordinator.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,OAAO,EAAE,oCAAoC,EAAE,8BAA8B,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AAG9K,OAAO,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AACzD,OAAO,EAAE,0BAA0B,EAAE,MAAM,8BAA8B,CAAC;AAG1E,OAAO,EAAE,0CAA0C,EAAE,6BAA6B,EAAE,MAAM,2BAA2B,CAAC;AACtH,OAAO,EAAE,uBAAuB,EAAE,MAAM,yBAAyB,CAAC;AAClE,OAAO,EAAE,uBAAuB,EAAE,MAAM,gBAAgB,CAAC;AAYzD,MAAM,OAAO,qBAAqB;IACvB,OAAO,CAAyB;IACxB,IAAI,CAAS;IACb,QAAQ,CAAuB;IAC/B,QAAQ,CAAuB;IAC/B,UAAU,CAAuB;IACjC,cAAc,CAAS;IACvB,gBAAgB,CAAS;IACzB,kBAAkB,CAAS;IAE5C,YAAY,KAAmC;QAC7C,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC;QACvB,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC;QAC/B,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC;QAC/B,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC,UAAU,CAAC;QACnC,IAAI,CAAC,cAAc,GAAG,eAAe,CAAC,KAAK,CAAC,cAAc,IAAI,GAAG,EAAE,2BAA2B,CAAC,CAAC;QAChG,IAAI,CAAC,gBAAgB,GAAG,eAAe,CAAC,KAAK,CAAC,gBAAgB,IAAI,MAAM,EAAE,6BAA6B,CAAC,CAAC;QACzG,IAAI,CAAC,kBAAkB,GAAG,uBAAuB,CAAC,KAAK,CAAC,kBAAkB,IAAI,0CAA0C,CAAC,CAAC;QAC1H,IAAI,CAAC,OAAO,GAAG,CAAC,SAAS,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;IAC5D,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,QAAgB;QACnC,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;QACxD,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QACzC,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAA+B,CAAC;QAC5E,OAAO,KAAK,CAAC,eAAe,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,QAAgB;QACrC,OAAO,CAAC,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAC1G,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,QAAgB,EAAE,WAA6B;QAC9D,OAAO,CAAC,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC;IACxH,CAAC;IAED,iBAAiB,CAAC,KAAiK;QACjL,OAAO,6BAA6B,CAAC;YACnC,GAAG,KAAK,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ;YAC3E,cAAc,EAAE,IAAI,CAAC,cAAc,EAAE,gBAAgB,EAAE,IAAI,CAAC,gBAAgB,EAAE,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;SAC1H,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,eAAe,CAAC,QAAgB;QAC5C,OAAO,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,KAAK,OAAO,IAAI,CAAC,MAAM,CAAC,UAAU,IAAI,MAAM,CAAC,MAAM,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC;IACxJ,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,KAA4C;QACpE,MAAM,eAAe,GAAG,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACjF,MAAM,MAAM,GAAG,MAAM,uBAAuB,CAAC;YAC3C,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,QAAQ;YAChF,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,EAAE,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;YAClF,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS;YACpE,eAAe;SAChB,CAAC,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,eAAe,CAAC,CAAC;QACrE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,QAAQ,CAAC;QACrD,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,IAAI,aAA4C,CAAC;QACjD,MAAM,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QACnD,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,kBAAkB,EAAE,OAAO,EAAE,KAAK,CAAC,QAAQ,CAAC,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,WAAW,EAAE,KAAK,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;QAChL,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,OAAO,EAAE,KAAK,CAAC,QAAQ,CAAC,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC9J,IAAI,CAAC;YACH,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;YACjG,IAAI,CAAC,eAAe,CAAC,aAAa,CAAC;gBAAE,MAAM,IAAI,UAAU,CAAC,sCAAsC,aAAa,CAAC,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,iBAAiB,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;YAClK,QAAQ,GAAG,IAAI,CAAC;YAChB,MAAM,KAAK,CAAC,MAAM,EAAE,CAAC;YACrB,MAAM,KAAK,CAAC,WAAW,EAAE,CAAC;YAC1B,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,KAAK,CAAC,QAAQ,CAAC,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;YAC/J,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,mBAAmB,EAAE,OAAO,EAAE,KAAK,CAAC,QAAQ,CAAC,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;YACxI,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,KAAK,IAAI,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,aAAa,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,KAAK,WAAW,IAAI,OAAO,CAAC,KAAK,KAAK,mBAAmB,IAAI,OAAO,CAAC,KAAK,KAAK,UAAU,CAAC,CAAC,CAAC;YACvN,QAAQ,GAAG,IAAI,CAAC;YAChB,aAAa,GAAG,SAAS,CAAC;YAC1B,IAAI,CAAC,SAAS,CAAC,QAAQ;gBAAE,MAAM,SAAS,CAAC,mDAAmD,CAAC,CAAC;YAC9F,MAAM,SAAS,GAAG,SAAS,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,KAAK,eAAe,CAAC,CAAC;YACvG,IAAI,SAAS,CAAC,QAAQ,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;gBAC9C,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,SAAS,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;gBAC5D,QAAQ,GAAG,IAAI,CAAC;gBAChB,OAAO,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,GAAG,EAAE,mBAAmB,CAAC,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE,CAAC;YAC9G,CAAC;YACD,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;gBAAE,MAAM,SAAS,CAAC,0DAA0D,CAAC,CAAC;YACxG,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAA6B,CAAC;YAC1D,MAAM,QAAQ,GAAG,MAAM,0BAA0B,CAAC;gBAChD,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,aAAa,EAAE,KAAK,CAAC,aAAa;gBAClC,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;gBACxC,IAAI,EAAE,KAAK,CAAC,QAAQ;gBACpB,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE;gBACtB,YAAY,EAAE,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,KAAK,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC;gBACjG,SAAS,EAAE,KAAK,CAAC,SAAS;gBAC1B,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACjD,GAAG,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,KAAK,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAClF,GAAG,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC/E,GAAG,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,KAAK,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC7E,CAAC,CAAC;YACH,MAAM,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;YAClC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,OAAO,EAAE,KAAK,CAAC,QAAQ,CAAC,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;YACvK,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,SAAS,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YAC5D,QAAQ,GAAG,IAAI,CAAC;YAChB,OAAO;gBACL,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,IAAI,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC;gBACpB,GAAG,EAAE,mBAAmB,CAAC,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,gBAAgB,CAAC;aAC9F,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,KAAK,CAAC,QAAQ,CAAC,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,IAAI,EAAG,KAA2B,CAAC,IAAI,IAAI,oBAAoB,EAAE,CAAC,CAAC;YACtM,IAAI,aAAa,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAC/B,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,aAAa,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YACzF,CAAC;YACD,IAAI,KAAK,CAAC,MAAM,EAAE,OAAO;gBAAE,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YAC7H,IAAI,CAAC,QAAQ;gBAAE,MAAM,IAAI,CAAC,2BAA2B,CAAC,KAAK,CAAC,aAAa,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;iBACpF,IAAI,CAAC,QAAQ;gBAAE,MAAM,sBAAsB,CAAC,EAAE,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,aAAa,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YAC9K,MAAM,KAAK,CAAC;QACd,CAAC;gBAAS,CAAC;YACT,SAAS,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;QACtD,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,QAAQ,CAAC,KAA4C,EAAE,MAA8B,EAAE,eAAkC;QACrI,SAAS,CAAC;YACR,IAAI,KAAK,CAAC,MAAM,EAAE,OAAO;gBAAE,MAAM,SAAS,EAAE,CAAC;YAC7C,MAAM,UAAU,GAAG,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,UAAU,IAAI,MAAM,CAAC,MAAM,CAAC,QAAQ,KAAK,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YAC7I,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,EAAE,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,gBAAgB,CAAC,QAAQ,EAAE,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC;YACzI,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAClD,MAAM,IAAI,UAAU,CAAC,6BAA6B,KAAK,CAAC,QAAQ,CAAC,OAAO,OAAO,KAAK,CAAC,gBAAgB,CAAC,QAAQ,EAAE,EAAE;oBAChH,IAAI,EAAE,gCAAgC,EAAE,QAAQ,EAAE,EAAE;iBACrD,CAAC,CAAC;YACL,CAAC;YACD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC;gBAC1H,MAAM,IAAI,UAAU,CAAC,2DAA2D,EAAE;oBAChF,IAAI,EAAE,gCAAgC,EAAE,QAAQ,EAAE,EAAE;iBACrD,CAAC,CAAC;YACL,CAAC;YACD,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,UAAU,CAAC,MAAM,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC/E,KAAK,MAAM,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC/C,MAAM,YAAY,GAAG,oBAAoB,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAW,EAAE,MAAM,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC;gBAClI,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC;gBAC1G,IAAI,CAAC,SAAS;oBAAE,SAAS;gBACzB,MAAM,KAAK,GAAG,MAAM,oBAAoB,CAAC;oBACvC,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,QAAQ,CAAC,OAAO;oBACxF,MAAM,EAAE;wBACN,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,SAAS;wBACjC,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,QAAQ;wBAChC,iBAAiB,EAAE,MAAM,CAAC,MAAM,CAAC,mBAAmB;qBACrD;oBACD,WAAW,EAAE,KAAK,CAAC,QAAQ,CAAC,WAAW;oBACvC,KAAK,EAAE,8BAA8B;oBACrC,YAAY,EAAE,SAAS;iBACxB,CAAC,CAAC;gBACH,IAAI,CAAC;oBACH,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,KAAK,CAAC,OAAO,EAAE,EAAE,KAAK,CAAC,QAAQ,EAAE,MAAM,EAAE,eAAe,CAAC,CAAC;oBACjI,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;gBAC7C,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;oBAC7C,SAAS,CAAC,OAAO,EAAE,CAAC;oBACpB,IAAK,KAA2B,CAAC,IAAI,KAAK,iBAAiB,IAAK,KAA2B,CAAC,IAAI,KAAK,oBAAoB;wBAAE,MAAM,KAAK,CAAC;gBACzI,CAAC;YACH,CAAC;YACD,MAAM,KAAK,CAAC,IAAI,CAAC,cAAc,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;QACjD,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,OAAO,CAAC,KAA4C,EAAE,OAA0B,EAAE,KAA4C;QAC1I,IAAI,KAAK,GAAG,OAAO,CAAC;QACpB,IAAI,gBAAoC,CAAC;QACzC,IAAI,mBAAmB,GAAG,KAAK,CAAC;QAChC,SAAS,CAAC;YACR,IAAI,KAAK,CAAC,KAAK,CAAC;gBAAE,OAAO,KAAK,CAAC;YAC/B,IAAI,KAAK,CAAC,MAAM,EAAE,OAAO;gBAAE,MAAM,SAAS,EAAE,CAAC;YAC7C,MAAM,KAAK,CAAC,IAAI,CAAC,cAAc,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;YAC/C,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,SAAS,CAAC,+BAA+B,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACzI,IAAI,KAAK,CAAC,KAAK,KAAK,UAAU,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS;gBAAE,OAAO,KAAK,CAAC;YAC1E,IAAI,KAAK,CAAC,KAAK,CAAC;gBAAE,OAAO,KAAK,CAAC;YAC/B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;YACxD,IAAI,MAAM,EAAE,UAAU;gBAAE,MAAM,SAAS,CAAC,8BAA8B,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC;YACzF,IAAI,uBAAuB,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC;gBAC3C,IAAI,mBAAmB;oBAAE,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,oBAAoB,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,QAAQ,EAAE,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,WAAW,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;gBAChK,gBAAgB,GAAG,SAAS,CAAC;gBAC7B,mBAAmB,GAAG,KAAK,CAAC;YAC9B,CAAC;iBAAM,CAAC;gBACN,gBAAgB,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC;gBAChC,IAAI,CAAC,mBAAmB;oBAAE,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,yBAAyB,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,QAAQ,EAAE,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,WAAW,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;gBACtK,mBAAmB,GAAG,IAAI,CAAC;gBAC3B,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,gBAAgB,IAAI,IAAI,CAAC,kBAAkB;oBAAE,MAAM,SAAS,CAAC,oCAAoC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC;YACvI,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,aAAa,CAAI,KAAuD,EAAE,SAA2B;QACjH,IAAI,OAAgB,CAAC;QACrB,IAAI,IAAI,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;QAC7B,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE;YAC7B,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;gBAC1B,MAAM,KAAK,CAAC,SAAS,EAAE,CAAC;YAC1B,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,GAAG,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9C,CAAC,EAAE,oCAAoC,CAAC,CAAC;QACzC,KAAK,CAAC,KAAK,EAAE,CAAC;QACd,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,SAAS,EAAE,CAAC;YACjC,MAAM,IAAI,CAAC;YACX,IAAI,OAAO,KAAK,SAAS;gBAAE,MAAM,OAAO,CAAC;YACzC,OAAO,MAAM,CAAC;QAChB,CAAC;gBAAS,CAAC;YACT,aAAa,CAAC,KAAK,CAAC,CAAC;YACrB,MAAM,IAAI,CAAC;QACb,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,2BAA2B,CAAC,aAAqB,EAAE,KAAwB,EAAE,KAAuD;QAChJ,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,uBAAuB,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE;YAC9G,MAAM,sBAAsB,CAAC,EAAE,aAAa,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,aAAa,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YACrI,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,SAAS;YAAE,OAAO;QACvB,IAAI,CAAC,eAAe,CAAC,SAAS,CAAC,EAAE,CAAC;YAChC,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YAC7C,OAAO;QACT,CAAC;QACD,MAAM,KAAK,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAC5C,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAC1F,MAAM,sBAAsB,CAAC,EAAE,aAAa,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,aAAa,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IACvI,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,KAA4C,EAAE,SAA4B,EAAE,KAAuB;QAC7H,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,yBAAyB,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC;QAC9H,IAAI,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC,SAAS,CAAC,SAAS,EAAE,SAAS,CAAC,QAAQ,CAAC,CAAC;QACxF,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,gBAAgB,CAAC;QACpD,OAAO,KAAK,CAAC,KAAK,KAAK,UAAU,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;YACrF,MAAM,KAAK,CAAC,IAAI,CAAC,cAAc,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;YAC/C,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC;QACjF,CAAC;QACD,IAAI,KAAK,CAAC,KAAK,KAAK,UAAU,EAAE,CAAC;YAC/B,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;YACtC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC;YACrH,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,2BAA2B,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,kBAAkB,EAAE,CAAC,EAAE,CAAC,CAAC;YACvJ,OAAO;QACT,CAAC;QACD,MAAM,sBAAsB,CAAC,EAAE,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE,OAAO,EAAE,KAAK,CAAC,QAAQ,EAAE,aAAa,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;QAC1H,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,wBAAwB,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,IAAI,EAAE,wBAAwB,EAAE,CAAC,CAAC;IAC/J,CAAC;IAEO,KAAK,CAAC,YAAY,CAAC,KAA4C,EAAE,KAAuB;QAC9F,MAAM,qBAAqB,CAAC,EAAE,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE,OAAO,EAAE,KAAK,CAAC,QAAQ,EAAE,aAAa,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;IAC3H,CAAC;CACF;AAED,SAAS,QAAQ,CAAC,MAAkC,EAAE,IAAuB,EAAE,QAAgB,EAAE,OAAuD;IACtJ,OAAO,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC;WACvH,CAAC,OAAO,EAAE,cAAc,KAAK,OAAO,IAAI,OAAO,EAAE,cAAc,KAAK,QAAQ,IAAI,OAAO,CAAC,QAAQ,KAAK,YAAY,IAAI,MAAM,CAAC,eAAe,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;AACzK,CAAC;AACD,SAAS,UAAU,CAAC,MAAkC,EAAE,IAAuB;IAC7E,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,KAAK,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,uBAAuB,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;AACnK,CAAC;AAED,SAAS,mBAAmB,CAAC,MAAkC,EAAE,OAA0B,EAAE,QAA2B,EAAE,KAAqC,EAAE,gBAA+B;IAC9L,MAAM,SAAS,GAAG,gBAAgB,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAChF,MAAM,SAAS,GAAG,QAAQ,CAAC,QAAQ,EAAE,MAAM,KAAK,WAAW,IAAI,KAAK,KAAK,IAAI,CAAC;IAC9E,OAAO;QACL,OAAO,EAAE;YACP,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,iBAAiB,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE;YACtE,OAAO,EAAE,MAAM,CAAC,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,KAAK,QAAQ,CAAC,EAAE,OAAO,IAAI,IAAI;YACpG,QAAQ,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,UAAU,EAAE;YACrF,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,mBAAmB,CAAC;YACtD,WAAW,EAAE,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,oBAAoB,CAAC,CAAC,CAAC,CAAC,IAAI;YAC1E,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,mBAAmB,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,mBAAmB,CAAC;YAC9G,iBAAiB,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC;SAC/F;QACD,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI;QACpD,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,aAAa,EAAE,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,IAAI;KAChF,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAAC,IAAgC,EAAE,KAAiC;IACtF,OAAO,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC;WACjK,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,aAAa,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;AACnE,CAAC;AAED,SAAS,WAAW,CAAC,SAA2B,EAAE,WAA6B;IAC7E,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,aAAa,CAAC,WAAW,EAAE,KAAK,CAAC,KAAK,CAAC;QAChF,CAAC,CAAC,aAAa,CAAC,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ;QACtD,CAAC,CAAC,aAAa,CAAC,SAAS,EAAE,KAAK,CAAC,GAAG,aAAa,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;AAC5E,CAAC;AAED,SAAS,IAAI,CAAC,SAA2B,EAAE,QAA0B,IAAa,OAAO,MAAM,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,aAAa,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,aAAa,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AACxL,SAAS,eAAe,CAAC,KAAwB;IAC/C,OAAO,IAAI,GAAG,CAAC,CAAC,UAAU,EAAE,kBAAkB,EAAE,WAAW,EAAE,mBAAmB,EAAE,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC;WAC1G,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ,IAAI,OAAO,KAAK,CAAC,qBAAqB,KAAK,QAAQ,CAAC;AAChG,CAAC;AACD,SAAS,uBAAuB,CAAC,MAAyC,EAAE,KAAwB;IAClG,IAAI,MAAM,EAAE,MAAM,CAAC,MAAM,KAAK,OAAO;QAAE,OAAO,KAAK,CAAC;IACpD,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IAC3C,OAAO,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,KAAK,KAAK,CAAC,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,KAAK,KAAK,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;AAC5H,CAAC;AACD,SAAS,MAAM,KAAoC,OAAO,CAAC,YAAY,EAAE,cAAc,EAAE,iBAAiB,EAAE,aAAa,EAAE,WAAW,EAAE,sBAAsB,CAAC,CAAC,CAAC,CAAC;AAClK,SAAS,aAAa,CAAC,SAA2B,EAAE,KAA6B,IAAY,OAAO,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAC5H,SAAS,eAAe,CAAC,KAAa,EAAE,KAAa,IAAY,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,MAAM;IAAE,MAAM,IAAI,SAAS,CAAC,GAAG,KAAK,aAAa,CAAC,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,CAAC;AAC7L,SAAS,uBAAuB,CAAC,KAAa,IAAY,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,GAAG,MAAM;IAAE,MAAM,IAAI,SAAS,CAAC,4CAA4C,CAAC,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,CAAC;AACjN,SAAS,SAAS,CAAC,OAAe,IAAgB,OAAO,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,2BAA2B,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;AACxI,SAAS,SAAS,KAAiB,OAAO,IAAI,UAAU,CAAC,2BAA2B,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;AAE5H,SAAS,KAAK,CAAC,YAAoB,EAAE,MAAoB;IACvD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;QAC7C,MAAM,KAAK,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;QACtC,SAAS,IAAI,CAAC,KAAa,IAAU,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QAC3I,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QACzD,IAAI,MAAM,EAAE,OAAO;YAAE,KAAK,EAAE,CAAC;IAC/B,CAAC,CAAC,CAAC;AACL,CAAC"}