agent-hitch 0.2.2 → 0.2.4

package/dist/src/evals/rerun.js

@@ -0,0 +1,398 @@
+import { randomUUID } from "node:crypto";
+import path from "node:path";
+import { loadPreparedArtifact, preparedArtifactDirectory } from "../artifacts/index.js";
+import { runHarborBackend, validateLocalGitTransportManifest, verifyLocalGitTransport, } from "../backends/index.js";
+import { useControllerRuntimeById } from "../controller-runtime/index.js";
+import { HitchError, SCHEMA_VERSION, atomicWriteJSON, ensureDir, invalidInput, readJSON, statePaths, withFileLock } from "../foundation/index.js";
+import { evalTrialKey, readEvalProgress, replaceInvalidEvalProgressTrial, writeEvalProgress } from "./progress.js";
+import { validateEvalId, validateEvalRequest } from "./request.js";
+import { summarizeTrialRefs } from "./service.js";
+import { importEvalTrialRun, importEvalTrialRuns, TrialBundlePendingError, validateEvalTrialReferences } from "./trial-import.js";
+export async function rerunEval(options) {
+    if (!options.root)
+        throw invalidInput("a Hitch state root is required for eval rerun");
+    const evalId = validateEvalId(options.evalId);
+    const evalDirectory = path.join(statePaths(options.root).evals, evalId);
+    return withFileLock(path.join(evalDirectory, "reruns"), "active", () => rerunEvalLocked({ ...options, evalId, evalDirectory }), { timeoutCode: "eval_rerun_active", timeoutExitCode: 2, ...(options.signal ? { signal: options.signal } : {}) });
+}
+async function rerunEvalLocked(options) {
+    const startedAt = new Date().toISOString();
+    const rerunId = `rerun_${randomUUID().replaceAll("-", "")}`;
+    const rerunDirectory = await ensureDir(path.join(options.evalDirectory, "reruns", rerunId));
+    const statePath = path.join(rerunDirectory, "state.json");
+    const requestValue = await readJSON(path.join(options.evalDirectory, "request.json"), null);
+    if (requestValue === null)
+        throw new HitchError(`eval not found: ${options.evalId}`, { code: "eval_not_found", exitCode: 3 });
+    const request = await loadPersistedRequest(requestValue);
+    const plan = parseRerunPlan(await readJSON(path.join(options.evalDirectory, "plan.json")), options.evalId, request);
+    const initialProgress = await readEvalProgress(options.evalDirectory);
+    if (initialProgress === null)
+        throw unavailable("eval has no task-level progress");
+    let progress = initialProgress;
+    validateProgressPlan(progress, plan, request, options.evalId);
+    const previousResult = await readJSON(path.join(options.evalDirectory, "result.json"), null);
+    if (previousResult?.status === "cancelled")
+        throw new HitchError("cancelled eval cannot be rerun", { code: "eval_rerun_cancelled", exitCode: 2 });
+    const selectedTasks = selectRerunTasks(plan.tasks, progress, options.selector);
+    await atomicWriteJSON(path.join(rerunDirectory, "request.json"), {
+        schema_version: SCHEMA_VERSION,
+        rerun_id: rerunId,
+        eval_id: options.evalId,
+        mode: options.selector.mode,
+        tasks: selectedTasks,
+        base_generation: progress.generation,
+        created_at: startedAt,
+    });
+    await writeRerunState(statePath, {
+        rerunId,
+        evalId: options.evalId,
+        status: "running",
+        tasks: selectedTasks,
+        repairedTasks: [],
+        startedAt,
+    });
+    const repaired = new Set();
+    let backendRun;
+    try {
+        if (selectedTasks.length > 0) {
+            const resolvedRevision = await loadResolvedRevision(options.evalDirectory, plan);
+            const preparedArtifact = await loadRerunArtifact(options.root, plan);
+            const runtimeId = requiredString(plan.controllerRuntime.runtime_id, "plan controller runtime id");
+            const runtime = await useControllerRuntimeById(statePaths(options.root), runtimeId.replace(/^sha256:/, ""));
+            if (runtime.runtime_id !== runtimeId
+                || runtime.manifest_digest !== requiredString(plan.controllerRuntime.manifest_digest, "plan controller runtime digest")) {
+                throw unavailable("controller runtime identity changed");
+            }
+            const localTransport = await loadLocalTransport(options.evalDirectory, plan, resolvedRevision, options.env, options.signal);
+            const backendDirectory = path.join(rerunDirectory, "harbor");
+            const harborJobDirectory = path.join(backendDirectory, "job");
+            const rerunRefs = [];
+            const publish = async (ref) => {
+                if (!selectedTasks.includes(ref.task_id) || ref.attempt !== 1) {
+                    throw new HitchError(`Harbor rerun returned an unselected task: ${ref.task_id}`, { code: "eval_rerun_task_mismatch", exitCode: 12 });
+                }
+                if (!rerunRefs.some((current) => current.trial_id === ref.trial_id))
+                    rerunRefs.push(ref);
+                if (ref.observation_status !== "valid")
+                    return;
+                await validateEvalTrialReferences(options.root, options.evalId, [ref], {
+                    benchmarkId: request.benchmark_id,
+                    benchmarkRevision: request.benchmark_revision,
+                });
+                const previousGeneration = progress.generation;
+                progress = replaceInvalidEvalProgressTrial(progress, ref);
+                if (progress.generation === previousGeneration)
+                    return;
+                repaired.add(ref.task_id);
+                await writeEvalProgress(options.evalDirectory, progress);
+                await writeRerunState(statePath, {
+                    rerunId,
+                    evalId: options.evalId,
+                    status: "running",
+                    tasks: selectedTasks,
+                    repairedTasks: [...repaired].sort(),
+                    startedAt,
+                });
+            };
+            backendRun = await runHarborBackend({
+                evalId: options.evalId,
+                evalDirectory: options.evalDirectory,
+                backendDirectory,
+                taskNames: selectedTasks,
+                request,
+                root: options.root,
+                resolvedRevision,
+                runtimeDirectory: runtime.directory,
+                runtimeId: runtime.runtime_id,
+                preparedArtifact,
+                ...(localTransport ? { localTransport } : {}),
+                env: options.env ?? process.env,
+                ...(options.harborExecutable === undefined ? {} : { harborExecutable: options.harborExecutable }),
+                ...(options.signal ? { signal: options.signal } : {}),
+                ...(options.trialBundleGraceMs === undefined ? {} : { trialBundleGraceMs: options.trialBundleGraceMs }),
+                onTrialSettled: async (trial, context) => {
+                    try {
+                        const ref = await importEvalTrialRun({
+                            root: options.root,
+                            evalId: options.evalId,
+                            evalDirectory: options.evalDirectory,
+                            harborJobDirectory,
+                            request,
+                            resolvedRevision,
+                            benchmarkId: request.benchmark_id,
+                            benchmarkRevision: request.benchmark_revision,
+                            runtimeId: runtime.runtime_id,
+                            requireCompleteMarker: true,
+                            allowMissingBundleDiagnostic: context.bundleWaitExpired,
+                        }, trial, rerunRefs.length, rerunRefs);
+                        await publish(ref);
+                        return true;
+                    }
+                    catch (error) {
+                        if (error instanceof TrialBundlePendingError)
+                            return false;
+                        throw error;
+                    }
+                },
+            });
+            const terminalRefs = await importEvalTrialRuns({
+                root: options.root,
+                evalId: options.evalId,
+                evalDirectory: options.evalDirectory,
+                harborJobDirectory,
+                request,
+                resolvedRevision,
+                benchmarkId: request.benchmark_id,
+                benchmarkRevision: request.benchmark_revision,
+                runtimeId: runtime.runtime_id,
+                rawResult: backendRun.rawResult,
+            }, rerunRefs);
+            for (const ref of terminalRefs)
+                await publish(ref);
+            if (backendRun.backend.process_exit_code !== 0 || backendRun.rawResult === null) {
+                throw new HitchError("Harbor rerun failed before producing a complete result", { code: "eval_rerun_harbor_failed", exitCode: 13 });
+            }
+        }
+        const result = await finalizeRerun(options.evalDirectory, request, plan, progress, previousResult, backendRun);
+        const remaining = invalidTasks(plan.tasks, progress);
+        const completedAt = new Date().toISOString();
+        const output = {
+            schema_version: "1",
+            kind: "eval-rerun",
+            rerun_id: rerunId,
+            eval_id: options.evalId,
+            status: "completed",
+            selected_tasks: selectedTasks,
+            repaired_tasks: [...repaired].sort(),
+            remaining_invalid_tasks: remaining,
+            eval_status: result.status,
+            started_at: startedAt,
+            completed_at: completedAt,
+        };
+        await writeRerunState(statePath, {
+            rerunId,
+            evalId: options.evalId,
+            status: "completed",
+            tasks: selectedTasks,
+            repairedTasks: output.repaired_tasks,
+            startedAt,
+            completedAt,
+            evalStatus: output.eval_status,
+            remainingInvalidTasks: remaining,
+        });
+        return output;
+    }
+    catch (error) {
+        if (progress.generation > 0) {
+            await finalizeRerun(options.evalDirectory, request, plan, progress, previousResult, backendRun).catch(() => { });
+        }
+        await writeRerunState(statePath, {
+            rerunId,
+            evalId: options.evalId,
+            status: "failed",
+            tasks: selectedTasks,
+            repairedTasks: [...repaired].sort(),
+            startedAt,
+            completedAt: new Date().toISOString(),
+            errorCode: error instanceof HitchError ? error.code : "eval_rerun_failed",
+        }).catch(() => { });
+        throw error;
+    }
+}
+export function selectRerunTasks(tasks, progress, selector) {
+    const planned = new Set(tasks);
+    if (planned.size !== tasks.length || tasks.some((task) => typeof task !== "string" || task.length === 0)) {
+        throw unavailable("eval task plan is invalid");
+    }
+    const invalid = new Set(invalidTasks(tasks, progress));
+    if (selector.mode === "invalid")
+        return [...invalid].sort();
+    const requested = [...new Set(selector.taskNames)].sort();
+    if (requested.length === 0)
+        throw invalidInput("eval rerun requires at least one --task");
+    for (const task of requested) {
+        if (!planned.has(task))
+            throw new HitchError(`eval task is not in the plan: ${task}`, { code: "eval_rerun_unknown_task", exitCode: 2 });
+        if (!invalid.has(task))
+            throw new HitchError(`eval task is already valid: ${task}`, { code: "eval_task_already_valid", exitCode: 2 });
+    }
+    return requested;
+}
+function invalidTasks(tasks, progress) {
+    const byKey = new Map();
+    for (const trial of progress.trials) {
+        const key = evalTrialKey(trial);
+        if (byKey.has(key))
+            throw unavailable(`eval has duplicate logical task: ${trial.task_id}`);
+        byKey.set(key, trial);
+    }
+    return [...tasks].filter((task) => byKey.get(`${task}\u00001`)?.observation_status !== "valid").sort();
+}
+function validateProgressPlan(progress, plan, request, evalId) {
+    if (progress.eval_id !== evalId)
+        throw unavailable("eval progress identity is invalid");
+    if (progress.benchmark_id !== request.benchmark_id || progress.benchmark_revision !== request.benchmark_revision) {
+        throw unavailable("eval progress benchmark identity changed");
+    }
+    if (progress.planned_tasks !== plan.tasks.length || progress.planned_trials !== plan.tasks.length) {
+        throw unavailable("eval progress does not match the attempts=1 task plan");
+    }
+    const planned = new Set(plan.tasks);
+    for (const trial of progress.trials) {
+        if (trial.attempt !== 1 || !planned.has(trial.task_id))
+            throw unavailable(`eval progress contains an unplanned task: ${trial.task_id}`);
+    }
+    invalidTasks(plan.tasks, progress);
+}
+function parseRerunPlan(value, evalId, request) {
+    if (!value || typeof value !== "object" || Array.isArray(value))
+        throw unavailable("eval plan is missing");
+    const plan = value;
+    if (plan.eval_id !== evalId || plan.schema_version !== "1" || plan.attempts !== 1) {
+        throw unavailable("eval rerun requires a schema v1 attempts=1 plan");
+    }
+    if (plan.dataset !== request.dataset || plan.benchmark_id !== request.benchmark_id
+        || plan.benchmark_revision !== request.benchmark_revision)
+        throw unavailable("eval request and plan identity differ");
+    if (!Array.isArray(plan.tasks) || plan.tasks.length === 0 || plan.tasks.some((task) => typeof task !== "string" || task.length === 0)) {
+        throw unavailable("eval rerun requires a frozen local task plan");
+    }
+    if (!plan.candidate || typeof plan.candidate !== "object" || Array.isArray(plan.candidate)
+        || !plan.prepared_artifact || typeof plan.prepared_artifact !== "object" || Array.isArray(plan.prepared_artifact)
+        || !plan.controller_runtime || typeof plan.controller_runtime !== "object" || Array.isArray(plan.controller_runtime)) {
+        throw unavailable("eval plan is incomplete");
+    }
+    return {
+        tasks: [...plan.tasks],
+        candidate: plan.candidate,
+        preparedArtifact: plan.prepared_artifact,
+        controllerRuntime: plan.controller_runtime,
+        ...(plan.local_source_transport && typeof plan.local_source_transport === "object" && !Array.isArray(plan.local_source_transport)
+            ? { localSourceTransport: plan.local_source_transport }
+            : {}),
+    };
+}
+async function loadPersistedRequest(value) {
+    if (!value || typeof value !== "object" || Array.isArray(value))
+        throw unavailable("eval request is invalid");
+    const persisted = value;
+    const input = Object.fromEntries(Object.entries(persisted).filter(([key]) => key !== "benchmark_id" && key !== "benchmark_revision"));
+    const request = await validateEvalRequest(input);
+    if (persisted.benchmark_id !== request.benchmark_id || persisted.benchmark_revision !== request.benchmark_revision) {
+        throw unavailable("eval dataset identity changed since the original run");
+    }
+    return request;
+}
+async function loadResolvedRevision(evalDirectory, plan) {
+    const resolution = await readJSON(path.join(evalDirectory, "resolution.json"));
+    if (!resolution || resolution.identity !== requiredString(plan.candidate.revision_identity, "candidate revision identity")
+        || resolution.harness_id !== requiredString(plan.candidate.harness_id, "candidate harness id")) {
+        throw unavailable("eval resolution identity changed");
+    }
+    return resolution;
+}
+async function loadRerunArtifact(root, plan) {
+    const summary = plan.preparedArtifact;
+    const artifactId = requiredString(summary.artifact_id, "prepared artifact id");
+    const artifact = await loadPreparedArtifact(preparedArtifactDirectory(root, artifactId), {
+        artifact_id: artifactId,
+        artifact_integrity: requiredString(summary.artifact_integrity, "prepared artifact integrity"),
+        entrypoint_integrity: requiredString(summary.entrypoint_integrity, "prepared artifact entrypoint integrity"),
+        harness_id: requiredString(summary.harness_id, "prepared artifact harness id"),
+        revision_identity: requiredString(summary.revision_identity, "prepared artifact revision identity"),
+        platform: requiredString(summary.platform, "prepared artifact platform"),
+    });
+    return {
+        directory: preparedArtifactDirectory(root, artifact.artifact_id),
+        artifact_id: artifact.artifact_id,
+        artifact_integrity: requiredString(artifact.artifact_integrity, "artifact integrity"),
+        entrypoint_integrity: requiredString(artifact.entrypoint_integrity, "artifact entrypoint integrity"),
+        harness_id: artifact.harness_id,
+        revision_identity: artifact.revision_identity,
+        adapter_version: artifact.adapter_version,
+        recipe_version: artifact.recipe_version,
+        platform: artifact.platform,
+        node_version: artifact.toolchain.node || process.version,
+        source_type: artifact.source_type,
+    };
+}
+async function loadLocalTransport(evalDirectory, plan, resolution, env, signal) {
+    if (plan.localSourceTransport === undefined)
+        return undefined;
+    const directory = path.join(evalDirectory, "local-source");
+    const manifestPath = path.join(directory, "manifest.json");
+    const use = {
+        directory,
+        manifestPath,
+        payloadPath: path.join(directory, "payload.pack"),
+        resolutionPath: path.join(directory, "resolution.json"),
+        manifest: validateLocalGitTransportManifest(await readJSON(manifestPath)),
+    };
+    await verifyLocalGitTransport(use, {
+        expected: {
+            harnessId: resolution.harness_id,
+            resolutionIdentity: resolution.identity,
+            commit: requiredString(resolution.revision.commit, "local source commit"),
+        },
+        env: env ?? process.env,
+        ...(signal ? { signal } : {}),
+    });
+    return use;
+}
+async function finalizeRerun(evalDirectory, request, plan, progress, previousResult, backendRun) {
+    const remaining = invalidTasks(plan.tasks, progress);
+    const succeeded = remaining.length === 0;
+    const completedAt = new Date().toISOString();
+    const result = {
+        ...(previousResult ?? {}),
+        schema_version: SCHEMA_VERSION,
+        eval_id: progress.eval_id,
+        status: succeeded ? "succeeded" : "failed",
+        exit_code: succeeded ? 0 : 13,
+        ...(backendRun ? { backend: backendRun.backend, backend_summary: backendRun.summary } : {}),
+        candidate: plan.candidate,
+        dataset: request.dataset,
+        benchmark_id: request.benchmark_id,
+        benchmark_revision: request.benchmark_revision,
+        generation: progress.generation,
+        trials: progress.trials,
+        summary: summarizeTrialRefs(progress.trials),
+        ...(succeeded ? { error: undefined } : {
+            error: {
+                code: "eval_has_invalid_tasks",
+                message: `eval has invalid or missing tasks: ${remaining.join(", ")}`,
+            },
+        }),
+        started_at: typeof previousResult?.started_at === "string" ? previousResult.started_at : progress.started_at,
+        completed_at: completedAt,
+    };
+    if (succeeded)
+        delete result.error;
+    await atomicWriteJSON(path.join(evalDirectory, "result.json"), result);
+    return result;
+}
+async function writeRerunState(file, input) {
+    await atomicWriteJSON(file, {
+        schema_version: SCHEMA_VERSION,
+        rerun_id: input.rerunId,
+        eval_id: input.evalId,
+        status: input.status,
+        tasks: [...input.tasks],
+        repaired_tasks: [...input.repairedTasks],
+        ...(input.evalStatus ? { eval_status: input.evalStatus } : {}),
+        ...(input.remainingInvalidTasks ? { remaining_invalid_tasks: [...input.remainingInvalidTasks] } : {}),
+        ...(input.errorCode ? { error: { code: input.errorCode } } : {}),
+        started_at: input.startedAt,
+        ...(input.completedAt ? { completed_at: input.completedAt } : {}),
+        updated_at: input.completedAt ?? new Date().toISOString(),
+    });
+}
+function requiredString(value, label) {
+    if (typeof value !== "string" || value.length === 0)
+        throw unavailable(`${label} is missing`);
+    return value;
+}
+function unavailable(message) {
+    return new HitchError(message, { code: "eval_rerun_unavailable", exitCode: 2 });
+}
+//# sourceMappingURL=rerun.js.map

package/dist/src/evals/rerun.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"rerun.js","sourceRoot":"","sources":["../../../src/evals/rerun.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,oBAAoB,EAAE,yBAAyB,EAAE,MAAM,uBAAuB,CAAC;AAExF,OAAO,EACL,gBAAgB,EAChB,iCAAiC,EACjC,uBAAuB,GACxB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAE,wBAAwB,EAAE,MAAM,gCAAgC,CAAC;AAE1E,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,eAAe,EAAE,SAAS,EAAE,YAAY,EAAE,QAAQ,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAClJ,OAAO,EAAE,YAAY,EAAE,gBAAgB,EAAE,+BAA+B,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AACnH,OAAO,EAAE,cAAc,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AACnE,OAAO,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAClD,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,uBAAuB,EAAE,2BAA2B,EAAE,MAAM,mBAAmB,CAAC;AAsClI,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,OAAyB;IACvD,IAAI,CAAC,OAAO,CAAC,IAAI;QAAE,MAAM,YAAY,CAAC,+CAA+C,CAAC,CAAC;IACvF,MAAM,MAAM,GAAG,cAAc,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAC9C,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IACxE,OAAO,YAAY,CACjB,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC,EAClC,QAAQ,EACR,GAAG,EAAE,CAAC,eAAe,CAAC,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,EAC5D,EAAE,WAAW,EAAE,mBAAmB,EAAE,eAAe,EAAE,CAAC,EAAE,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAChH,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,eAAe,CAAC,OAAqE;IAClG,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC3C,MAAM,OAAO,GAAG,SAAS,UAAU,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,CAAC;IAC5D,MAAM,cAAc,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;IAC5F,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,YAAY,CAAC,CAAC;IAC1D,MAAM,YAAY,GAAG,MAAM,QAAQ,CAAiB,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,cAAc,CAAC,EAAE,IAAI,CAAC,CAAC;IAC5G,IAAI,YAAY,KAAK,IAAI;QAAE,MAAM,IAAI,UAAU,CAAC,mBAAmB,OAAO,CAAC,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,gBAAgB,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;IAC9H,MAAM,OAAO,GAAG,MAAM,oBAAoB,CAAC,YAAY,CAAC,CAAC;IACzD,MAAM,IAAI,GAAG,cAAc,CAAC,MAAM,QAAQ,CAAU,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC7H,MAAM,eAAe,GAAG,MAAM,gBAAgB,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;IACtE,IAAI,eAAe,KAAK,IAAI;QAAE,MAAM,WAAW,CAAC,iCAAiC,CAAC,CAAC;IACnF,IAAI,QAAQ,GAAmB,eAAe,CAAC;IAC/C,oBAAoB,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAC9D,MAAM,cAAc,GAAG,MAAM,QAAQ,CAAiC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,aAAa,CAAC,EAAE,IAAI,CAAC,CAAC;IAC7H,IAAI,cAAc,EAAE,MAAM,KAAK,WAAW;QAAE,MAAM,IAAI,UAAU,CAAC,gCAAgC,EAAE,EAAE,IAAI,EAAE,sBAAsB,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;IAClJ,MAAM,aAAa,GAAG,gBAAgB,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC/E,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,cAAc,CAAC,EAAE;QAC/D,cAAc,EAAE,cAAc;QAC9B,QAAQ,EAAE,OAAO;QACjB,OAAO,EAAE,OAAO,CAAC,MAAM;QACvB,IAAI,EAAE,OAAO,CAAC,QAAQ,CAAC,IAAI;QAC3B,KAAK,EAAE,aAAa;QACpB,eAAe,EAAE,QAAQ,CAAC,UAAU;QACpC,UAAU,EAAE,SAAS;KACtB,CAAC,CAAC;IACH,MAAM,eAAe,CAAC,SAAS,EAAE;QAC/B,OAAO;QACP,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,MAAM,EAAE,SAAS;QACjB,KAAK,EAAE,aAAa;QACpB,aAAa,EAAE,EAAE;QACjB,SAAS;KACV,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IACnC,IAAI,UAA2C,CAAC;IAChD,IAAI,CAAC;QACH,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7B,MAAM,gBAAgB,GAAG,MAAM,oBAAoB,CAAC,OAAO,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC;YACjF,MAAM,gBAAgB,GAAG,MAAM,iBAAiB,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YACrE,MAAM,SAAS,GAAG,cAAc,CAAC,IAAI,CAAC,iBAAiB,CAAC,UAAU,EAAE,4BAA4B,CAAC,CAAC;YAClG,MAAM,OAAO,GAAG,MAAM,wBAAwB,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,SAAS,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,CAAC;YAC5G,IAAI,OAAO,CAAC,UAAU,KAAK,SAAS;mBAC/B,OAAO,CAAC,eAAe,KAAK,cAAc,CAAC,IAAI,CAAC,iBAAiB,CAAC,eAAe,EAAE,gCAAgC,CAAC,EAAE,CAAC;gBAC1H,MAAM,WAAW,CAAC,qCAAqC,CAAC,CAAC;YAC3D,CAAC;YACD,MAAM,cAAc,GAAG,MAAM,kBAAkB,CAAC,OAAO,CAAC,aAAa,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;YAC5H,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,QAAQ,CAAC,CAAC;YAC7D,MAAM,kBAAkB,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,KAAK,CAAC,CAAC;YAC9D,MAAM,SAAS,GAAqB,EAAE,CAAC;YACvC,MAAM,OAAO,GAAG,KAAK,EAAE,GAAmB,EAAiB,EAAE;gBAC3D,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,OAAO,KAAK,CAAC,EAAE,CAAC;oBAC9D,MAAM,IAAI,UAAU,CAAC,6CAA6C,GAAG,CAAC,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,0BAA0B,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;gBACvI,CAAC;gBACD,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,KAAK,GAAG,CAAC,QAAQ,CAAC;oBAAE,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACzF,IAAI,GAAG,CAAC,kBAAkB,KAAK,OAAO;oBAAE,OAAO;gBAC/C,MAAM,2BAA2B,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE;oBACrE,WAAW,EAAE,OAAO,CAAC,YAAY;oBACjC,iBAAiB,EAAE,OAAO,CAAC,kBAAkB;iBAC9C,CAAC,CAAC;gBACH,MAAM,kBAAkB,GAAG,QAAQ,CAAC,UAAU,CAAC;gBAC/C,QAAQ,GAAG,+BAA+B,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;gBAC1D,IAAI,QAAQ,CAAC,UAAU,KAAK,kBAAkB;oBAAE,OAAO;gBACvD,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;gBAC1B,MAAM,iBAAiB,CAAC,OAAO,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;gBACzD,MAAM,eAAe,CAAC,SAAS,EAAE;oBAC/B,OAAO;oBACP,MAAM,EAAE,OAAO,CAAC,MAAM;oBACtB,MAAM,EAAE,SAAS;oBACjB,KAAK,EAAE,aAAa;oBACpB,aAAa,EAAE,CAAC,GAAG,QAAQ,CAAC,CAAC,IAAI,EAAE;oBACnC,SAAS;iBACV,CAAC,CAAC;YACL,CAAC,CAAC;YACF,UAAU,GAAG,MAAM,gBAAgB,CAAC;gBAClC,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,aAAa,EAAE,OAAO,CAAC,aAAa;gBACpC,gBAAgB;gBAChB,SAAS,EAAE,aAAa;gBACxB,OAAO;gBACP,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,gBAAgB;gBAChB,gBAAgB,EAAE,OAAO,CAAC,SAAS;gBACnC,SAAS,EAAE,OAAO,CAAC,UAAU;gBAC7B,gBAAgB;gBAChB,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC7C,GAAG,EAAE,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG;gBAC/B,GAAG,CAAC,OAAO,CAAC,gBAAgB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,EAAE,CAAC;gBACjG,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACrD,GAAG,CAAC,OAAO,CAAC,kBAAkB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,kBAAkB,EAAE,OAAO,CAAC,kBAAkB,EAAE,CAAC;gBACvG,cAAc,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAoB,EAAE;oBACzD,IAAI,CAAC;wBACH,MAAM,GAAG,GAAG,MAAM,kBAAkB,CAAC;4BACnC,IAAI,EAAE,OAAO,CAAC,IAAI;4BAClB,MAAM,EAAE,OAAO,CAAC,MAAM;4BACtB,aAAa,EAAE,OAAO,CAAC,aAAa;4BACpC,kBAAkB;4BAClB,OAAO;4BACP,gBAAgB;4BAChB,WAAW,EAAE,OAAO,CAAC,YAAY;4BACjC,iBAAiB,EAAE,OAAO,CAAC,kBAAkB;4BAC7C,SAAS,EAAE,OAAO,CAAC,UAAU;4BAC7B,qBAAqB,EAAE,IAAI;4BAC3B,4BAA4B,EAAE,OAAO,CAAC,iBAAiB;yBACxD,EAAE,KAAK,EAAE,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;wBACvC,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC;wBACnB,OAAO,IAAI,CAAC;oBACd,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBACf,IAAI,KAAK,YAAY,uBAAuB;4BAAE,OAAO,KAAK,CAAC;wBAC3D,MAAM,KAAK,CAAC;oBACd,CAAC;gBACH,CAAC;aACF,CAAC,CAAC;YACH,MAAM,YAAY,GAAG,MAAM,mBAAmB,CAAC;gBAC7C,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,aAAa,EAAE,OAAO,CAAC,aAAa;gBACpC,kBAAkB;gBAClB,OAAO;gBACP,gBAAgB;gBAChB,WAAW,EAAE,OAAO,CAAC,YAAY;gBACjC,iBAAiB,EAAE,OAAO,CAAC,kBAAkB;gBAC7C,SAAS,EAAE,OAAO,CAAC,UAAU;gBAC7B,SAAS,EAAE,UAAU,CAAC,SAAS;aAChC,EAAE,SAAS,CAAC,CAAC;YACd,KAAK,MAAM,GAAG,IAAI,YAAY;gBAAE,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC;YACnD,IAAI,UAAU,CAAC,OAAO,CAAC,iBAAiB,KAAK,CAAC,IAAI,UAAU,CAAC,SAAS,KAAK,IAAI,EAAE,CAAC;gBAChF,MAAM,IAAI,UAAU,CAAC,wDAAwD,EAAE,EAAE,IAAI,EAAE,0BAA0B,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;YACrI,CAAC;QACH,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,OAAO,CAAC,aAAa,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,cAAc,EAAE,UAAU,CAAC,CAAC;QAC/G,MAAM,SAAS,GAAG,YAAY,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QACrD,MAAM,WAAW,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC7C,MAAM,MAAM,GAAoB;YAC9B,cAAc,EAAE,GAAG;YACnB,IAAI,EAAE,YAAY;YAClB,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,OAAO,CAAC,MAAM;YACvB,MAAM,EAAE,WAAW;YACnB,cAAc,EAAE,aAAa;YAC7B,cAAc,EAAE,CAAC,GAAG,QAAQ,CAAC,CAAC,IAAI,EAAE;YACpC,uBAAuB,EAAE,SAAS;YAClC,WAAW,EAAE,MAAM,CAAC,MAAgC;YACpD,UAAU,EAAE,SAAS;YACrB,YAAY,EAAE,WAAW;SAC1B,CAAC;QACF,MAAM,eAAe,CAAC,SAAS,EAAE;YAC/B,OAAO;YACP,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,MAAM,EAAE,WAAW;YACnB,KAAK,EAAE,aAAa;YACpB,aAAa,EAAE,MAAM,CAAC,cAAc;YACpC,SAAS;YACT,WAAW;YACX,UAAU,EAAE,MAAM,CAAC,WAAW;YAC9B,qBAAqB,EAAE,SAAS;SACjC,CAAC,CAAC;QACH,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,QAAQ,CAAC,UAAU,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,aAAa,CAAC,OAAO,CAAC,aAAa,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,cAAc,EAAE,UAAU,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAClH,CAAC;QACD,MAAM,eAAe,CAAC,SAAS,EAAE;YAC/B,OAAO;YACP,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,MAAM,EAAE,QAAQ;YAChB,KAAK,EAAE,aAAa;YACpB,aAAa,EAAE,CAAC,GAAG,QAAQ,CAAC,CAAC,IAAI,EAAE;YACnC,SAAS;YACT,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACrC,SAAS,EAAE,KAAK,YAAY,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,mBAAmB;SAC1E,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACnB,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,KAAwB,EAAE,QAAwB,EAAE,QAAuB;IAC1G,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;IAC/B,IAAI,OAAO,CAAC,IAAI,KAAK,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,EAAE,CAAC;QACzG,MAAM,WAAW,CAAC,2BAA2B,CAAC,CAAC;IACjD,CAAC;IACD,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC;IACvD,IAAI,QAAQ,CAAC,IAAI,KAAK,SAAS;QAAE,OAAO,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;IAC5D,MAAM,SAAS,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC1D,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,YAAY,CAAC,yCAAyC,CAAC,CAAC;IAC1F,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,MAAM,IAAI,UAAU,CAAC,iCAAiC,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,yBAAyB,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;QACxI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,MAAM,IAAI,UAAU,CAAC,+BAA+B,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,yBAAyB,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;IACxI,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,YAAY,CAAC,KAAwB,EAAE,QAAwB;IACtE,MAAM,KAAK,GAAG,IAAI,GAAG,EAA0B,CAAC;IAChD,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;QACpC,MAAM,GAAG,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;QAChC,IAAI,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,MAAM,WAAW,CAAC,oCAAoC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAC3F,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACxB,CAAC;IACD,OAAO,CAAC,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,IAAI,SAAS,CAAC,EAAE,kBAAkB,KAAK,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;AACzG,CAAC;AAED,SAAS,oBAAoB,CAAC,QAAwB,EAAE,IAAe,EAAE,OAAoB,EAAE,MAAc;IAC3G,IAAI,QAAQ,CAAC,OAAO,KAAK,MAAM;QAAE,MAAM,WAAW,CAAC,mCAAmC,CAAC,CAAC;IACxF,IAAI,QAAQ,CAAC,YAAY,KAAK,OAAO,CAAC,YAAY,IAAI,QAAQ,CAAC,kBAAkB,KAAK,OAAO,CAAC,kBAAkB,EAAE,CAAC;QACjH,MAAM,WAAW,CAAC,0CAA0C,CAAC,CAAC;IAChE,CAAC;IACD,IAAI,QAAQ,CAAC,aAAa,KAAK,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,QAAQ,CAAC,cAAc,KAAK,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QAClG,MAAM,WAAW,CAAC,uDAAuD,CAAC,CAAC;IAC7E,CAAC;IACD,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACpC,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;QACpC,IAAI,KAAK,CAAC,OAAO,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC;YAAE,MAAM,WAAW,CAAC,6CAA6C,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;IAC1I,CAAC;IACD,YAAY,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;AACrC,CAAC;AAED,SAAS,cAAc,CAAC,KAAc,EAAE,MAAc,EAAE,OAAoB;IAC1E,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,WAAW,CAAC,sBAAsB,CAAC,CAAC;IAC3G,MAAM,IAAI,GAAG,KAAgC,CAAC;IAC9C,IAAI,IAAI,CAAC,OAAO,KAAK,MAAM,IAAI,IAAI,CAAC,cAAc,KAAK,GAAG,IAAI,IAAI,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;QAClF,MAAM,WAAW,CAAC,iDAAiD,CAAC,CAAC;IACvE,CAAC;IACD,IAAI,IAAI,CAAC,OAAO,KAAK,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,YAAY,KAAK,OAAO,CAAC,YAAY;WAC7E,IAAI,CAAC,kBAAkB,KAAK,OAAO,CAAC,kBAAkB;QAAE,MAAM,WAAW,CAAC,uCAAuC,CAAC,CAAC;IACxH,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,EAAE,CAAC;QACtI,MAAM,WAAW,CAAC,8CAA8C,CAAC,CAAC;IACpE,CAAC;IACD,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,OAAO,IAAI,CAAC,SAAS,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC;WACrF,CAAC,IAAI,CAAC,iBAAiB,IAAI,OAAO,IAAI,CAAC,iBAAiB,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,iBAAiB,CAAC;WAC9G,CAAC,IAAI,CAAC,kBAAkB,IAAI,OAAO,IAAI,CAAC,kBAAkB,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,kBAAkB,CAAC,EAAE,CAAC;QACvH,MAAM,WAAW,CAAC,yBAAyB,CAAC,CAAC;IAC/C,CAAC;IACD,OAAO;QACL,KAAK,EAAE,CAAC,GAAG,IAAI,CAAC,KAAiB,CAAC;QAClC,SAAS,EAAE,IAAI,CAAC,SAAoC;QACpD,gBAAgB,EAAE,IAAI,CAAC,iBAA4C;QACnE,iBAAiB,EAAE,IAAI,CAAC,kBAA6C;QACrE,GAAG,CAAC,IAAI,CAAC,sBAAsB,IAAI,OAAO,IAAI,CAAC,sBAAsB,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,sBAAsB,CAAC;YAC/H,CAAC,CAAC,EAAE,oBAAoB,EAAE,IAAI,CAAC,sBAAiD,EAAE;YAClF,CAAC,CAAC,EAAE,CAAC;KACR,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,oBAAoB,CAAC,KAAc;IAChD,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,WAAW,CAAC,yBAAyB,CAAC,CAAC;IAC9G,MAAM,SAAS,GAAG,KAAgC,CAAC;IACnD,MAAM,KAAK,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,GAAG,KAAK,cAAc,IAAI,GAAG,KAAK,oBAAoB,CAAC,CAAC,CAAC;IACtI,MAAM,OAAO,GAAG,MAAM,mBAAmB,CAAC,KAAK,CAAC,CAAC;IACjD,IAAI,SAAS,CAAC,YAAY,KAAK,OAAO,CAAC,YAAY,IAAI,SAAS,CAAC,kBAAkB,KAAK,OAAO,CAAC,kBAAkB,EAAE,CAAC;QACnH,MAAM,WAAW,CAAC,sDAAsD,CAAC,CAAC;IAC5E,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,KAAK,UAAU,oBAAoB,CAAC,aAAqB,EAAE,IAAe;IACxE,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAmB,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,iBAAiB,CAAC,CAAC,CAAC;IACjG,IAAI,CAAC,UAAU,IAAI,UAAU,CAAC,QAAQ,KAAK,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE,6BAA6B,CAAC;WACrH,UAAU,CAAC,UAAU,KAAK,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,sBAAsB,CAAC,EAAE,CAAC;QACjG,MAAM,WAAW,CAAC,kCAAkC,CAAC,CAAC;IACxD,CAAC;IACD,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,IAAY,EAAE,IAAe;IAC5D,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC;IACtC,MAAM,UAAU,GAAG,cAAc,CAAC,OAAO,CAAC,WAAW,EAAE,sBAAsB,CAAC,CAAC;IAC/E,MAAM,QAAQ,GAAG,MAAM,oBAAoB,CAAC,yBAAyB,CAAC,IAAI,EAAE,UAAU,CAAC,EAAE;QACvF,WAAW,EAAE,UAAU;QACvB,kBAAkB,EAAE,cAAc,CAAC,OAAO,CAAC,kBAAkB,EAAE,6BAA6B,CAAC;QAC7F,oBAAoB,EAAE,cAAc,CAAC,OAAO,CAAC,oBAAoB,EAAE,wCAAwC,CAAC;QAC5G,UAAU,EAAE,cAAc,CAAC,OAAO,CAAC,UAAU,EAAE,8BAA8B,CAAC;QAC9E,iBAAiB,EAAE,cAAc,CAAC,OAAO,CAAC,iBAAiB,EAAE,qCAAqC,CAAC;QACnG,QAAQ,EAAE,cAAc,CAAC,OAAO,CAAC,QAAQ,EAAE,4BAA4B,CAAC;KACzE,CAAC,CAAC;IACH,OAAO;QACL,SAAS,EAAE,yBAAyB,CAAC,IAAI,EAAE,QAAQ,CAAC,WAAW,CAAC;QAChE,WAAW,EAAE,QAAQ,CAAC,WAAW;QACjC,kBAAkB,EAAE,cAAc,CAAC,QAAQ,CAAC,kBAAkB,EAAE,oBAAoB,CAAC;QACrF,oBAAoB,EAAE,cAAc,CAAC,QAAQ,CAAC,oBAAoB,EAAE,+BAA+B,CAAC;QACpG,UAAU,EAAE,QAAQ,CAAC,UAAU;QAC/B,iBAAiB,EAAE,QAAQ,CAAC,iBAAiB;QAC7C,eAAe,EAAE,QAAQ,CAAC,eAAe;QACzC,cAAc,EAAE,QAAQ,CAAC,cAAc;QACvC,QAAQ,EAAE,QAAQ,CAAC,QAAQ;QAC3B,YAAY,EAAE,QAAQ,CAAC,SAAS,CAAC,IAAI,IAAI,OAAO,CAAC,OAAO;QACxD,WAAW,EAAE,QAAQ,CAAC,WAAW;KAClC,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,kBAAkB,CAC/B,aAAqB,EACrB,IAAe,EACf,UAA4B,EAC5B,GAAkC,EAClC,MAA+B;IAE/B,IAAI,IAAI,CAAC,oBAAoB,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC9D,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,cAAc,CAAC,CAAC;IAC3D,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;IAC3D,MAAM,GAAG,GAAyB;QAChC,SAAS;QACT,YAAY;QACZ,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC;QACjD,cAAc,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,iBAAiB,CAAC;QACvD,QAAQ,EAAE,iCAAiC,CAAC,MAAM,QAAQ,CAAC,YAAY,CAAC,CAAC;KAC1E,CAAC;IACF,MAAM,uBAAuB,CAAC,GAAG,EAAE;QACjC,QAAQ,EAAE;YACR,SAAS,EAAE,UAAU,CAAC,UAAU;YAChC,kBAAkB,EAAE,UAAU,CAAC,QAAQ;YACvC,MAAM,EAAE,cAAc,CAAC,UAAU,CAAC,QAAQ,CAAC,MAAM,EAAE,qBAAqB,CAAC;SAC1E;QACD,GAAG,EAAE,GAAG,IAAI,OAAO,CAAC,GAAG;QACvB,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC9B,CAAC,CAAC;IACH,OAAO,GAAG,CAAC;AACb,CAAC;AAED,KAAK,UAAU,aAAa,CAC1B,aAAqB,EACrB,OAAoB,EACpB,IAAe,EACf,QAAwB,EACxB,cAA8C,EAC9C,UAAgC;IAEhC,MAAM,SAAS,GAAG,YAAY,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;IACrD,MAAM,SAAS,GAAG,SAAS,CAAC,MAAM,KAAK,CAAC,CAAC;IACzC,MAAM,WAAW,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC7C,MAAM,MAAM,GAA4B;QACtC,GAAG,CAAC,cAAc,IAAI,EAAE,CAAC;QACzB,cAAc,EAAE,cAAc;QAC9B,OAAO,EAAE,QAAQ,CAAC,OAAO;QACzB,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ;QAC1C,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE;QAC7B,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,UAAU,CAAC,OAAO,EAAE,eAAe,EAAE,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3F,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,kBAAkB,EAAE,OAAO,CAAC,kBAAkB;QAC9C,UAAU,EAAE,QAAQ,CAAC,UAAU;QAC/B,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,OAAO,EAAE,kBAAkB,CAAC,QAAQ,CAAC,MAAM,CAAC;QAC5C,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC;YACrC,KAAK,EAAE;gBACL,IAAI,EAAE,wBAAwB;gBAC9B,OAAO,EAAE,sCAAsC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;aACtE;SACF,CAAC;QACF,UAAU,EAAE,OAAO,cAAc,EAAE,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU;QAC5G,YAAY,EAAE,WAAW;KAC1B,CAAC;IACF,IAAI,SAAS;QAAE,OAAO,MAAM,CAAC,KAAK,CAAC;IACnC,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,aAAa,CAAC,EAAE,MAAM,CAAC,CAAC;IACvE,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,KAAK,UAAU,eAAe,CAAC,IAAY,EAAE,KAW5C;IACC,MAAM,eAAe,CAAC,IAAI,EAAE;QAC1B,cAAc,EAAE,cAAc;QAC9B,QAAQ,EAAE,KAAK,CAAC,OAAO;QACvB,OAAO,EAAE,KAAK,CAAC,MAAM;QACrB,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,KAAK,EAAE,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC;QACvB,cAAc,EAAE,CAAC,GAAG,KAAK,CAAC,aAAa,CAAC;QACxC,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9D,GAAG,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC,CAAC,EAAE,uBAAuB,EAAE,CAAC,GAAG,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACrG,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,KAAK,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAChE,UAAU,EAAE,KAAK,CAAC,SAAS;QAC3B,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACjE,UAAU,EAAE,KAAK,CAAC,WAAW,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KAC1D,CAAC,CAAC;AACL,CAAC;AAED,SAAS,cAAc,CAAC,KAAc,EAAE,KAAa;IACnD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,WAAW,CAAC,GAAG,KAAK,aAAa,CAAC,CAAC;IAC9F,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,WAAW,CAAC,OAAe;IAClC,OAAO,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,wBAAwB,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;AAClF,CAAC"}

package/dist/src/evals/service.js

@@ -1,23 +1,37 @@
 import path from "node:path";
+import { mkdir } from "node:fs/promises";
 import { prepareHarness, preparedArtifactDirectory, resolveHarness } from "../artifacts/index.js";
 import { HitchError, SCHEMA_VERSION, atomicWriteJSON, ensureDir, invalidInput, statePaths } from "../foundation/index.js";
 import { parseHarnessReference } from "../revisions/index.js";
 import { buildLocalGitTransport, lockedHarnessRef, runHarborBackend, verifyLocalGitTransport } from "../backends/index.js";
 import { ensureControllerRuntime, writeRuntimeReference } from "../controller-runtime/index.js";
-import { importEvalTrialRuns, validateEvalTrialReferences } from "./trial-import.js";
+import { importEvalTrialRun, importEvalTrialRuns, TrialBundlePendingError, validateEvalTrialReferences } from "./trial-import.js";
 import { EvalEventSink } from "./events.js";
-import { newEvalId, validateEvalRequest } from "./request.js";
-export async function runEval({ evalId = newEvalId(), request, root, env = process.env, harborExecutable, signal, onEvent }) {
+import { createEvalProgress, mergeEvalProgressTrial, writeEvalProgress } from "./progress.js";
+import { newEvalId, resolveLocalDatasetTaskIds, validateEvalId, validateEvalRequest } from "./request.js";
+export async function runEval({ evalId = newEvalId(), request, root, env = process.env, harborExecutable, signal, onEvent, trialBundleGraceMs }) {
     if (!root)
         throw invalidInput("a Hitch state root is required for eval");
+    evalId = validateEvalId(evalId);
     const normalized = await validateEvalRequest(request);
-    const evalDirectory = await ensureDir(path.join(statePaths(root).evals, evalId));
+    const evalsDirectory = await ensureDir(statePaths(root).evals);
+    const evalDirectory = path.join(evalsDirectory, evalId);
+    try {
+        await mkdir(evalDirectory, { mode: 0o700 });
+    }
+    catch (error) {
+        if (error.code === "EEXIST") {
+            throw new HitchError(`eval ID already exists: ${evalId}`, { code: "eval_id_conflict", exitCode: 2 });
+        }
+        throw error;
+    }
     const startedAt = new Date();
     const sink = new EvalEventSink(evalDirectory, evalId, onEvent);
     await sink.open();
     await atomicWriteJSON(path.join(evalDirectory, "request.json"), normalized);
     let result;
     let trialRefs = [];
+    let progress = null;
     try {
         sink.emit({ type: "eval.started", backend: normalized.backend, dataset: normalized.dataset });
         const requestedReference = parseHarnessReference(normalized.harness_ref);
@@ -75,7 +89,10 @@ export async function runEval({ evalId = newEvalId(), request, root, env = proce
             entrypoint_integrity: artifact.entrypoint_integrity,
             harness_id: artifact.harness_id,
             revision_identity: artifact.revision_identity,
+            adapter_version: artifact.adapter_version,
+            recipe_version: artifact.recipe_version,
             platform: artifact.platform,
+            node_version: artifact.toolchain.node || process.version,
             source_type: artifact.source_type,
         };
         sink.emit({
@@ -95,6 +112,11 @@ export async function runEval({ evalId = newEvalId(), request, root, env = proce
             cache_hit: controllerRuntime.cache_hit,
             reference: runtimeRefFile,
         });
+        const localTaskIds = await resolveLocalDatasetTaskIds(normalized.dataset);
+        const plannedTasks = localTaskIds?.length ?? null;
+        const plannedTrials = plannedTasks === null ? null : plannedTasks * normalized.attempts;
+        if (plannedTrials !== null && !Number.isSafeInteger(plannedTrials))
+            throw invalidInput("planned trial count exceeds the safe integer range");
         const plan = {
             schema_version: SCHEMA_VERSION,
             eval_id: evalId,
@@ -112,6 +134,7 @@ export async function runEval({ evalId = newEvalId(), request, root, env = proce
             benchmark_revision: normalized.benchmark_revision,
             attempts: normalized.attempts,
             max_concurrent: normalized.max_concurrent,
+            ...(localTaskIds === null ? {} : { tasks: localTaskIds }),
             controller_runtime: {
                 runtime_id: controllerRuntime.runtime_id,
                 manifest_digest: controllerRuntime.manifest_digest,
@@ -122,11 +145,22 @@ export async function runEval({ evalId = newEvalId(), request, root, env = proce
         };
         await atomicWriteJSON(path.join(evalDirectory, "resolution.json"), resolvedRevision);
         await atomicWriteJSON(path.join(evalDirectory, "plan.json"), plan);
+        progress = createEvalProgress({
+            evalId,
+            benchmarkId: normalized.benchmark_id,
+            benchmarkRevision: normalized.benchmark_revision,
+            plannedTasks,
+            plannedTrials,
+            startedAt: startedAt.toISOString(),
+        });
+        await writeEvalProgress(evalDirectory, progress);
         sink.emit({
             type: "eval.planned",
             harness: resolvedRevision.harness_id,
             revision_identity: resolvedRevision.identity,
             attempts: normalized.attempts,
+            planned_tasks: plannedTasks,
+            planned_trials: plannedTrials,
         });
         if (localTransport) {
             await verifyLocalGitTransport(localTransport, {
@@ -152,7 +186,51 @@ export async function runEval({ evalId = newEvalId(), request, root, env = proce
             env,
             ...(harborExecutable !== undefined ? { harborExecutable } : {}),
             ...(signal ? { signal } : {}),
+            ...(trialBundleGraceMs === undefined ? {} : { trialBundleGraceMs }),
             emit: (event) => sink.emit(event),
+            onTrialSettled: async (trial, context) => {
+                try {
+                    const ref = await importEvalTrialRun({
+                        root,
+                        evalId,
+                        evalDirectory,
+                        request: normalized,
+                        resolvedRevision,
+                        benchmarkId: normalized.benchmark_id,
+                        benchmarkRevision: normalized.benchmark_revision,
+                        runtimeId: controllerRuntime.runtime_id,
+                        requireCompleteMarker: true,
+                        allowMissingBundleDiagnostic: context.bundleWaitExpired,
+                    }, trial, progress?.trials.length ?? 0, progress?.trials ?? []);
+                    if (progress === null)
+                        throw new Error("eval progress was not initialized");
+                    const previousGeneration = progress.generation;
+                    progress = mergeEvalProgressTrial(progress, ref);
+                    if (progress.generation !== previousGeneration) {
+                        await validateEvalTrialReferences(root, evalId, [ref], {
+                            benchmarkId: normalized.benchmark_id,
+                            benchmarkRevision: normalized.benchmark_revision,
+                        });
+                        await writeEvalProgress(evalDirectory, progress);
+                        sink.emit({
+                            type: "eval.trial.published",
+                            trial_id: ref.trial_id,
+                            task_id: ref.task_id,
+                            attempt: ref.attempt,
+                            run_id: ref.run_id,
+                            observation_status: ref.observation_status,
+                            settled_trials: progress.trials.length,
+                            generation: progress.generation,
+                        });
+                    }
+                    return true;
+                }
+                catch (error) {
+                    if (error instanceof TrialBundlePendingError)
+                        return false;
+                    throw error;
+                }
+            },
         });
         trialRefs = await importEvalTrialRuns({
             root,
@@ -164,7 +242,13 @@ export async function runEval({ evalId = newEvalId(), request, root, env = proce
             benchmarkRevision: normalized.benchmark_revision,
             runtimeId: controllerRuntime.runtime_id,
             rawResult: backendRun.rawResult,
-        });
+        }, progress?.trials ?? []);
+        if (progress === null)
+            throw new Error("eval progress was not initialized");
+        for (const ref of trialRefs)
+            progress = mergeEvalProgressTrial(progress, ref);
+        await writeEvalProgress(evalDirectory, progress);
+        assertBackendTrialSet(backendRun.rawResult, progress.trials);
         await validateEvalTrialReferences(root, evalId, trialRefs, {
             benchmarkId: normalized.benchmark_id,
             benchmarkRevision: normalized.benchmark_revision,
@@ -182,6 +266,7 @@ export async function runEval({ evalId = newEvalId(), request, root, env = proce
             dataset: normalized.dataset,
             benchmark_id: normalized.benchmark_id,
             benchmark_revision: normalized.benchmark_revision,
+            generation: progress.generation,
             trials: trialRefs,
             summary: summarizeTrialRefs(trialRefs),
             backend_summary: backendRun.summary,
@@ -204,6 +289,7 @@ export async function runEval({ evalId = newEvalId(), request, root, env = proce
         };
     }
     catch (error) {
+        trialRefs = progress?.trials ?? trialRefs;
         const typed = error instanceof HitchError;
         result = {
             schema_version: SCHEMA_VERSION,
@@ -216,6 +302,7 @@ export async function runEval({ evalId = newEvalId(), request, root, env = proce
             },
             benchmark_id: normalized.benchmark_id,
             benchmark_revision: normalized.benchmark_revision,
+            ...(progress === null ? {} : { generation: progress.generation }),
             trials: trialRefs,
             started_at: startedAt.toISOString(),
             completed_at: new Date().toISOString(),
@@ -226,7 +313,15 @@ export async function runEval({ evalId = newEvalId(), request, root, env = proce
     await sink.close();
     return result;
 }
-function summarizeTrialRefs(trials) {
+function assertBackendTrialSet(rawResult, refs) {
+    const trials = Array.isArray(rawResult?.trial_results) ? rawResult.trial_results : [];
+    const backendIds = new Set(trials.map((trial) => typeof trial.trial_name === "string" ? trial.trial_name : null).filter((value) => value !== null));
+    const refIds = new Set(refs.map((ref) => ref.trial_id));
+    if (backendIds.size !== trials.length || backendIds.size !== refIds.size || [...backendIds].some((id) => !refIds.has(id))) {
+        throw new Error("Harbor terminal trial set does not match published eval progress");
+    }
+}
+export function summarizeTrialRefs(trials) {
     const valid = trials.filter((trial) => trial.observation_status === "valid" && typeof trial.reward === "number");
     const rewards = valid.map((trial) => trial.reward);
     const aggregate = rewards.length