lastlight-evals 0.1.0

package/dist/run.js

@@ -0,0 +1,476 @@
+#!/usr/bin/env node
+/**
+ * Eval runner (a measurement, not a test).
+ *
+ * Drives the REAL production workflows (issue-triage / build / …) against a
+ * fake GitHub for each model under test, grades deterministically, and prints
+ * a model-comparison scorecard + writes SWE-bench-compatible artifacts. It
+ * exits non-zero only if the HARNESS itself errors — never because a model
+ * scored poorly (that's the signal we're measuring).
+ *
+ * Run:
+ *   npm run eval                       # triage tier, default model
+ *   npm run eval -- code-fix           # code-fix tier
+ *   npm run eval -- triage code-fix    # both
+ *   EVAL_MODELS="openai/gpt-5.5,openai/gpt-5.4-mini" npm run eval
+ *
+ * The deterministic, AI-free plumbing is covered separately by
+ * `evals/mechanism.test.ts` in the normal `npm test` suite.
+ */
+import { existsSync } from "node:fs";
+import { join } from "node:path";
+import { spawn } from "node:child_process";
+import * as p from "@clack/prompts";
+import chalk from "chalk";
+import { loadDotEnv, hasProviderKey, evalModels, compareModels, modelLabels, resolveModel, setModelsPath } from "./env.js";
+import { runInstance, applyEvalEnv } from "./run-instance.js";
+import { summarize, writeArtifacts, aggregateTrials } from "./report.js";
+import { writeHtml } from "./html-report.js";
+import { bootstrapAssets } from "./bootstrap.js";
+import { discoverTiers, loadInstances, workflowFor } from "./discovery.js";
+import { builtinDatasetsRoot, resultsRoot } from "./paths.js";
+import { runInit } from "./init.js";
+/** Open a file in the OS default browser (best-effort, never throws). */
+function openInBrowser(file) {
+    const url = `file://${file}`;
+    const [cmd, args] = process.platform === "darwin"
+        ? ["open", [url]]
+        : process.platform === "win32"
+            ? ["cmd", ["/c", "start", "", url]]
+            : ["xdg-open", [url]];
+    try {
+        spawn(cmd, args, { detached: true, stdio: "ignore" }).unref();
+    }
+    catch {
+        /* headless / no browser — the path is printed anyway */
+    }
+}
+/**
+ * Run `fn` with `console.*` captured into a buffer so the deep workflow chatter
+ * (`[executor] …`, octokit deprecation warnings) doesn't shred the clack
+ * spinner — which writes via `process.stdout.write`, a different channel. The
+ * captured logs are returned so we can replay them only when a run errors.
+ */
+async function quiet(fn) {
+    const buf = [];
+    const cap = (orig) => (...a) => {
+        buf.push(a.map((x) => (typeof x === "string" ? x : String(x))).join(" "));
+        void orig;
+    };
+    const { log, warn, error, info } = console;
+    console.log = cap(log);
+    console.warn = cap(warn);
+    console.error = cap(error);
+    console.info = cap(info);
+    try {
+        return { value: await fn(), logs: buf.join("\n") };
+    }
+    finally {
+        Object.assign(console, { log, warn, error, info });
+    }
+}
+function fmtMs(ms) {
+    if (!ms)
+        return "—";
+    if (ms < 1000)
+        return `${ms}ms`;
+    const s = Math.round(ms / 1000);
+    return s < 60 ? `${s}s` : `${Math.floor(s / 60)}m${String(s % 60).padStart(2, "0")}`;
+}
+/** Friendly provider-family name from an env-key (OPENAI_API_KEY → "openai"). */
+function familyLabel(envKey) {
+    return envKey.replace(/_API_KEY$/i, "").toLowerCase() || "default";
+}
+/**
+ * Silence `console.*` for the whole batch (parallel mode). The per-run
+ * `quiet()` swap saves/restores console and would corrupt under concurrent
+ * runs (nested swaps), so parallel mode drops console output once instead.
+ * The clack spinner is untouched — it writes via `process.stdout.write`.
+ */
+function silenceConsole() {
+    const { log, warn, error, info } = console;
+    const sink = () => { };
+    Object.assign(console, { log: sink, warn: sink, error: sink, info: sink });
+    return () => Object.assign(console, { log, warn, error, info });
+}
+/** Colored one-line verdict for a finished run (with N/N pass count if N>1). */
+function verdictLine(tierName, inst, r) {
+    const head = `${chalk.cyan(tierName)}/${inst.instance_id}`;
+    if (r.error)
+        return `${head}  ${chalk.red("harness error")}`;
+    const count = (pass) => (pass !== undefined && r.trials ? chalk.dim(` ${pass}/${r.trials}`) : "");
+    const parts = [];
+    if (r.resolved !== undefined)
+        parts.push((r.resolved ? chalk.green("resolved") : chalk.red("unresolved")) + count(r.resolvedPass));
+    if (r.behavioral)
+        parts.push((r.behavioral.ok ? chalk.green("behavioral ✓") : chalk.red("behavioral ✗")) + count(r.behavioralPass));
+    parts.push(chalk.dim(`$${r.costUsd.toFixed(4)}`));
+    parts.push(chalk.dim(fmtMs(r.durationMs)));
+    return `${head}  ${parts.join("  ")}`;
+}
+/** Parse an integer CLI flag (`--name N` or `--name=N`) or `EVAL_NAME` env. */
+function intFlag(name, def) {
+    const argv = process.argv;
+    for (let i = 0; i < argv.length; i++) {
+        if (argv[i] === `--${name}`) {
+            const n = parseInt(argv[i + 1] ?? "", 10);
+            if (n > 0)
+                return n;
+        }
+        const m = argv[i].match(new RegExp(`^--${name}=(\\d+)$`));
+        if (m)
+            return parseInt(m[1], 10);
+    }
+    const env = parseInt(process.env[`EVAL_${name.toUpperCase()}`] ?? "", 10);
+    return env > 0 ? env : def;
+}
+/** Parse a string CLI flag (`--name V` or `--name=V`) or `EVAL_NAME` env. */
+function strFlag(name) {
+    const argv = process.argv;
+    for (let i = 0; i < argv.length; i++) {
+        if (argv[i] === `--${name}` && argv[i + 1] !== undefined)
+            return argv[i + 1];
+        const m = argv[i].match(new RegExp(`^--${name}=(.+)$`));
+        if (m)
+            return m[1];
+    }
+    return process.env[`EVAL_${name.toUpperCase()}`];
+}
+/** CLI flags that take a following value (so it isn't read as a tier name). */
+const VALUE_FLAGS = new Set(["--runs", "--model", "--models", "--overlay", "--datasets", "--models-file"]);
+async function runEval() {
+    loadDotEnv();
+    p.intro(chalk.bold(`Last Light ${chalk.yellow("·")} eval`));
+    // Asset roots FIRST — before any getWorkflow/runWorkflow. `--overlay` (or
+    // LASTLIGHT_OVERLAY_DIR) layers a deployment's own workflows/skills over the
+    // built-ins, and also contributes its `evals/datasets/` (see discovery).
+    const overlayDir = strFlag("overlay") ?? process.env.LASTLIGHT_OVERLAY_DIR;
+    bootstrapAssets({ overlayDir });
+    // A user/overlay can ship its own model registry too: explicit --models-file
+    // wins, else an overlay's `evals/models.json` if present, else the built-in.
+    const overlayModels = overlayDir ? join(overlayDir, "evals", "models.json") : undefined;
+    const modelsFile = strFlag("models-file") ?? (overlayModels && existsSync(overlayModels) ? overlayModels : undefined);
+    if (modelsFile)
+        setModelsPath(modelsFile);
+    // Discover tiers across built-in + user (--datasets) + overlay roots.
+    const userDatasetsDir = strFlag("datasets") ?? process.env.LASTLIGHT_EVALS_DATASETS;
+    const discovered = discoverTiers({
+        builtinRoot: builtinDatasetsRoot(),
+        userDatasetsDir,
+        overlayDir,
+    });
+    if (!hasProviderKey()) {
+        p.log.error("No provider key found. Set OPENAI_API_KEY (or ANTHROPIC_API_KEY / OPENROUTER_API_KEY)\n" +
+            "in your environment or .env, then re-run `npm run eval`.");
+        p.outro(chalk.red("aborted"));
+        return 1;
+    }
+    const compare = process.argv.includes("--compare");
+    const noOpen = process.argv.includes("--no-open") || !!process.env.CI;
+    const runs = intFlag("runs", 1);
+    // Positional tier names — skip flags AND the values that follow value-flags.
+    const argv = process.argv.slice(2);
+    const requested = [];
+    for (let i = 0; i < argv.length; i++) {
+        if (VALUE_FLAGS.has(argv[i])) {
+            i++; // its value isn't a tier
+            continue;
+        }
+        if (!argv[i].startsWith("-"))
+            requested.push(argv[i]);
+    }
+    const known = [...discovered.keys()];
+    if (!known.length) {
+        p.log.error("No datasets found. The package ships `triage`/`code-fix`; add your own via\n" +
+            "--datasets <dir> (or LASTLIGHT_EVALS_DATASETS), or --overlay <repo> with an\n" +
+            "`evals/datasets/` folder.");
+        p.outro(chalk.red("aborted"));
+        return 1;
+    }
+    const defaultTier = known.includes("triage") ? "triage" : known[0];
+    // Tiers come from argv when given; otherwise ask interactively (one or all).
+    // Non-interactive (CI / piped stdin) falls back to the cheapest default
+    // so automation never blocks on a prompt.
+    let chosen;
+    if (requested.length) {
+        chosen = requested;
+    }
+    else if (process.stdin.isTTY) {
+        const picked = await p.multiselect({
+            message: "Which tiers to run?",
+            options: known.map((name) => ({
+                value: name,
+                label: name + (discovered.get(name).source !== "builtin" ? chalk.dim(` (${discovered.get(name).source})`) : ""),
+                hint: discovered.get(name).description,
+            })),
+            initialValues: [defaultTier],
+            required: true,
+        });
+        if (p.isCancel(picked)) {
+            p.cancel("aborted");
+            return 1;
+        }
+        chosen = picked;
+    }
+    else {
+        chosen = [defaultTier];
+    }
+    // Stable display order (discovery order) regardless of pick order.
+    const tiers = known.filter((t) => chosen.includes(t));
+    for (const t of chosen) {
+        if (!discovered.has(t))
+            p.log.warn(`Unknown tier "${t}". Known: ${known.join(", ")}`);
+    }
+    // Model selection precedence:
+    //   1. --model / --models (or EVAL_MODEL[S]) — an explicit list, fuzzy-matched
+    //      against models.json; lets you test one model quickly.
+    //   2. --compare — the full cross-vendor set (key-gated).
+    //   3. default single model from models.json.
+    // Each entry carries its provider family (env-key) for parallel grouping.
+    const modelArg = strFlag("model") ?? strFlag("models");
+    const mode = modelArg ? "select" : compare ? "compare" : "single";
+    const entries = modelArg
+        ? modelArg
+            .split(",")
+            .map((s) => s.trim())
+            .filter(Boolean)
+            .map((tok) => {
+            const r = resolveModel(tok);
+            return { id: r.id, family: r.family };
+        })
+        : compare
+            ? compareModels().map((m) => ({ id: m.id, family: m.envKey ?? m.provider ?? "default" }))
+            : evalModels().map((id) => ({ id, family: "default" }));
+    if (!entries.length) {
+        p.log.error("No comparison models available — set provider keys (OPENAI_API_KEY / ANTHROPIC_API_KEY /\n" +
+            "FIREWORKS_API_KEY …) for the entries in evals/models.json.");
+        p.outro(chalk.red("aborted"));
+        return 1;
+    }
+    const labels = modelLabels();
+    // Resolve the work-list up front so we can show deterministic progress.
+    const work = [];
+    for (const tierName of tiers) {
+        const tier = discovered.get(tierName);
+        const instances = loadInstances(tier);
+        if (!instances.length) {
+            p.log.warn(`tier "${tierName}": no instances at ${tier.instancesPath} — skipping`);
+            continue;
+        }
+        for (const e of entries) {
+            for (const inst of instances) {
+                // Per-instance workflow wins, else the tier's defaultWorkflow (throws if
+                // neither is set — surfaced as a harness error for that case).
+                work.push({
+                    tierName,
+                    defaultWorkflow: workflowFor(tier, inst),
+                    datasetDir: tier.root,
+                    model: e.id,
+                    family: e.family,
+                    inst,
+                });
+            }
+        }
+    }
+    if (!work.length) {
+        p.log.error("Nothing to run — no datasets matched the requested tiers.");
+        p.outro(chalk.red("aborted"));
+        return 1;
+    }
+    // Group work by provider family. Families run CONCURRENTLY (independent
+    // provider keys / rate limits); within a family runs stay serial. Force
+    // serial with --serial or when there's only one family.
+    const byFamily = new Map();
+    for (const w of work) {
+        const arr = byFamily.get(w.family);
+        if (arr)
+            arr.push(w);
+        else
+            byFamily.set(w.family, [w]);
+    }
+    const parallel = !process.argv.includes("--serial") && byFamily.size > 1;
+    const resultsDir = join(resultsRoot(), `${tiers.join("+")}${compare ? "-compare" : ""}`);
+    const htmlBase = {
+        models: entries.map((e) => labels[e.id] ?? e.id),
+        tiers,
+        labels,
+        runs,
+    };
+    p.note(`${chalk.bold("mode")}    ${mode}${parallel ? chalk.dim(` (parallel · ${byFamily.size} families)`) : ""}\n` +
+        `${chalk.bold("models")}  ${entries.map((e) => labels[e.id] ?? e.id).join(", ")}\n` +
+        `${chalk.bold("tiers")}   ${tiers.join(", ")}\n` +
+        `${chalk.bold("cases")}   ${work.length}${runs > 1
+            ? chalk.dim(` × ${runs} trials = ${work.length * runs} runs · worst-case verdict, mean cost`)
+            : ""}`, "plan");
+    // `total` counts individual trials so live progress advances per model call.
+    const total = work.length * runs;
+    // Open the report immediately (live placeholder) so it fills in as we go.
+    writeHtml(resultsDir, summarize([]), { ...htmlBase, generatedAt: new Date().toISOString(), live: true, progress: `0/${total}` });
+    const htmlFile = join(resultsDir, "index.html");
+    if (!noOpen) {
+        openInBrowser(htmlFile);
+        p.log.info(`Live report → ${chalk.cyan(htmlFile)} ${chalk.dim("(auto-refreshing)")}`);
+    }
+    const all = [];
+    let harnessErrors = 0;
+    let completed = 0;
+    // Track in-flight cases so the live report can show running / queued rows.
+    const caseKey = (tier, model, id) => `${tier}|${model}|${id}`;
+    const running = new Set();
+    // writeHtml/summarize/all.push run synchronously to completion inside one
+    // event-loop turn, so even with concurrent families they never interleave.
+    const refresh = () => {
+        const done = new Set(all.map((r) => caseKey(r.tier ?? "", r.model, r.instance_id)));
+        const pending = work
+            .map((w) => ({ w, k: caseKey(w.tierName, w.model, w.inst.instance_id) }))
+            .filter(({ k }) => !done.has(k))
+            .map(({ w, k }) => ({
+            tier: w.tierName,
+            model: w.model,
+            instance_id: w.inst.instance_id,
+            status: (running.has(k) ? "running" : "pending"),
+        }));
+        writeHtml(resultsDir, summarize(all), {
+            ...htmlBase,
+            generatedAt: new Date().toISOString(),
+            live: true,
+            progress: `${completed}/${total}`,
+            pending,
+        });
+    };
+    // Run one case `runs` times and fold the trials into a single result
+    // (worst-case verdict, mean metrics). `onTrial` ticks per model call.
+    const runItem = async (w, onTrial) => {
+        const trials = [];
+        for (let t = 0; t < runs; t++) {
+            const r = await runInstance(w.inst, {
+                model: w.model,
+                datasetDir: w.datasetDir,
+                defaultWorkflow: w.defaultWorkflow,
+                manageEnv: false,
+            });
+            r.tier = w.tierName;
+            trials.push(r);
+            completed++;
+            onTrial();
+        }
+        return aggregateTrials(trials);
+    };
+    // Install the eval's static-token env ONCE for the whole batch so concurrent
+    // runs share one stable baseline (manageEnv:false on every runInstance).
+    const restoreEvalEnv = applyEvalEnv();
+    try {
+        if (parallel) {
+            // Per-family progress for the aggregate spinner line.
+            const fam = new Map();
+            for (const [f, items] of byFamily)
+                fam.set(f, { done: 0, total: items.length * runs });
+            const status = () => {
+                const segs = [...fam].map(([f, c]) => {
+                    const done = c.done === c.total ? chalk.green(`${c.done}/${c.total}`) : `${c.done}/${c.total}`;
+                    return `${familyLabel(f)} ${done}`;
+                });
+                return `${chalk.dim(`${completed}/${total}`)}  ${segs.join(chalk.dim(" · "))}`;
+            };
+            const s = p.spinner();
+            s.start(status());
+            const restoreConsole = silenceConsole();
+            const verdicts = [];
+            try {
+                await Promise.all([...byFamily].map(async ([f, items]) => {
+                    for (const w of items) {
+                        const k = caseKey(w.tierName, w.model, w.inst.instance_id);
+                        running.add(k);
+                        refresh();
+                        const result = await runItem(w, () => {
+                            fam.get(f).done++;
+                            s.message(status());
+                            refresh();
+                        });
+                        running.delete(k);
+                        all.push(result);
+                        if (result.error)
+                            harnessErrors++;
+                        const mark = result.error ? chalk.red("✗") : chalk.green("✓");
+                        verdicts.push(`${mark} ${chalk.dim(familyLabel(f))}  ${verdictLine(w.tierName, w.inst, result)}`);
+                        refresh();
+                    }
+                }));
+            }
+            finally {
+                restoreConsole();
+            }
+            s.stop(`${chalk.dim(`${completed}/${total}`)} ${chalk.green("done")}`);
+            p.log.message(verdicts.join("\n"));
+        }
+        else {
+            // Serial: one spinner per case (updates per trial) + a verdict line.
+            for (let i = 0; i < work.length; i++) {
+                const w = work[i];
+                const s = p.spinner();
+                const head = `${chalk.dim(`[${i + 1}/${work.length}]`)} ${chalk.cyan(w.tierName)}/${w.inst.instance_id}  ${chalk.dim(labels[w.model] ?? w.model)}`;
+                s.start(head);
+                const k = caseKey(w.tierName, w.model, w.inst.instance_id);
+                running.add(k);
+                refresh();
+                let t = 0;
+                const { value: result, logs } = await quiet(() => runItem(w, () => {
+                    t++;
+                    if (runs > 1)
+                        s.message(`${head}  ${chalk.dim(`trial ${t}/${runs}`)}`);
+                    refresh();
+                }));
+                running.delete(k);
+                all.push(result);
+                if (result.error)
+                    harnessErrors++;
+                const mark = result.error ? chalk.red("✗") : chalk.green("✓");
+                s.stop(`${chalk.dim(`[${i + 1}/${work.length}]`)} ${mark} ${verdictLine(w.tierName, w.inst, result)}`);
+                if (result.error) {
+                    p.log.error(chalk.dim(result.error));
+                    const tail = logs.split("\n").filter(Boolean).slice(-12).join("\n");
+                    if (tail)
+                        p.log.message(chalk.dim(tail));
+                }
+                refresh();
+            }
+        }
+    }
+    finally {
+        restoreEvalEnv();
+    }
+    // Final, static report + machine artifacts.
+    const card = summarize(all);
+    writeArtifacts(resultsDir, card);
+    const html = writeHtml(resultsDir, card, { ...htmlBase, generatedAt: new Date().toISOString() });
+    p.log.success(`Scorecard → ${chalk.cyan(html)}`);
+    p.log.success(`Artifacts → ${chalk.cyan(resultsDir)}/{scorecard.json,predictions.jsonl}`);
+    const ran = runs > 1 ? `${completed} runs (${all.length} cases × ${runs})` : `${all.length} runs`;
+    if (harnessErrors > 0) {
+        p.outro(chalk.yellow(`done — ${ran}, ${harnessErrors} harness error${harnessErrors === 1 ? "" : "s"} (see above)`));
+    }
+    else {
+        p.outro(chalk.green(`done — ${ran}, report at ${html}`));
+    }
+    // Non-zero ONLY on harness failure — model quality is the measurement.
+    return harnessErrors > 0 ? 1 : 0;
+}
+/** Top-level subcommand dispatcher: `run` (default) | `init`. */
+async function main() {
+    const sub = process.argv[2];
+    if (sub === "init") {
+        // `init [dir]` — scaffold a fresh overlay+evals repo.
+        return runInit(process.argv[3]);
+    }
+    // `run` is the default; allow an explicit leading `run` token too.
+    if (sub === "run")
+        process.argv.splice(2, 1);
+    return runEval();
+}
+main()
+    .then((code) => process.exit(code))
+    .catch((err) => {
+    console.error(err);
+    process.exit(1);
+});
+//# sourceMappingURL=run.js.map

package/dist/run.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"run.js","sourceRoot":"","sources":["../src/run.ts"],"names":[],"mappings":";AACA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAE3C,OAAO,KAAK,CAAC,MAAM,gBAAgB,CAAC;AACpC,OAAO,KAAK,MAAM,OAAO,CAAC;AAE1B,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,UAAU,EAAE,aAAa,EAAE,WAAW,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAC3H,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAC9D,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AACzE,OAAO,EAAE,SAAS,EAAiB,MAAM,kBAAkB,CAAC;AAE5D,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,WAAW,EAAa,MAAM,gBAAgB,CAAC;AACtF,OAAO,EAAE,mBAAmB,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAC9D,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,yEAAyE;AACzE,SAAS,aAAa,CAAC,IAAY;IACjC,MAAM,GAAG,GAAG,UAAU,IAAI,EAAE,CAAC;IAC7B,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,GACf,OAAO,CAAC,QAAQ,KAAK,QAAQ;QAC3B,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC;QACjB,CAAC,CAAC,OAAO,CAAC,QAAQ,KAAK,OAAO;YAC5B,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,GAAG,CAAC,CAAC;YACnC,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IAC5B,IAAI,CAAC;QACH,KAAK,CAAC,GAAa,EAAE,IAAgB,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;IACtF,CAAC;IAAC,MAAM,CAAC;QACP,wDAAwD;IAC1D,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,KAAK,CAAI,EAAoB;IAC1C,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,MAAM,GAAG,GACP,CAAC,IAA+B,EAAE,EAAE,CACpC,CAAC,GAAG,CAAY,EAAE,EAAE;QAClB,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QAC1E,KAAK,IAAI,CAAC;IACZ,CAAC,CAAC;IACJ,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC;IAC3C,OAAO,CAAC,GAAG,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;IACvB,OAAO,CAAC,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC;IACzB,OAAO,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC;IAC3B,OAAO,CAAC,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC;IACzB,IAAI,CAAC;QACH,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;IACrD,CAAC;YAAS,CAAC;QACT,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACrD,CAAC;AACH,CAAC;AAED,SAAS,KAAK,CAAC,EAAU;IACvB,IAAI,CAAC,EAAE;QAAE,OAAO,GAAG,CAAC;IACpB,IAAI,EAAE,GAAG,IAAI;QAAE,OAAO,GAAG,EAAE,IAAI,CAAC;IAChC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC;IAChC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,CAAC,IAAI,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;AACvF,CAAC;AAED,iFAAiF;AACjF,SAAS,WAAW,CAAC,MAAc;IACjC,OAAO,MAAM,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,IAAI,SAAS,CAAC;AACrE,CAAC;AAED;;;;;GAKG;AACH,SAAS,cAAc;IACrB,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC;IAC3C,MAAM,IAAI,GAAG,GAAG,EAAE,GAAE,CAAC,CAAC;IACtB,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3E,OAAO,GAAG,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AAClE,CAAC;AAED,gFAAgF;AAChF,SAAS,WAAW,CAAC,QAAgB,EAAE,IAAsB,EAAE,CAAiB;IAC9E,MAAM,IAAI,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;IAC3D,IAAI,CAAC,CAAC,KAAK;QAAE,OAAO,GAAG,IAAI,KAAK,KAAK,CAAC,GAAG,CAAC,eAAe,CAAC,EAAE,CAAC;IAC7D,MAAM,KAAK,GAAG,CAAC,IAAa,EAAE,EAAE,CAAC,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC3G,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,CAAC,CAAC,QAAQ,KAAK,SAAS;QAC1B,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC;IACvG,IAAI,CAAC,CAAC,UAAU;QACd,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC;IACpH,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAClD,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;IAC3C,OAAO,GAAG,IAAI,KAAK,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;AACxC,CAAC;AAED,+EAA+E;AAC/E,SAAS,OAAO,CAAC,IAAY,EAAE,GAAW;IACxC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAC1B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,KAAK,IAAI,EAAE,EAAE,CAAC;YAC5B,MAAM,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;YAC1C,IAAI,CAAC,GAAG,CAAC;gBAAE,OAAO,CAAC,CAAC;QACtB,CAAC;QACD,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,IAAI,UAAU,CAAC,CAAC,CAAC;QAC1D,IAAI,CAAC;YAAE,OAAO,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACnC,CAAC;IACD,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;IAC1E,OAAO,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;AAC7B,CAAC;AAED,6EAA6E;AAC7E,SAAS,OAAO,CAAC,IAAY;IAC3B,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAC1B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,KAAK,IAAI,EAAE,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,SAAS;YAAE,OAAO,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7E,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,IAAI,QAAQ,CAAC,CAAC,CAAC;QACxD,IAAI,CAAC;YAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IACrB,CAAC;IACD,OAAO,OAAO,CAAC,GAAG,CAAC,QAAQ,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;AACnD,CAAC;AAED,+EAA+E;AAC/E,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,CAAC,QAAQ,EAAE,SAAS,EAAE,UAAU,EAAE,WAAW,EAAE,YAAY,EAAE,eAAe,CAAC,CAAC,CAAC;AAE3G,KAAK,UAAU,OAAO;IACpB,UAAU,EAAE,CAAC;IACb,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,cAAc,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;IAE5D,0EAA0E;IAC1E,6EAA6E;IAC7E,yEAAyE;IACzE,MAAM,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,IAAI,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC;IAC3E,eAAe,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC;IAEhC,6EAA6E;IAC7E,6EAA6E;IAC7E,MAAM,aAAa,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,EAAE,OAAO,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACxF,MAAM,UAAU,GAAG,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,aAAa,IAAI,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACtH,IAAI,UAAU;QAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IAE1C,sEAAsE;IACtE,MAAM,eAAe,GAAG,OAAO,CAAC,UAAU,CAAC,IAAI,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC;IACpF,MAAM,UAAU,GAAG,aAAa,CAAC;QAC/B,WAAW,EAAE,mBAAmB,EAAE;QAClC,eAAe;QACf,UAAU;KACX,CAAC,CAAC;IAEH,IAAI,CAAC,cAAc,EAAE,EAAE,CAAC;QACtB,CAAC,CAAC,GAAG,CAAC,KAAK,CACT,yFAAyF;YACvF,0DAA0D,CAC7D,CAAC;QACF,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC;QAC9B,OAAO,CAAC,CAAC;IACX,CAAC;IAED,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IACnD,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;IACtE,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAEhC,6EAA6E;IAC7E,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACnC,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,IAAI,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7B,CAAC,EAAE,CAAC,CAAC,yBAAyB;YAC9B,SAAS;QACX,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IACxD,CAAC;IAED,MAAM,KAAK,GAAG,CAAC,GAAG,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC;IACrC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QAClB,CAAC,CAAC,GAAG,CAAC,KAAK,CACT,8EAA8E;YAC5E,+EAA+E;YAC/E,2BAA2B,CAC9B,CAAC;QACF,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC;QAC9B,OAAO,CAAC,CAAC;IACX,CAAC;IACD,MAAM,WAAW,GAAG,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAEnE,6EAA6E;IAC7E,wEAAwE;IACxE,0CAA0C;IAC1C,IAAI,MAAgB,CAAC;IACrB,IAAI,SAAS,CAAC,MAAM,EAAE,CAAC;QACrB,MAAM,GAAG,SAAS,CAAC;IACrB,CAAC;SAAM,IAAI,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,WAAW,CAAC;YACjC,OAAO,EAAE,qBAAqB;YAC9B,OAAO,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;gBAC5B,KAAK,EAAE,IAAI;gBACX,KAAK,EAAE,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,UAAU,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBACjH,IAAI,EAAE,UAAU,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC,WAAW;aACxC,CAAC,CAAC;YACH,aAAa,EAAE,CAAC,WAAW,CAAC;YAC5B,QAAQ,EAAE,IAAI;SACf,CAAC,CAAC;QACH,IAAI,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YACvB,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YACpB,OAAO,CAAC,CAAC;QACX,CAAC;QACD,MAAM,GAAG,MAAkB,CAAC;IAC9B,CAAC;SAAM,CAAC;QACN,MAAM,GAAG,CAAC,WAAW,CAAC,CAAC;IACzB,CAAC;IAED,mEAAmE;IACnE,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;IACtD,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,iBAAiB,CAAC,aAAa,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACxF,CAAC;IAED,8BAA8B;IAC9B,+EAA+E;IAC/E,6DAA6D;IAC7D,0DAA0D;IAC1D,8CAA8C;IAC9C,0EAA0E;IAC1E,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,CAAC;IACvD,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC;IAClE,MAAM,OAAO,GAAqC,QAAQ;QACxD,CAAC,CAAC,QAAQ;aACL,KAAK,CAAC,GAAG,CAAC;aACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;aACpB,MAAM,CAAC,OAAO,CAAC;aACf,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YACX,MAAM,CAAC,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;YAC5B,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC;QACxC,CAAC,CAAC;QACN,CAAC,CAAC,OAAO;YACP,CAAC,CAAC,aAAa,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,QAAQ,IAAI,SAAS,EAAE,CAAC,CAAC;YACzF,CAAC,CAAC,UAAU,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC;IAC5D,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QACpB,CAAC,CAAC,GAAG,CAAC,KAAK,CACT,4FAA4F;YAC1F,4DAA4D,CAC/D,CAAC;QACF,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC;QAC9B,OAAO,CAAC,CAAC;IACX,CAAC;IACD,MAAM,MAAM,GAAG,WAAW,EAAE,CAAC;IAW7B,wEAAwE;IACxE,MAAM,IAAI,GAAe,EAAE,CAAC;IAC5B,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;QAC7B,MAAM,IAAI,GAAS,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC;QAC7C,MAAM,SAAS,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;QACtC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC;YACtB,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,QAAQ,sBAAsB,IAAI,CAAC,aAAa,aAAa,CAAC,CAAC;YACnF,SAAS;QACX,CAAC;QACD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;gBAC7B,yEAAyE;gBACzE,+DAA+D;gBAC/D,IAAI,CAAC,IAAI,CAAC;oBACR,QAAQ;oBACR,eAAe,EAAE,WAAW,CAAC,IAAI,EAAE,IAAI,CAAC;oBACxC,UAAU,EAAE,IAAI,CAAC,IAAI;oBACrB,KAAK,EAAE,CAAC,CAAC,EAAE;oBACX,MAAM,EAAE,CAAC,CAAC,MAAM;oBAChB,IAAI;iBACL,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QACjB,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,2DAA2D,CAAC,CAAC;QACzE,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC;QAC9B,OAAO,CAAC,CAAC;IACX,CAAC;IAED,wEAAwE;IACxE,wEAAwE;IACxE,wDAAwD;IACxD,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAsB,CAAC;IAC/C,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACnC,IAAI,GAAG;YAAE,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;;YAChB,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACnC,CAAC;IACD,MAAM,QAAQ,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,QAAQ,CAAC,IAAI,GAAG,CAAC,CAAC;IAEzE,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,EAAE,EAAE,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACzF,MAAM,QAAQ,GAAwD;QACpE,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QAChD,KAAK;QACL,MAAM;QACN,IAAI;KACL,CAAC;IAEF,CAAC,CAAC,IAAI,CACJ,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,GAC9B,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,gBAAgB,QAAQ,CAAC,IAAI,YAAY,CAAC,CAAC,CAAC,CAAC,EACpE,IAAI;QACF,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI;QACnF,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI;QAChD,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,CAAC,MAAM,GACrC,IAAI,GAAG,CAAC;YACN,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,IAAI,aAAa,IAAI,CAAC,MAAM,GAAG,IAAI,uCAAuC,CAAC;YAC7F,CAAC,CAAC,EACN,EAAE,EACJ,MAAM,CACP,CAAC;IAEF,6EAA6E;IAC7E,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;IAEjC,0EAA0E;IAC1E,SAAS,CAAC,UAAU,EAAE,SAAS,CAAC,EAAE,CAAC,EAAE,EAAE,GAAG,QAAQ,EAAE,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,KAAK,EAAE,EAAE,CAAC,CAAC;IACjI,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;IAChD,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,aAAa,CAAC,QAAQ,CAAC,CAAC;QACxB,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,iBAAiB,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,mBAAmB,CAAC,EAAE,CAAC,CAAC;IACxF,CAAC;IAED,MAAM,GAAG,GAAqB,EAAE,CAAC;IACjC,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,2EAA2E;IAC3E,MAAM,OAAO,GAAG,CAAC,IAAY,EAAE,KAAa,EAAE,EAAU,EAAE,EAAE,CAAC,GAAG,IAAI,IAAI,KAAK,IAAI,EAAE,EAAE,CAAC;IACtF,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAElC,0EAA0E;IAC1E,2EAA2E;IAC3E,MAAM,OAAO,GAAG,GAAG,EAAE;QACnB,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;QACpF,MAAM,OAAO,GAAG,IAAI;aACjB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;aACxE,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;aAC/B,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;YAClB,IAAI,EAAE,CAAC,CAAC,QAAQ;YAChB,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,WAAW,EAAE,CAAC,CAAC,IAAI,CAAC,WAAW;YAC/B,MAAM,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAA0B;SAC1E,CAAC,CAAC,CAAC;QACN,SAAS,CAAC,UAAU,EAAE,SAAS,CAAC,GAAG,CAAC,EAAE;YACpC,GAAG,QAAQ;YACX,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACrC,IAAI,EAAE,IAAI;YACV,QAAQ,EAAE,GAAG,SAAS,IAAI,KAAK,EAAE;YACjC,OAAO;SACR,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,qEAAqE;IACrE,sEAAsE;IACtE,MAAM,OAAO,GAAG,KAAK,EAAE,CAAW,EAAE,OAAmB,EAA2B,EAAE;QAClF,MAAM,MAAM,GAAqB,EAAE,CAAC;QACpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;YAC9B,MAAM,CAAC,GAAG,MAAM,WAAW,CAAC,CAAC,CAAC,IAAI,EAAE;gBAClC,KAAK,EAAE,CAAC,CAAC,KAAK;gBACd,UAAU,EAAE,CAAC,CAAC,UAAU;gBACxB,eAAe,EAAE,CAAC,CAAC,eAAe;gBAClC,SAAS,EAAE,KAAK;aACjB,CAAC,CAAC;YACH,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,QAAQ,CAAC;YACpB,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACf,SAAS,EAAE,CAAC;YACZ,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,OAAO,eAAe,CAAC,MAAM,CAAC,CAAC;IACjC,CAAC,CAAC;IAEF,6EAA6E;IAC7E,yEAAyE;IACzE,MAAM,cAAc,GAAG,YAAY,EAAE,CAAC;IACtC,IAAI,CAAC;QACH,IAAI,QAAQ,EAAE,CAAC;YACb,sDAAsD;YACtD,MAAM,GAAG,GAAG,IAAI,GAAG,EAA2C,CAAC;YAC/D,KAAK,MAAM,CAAC,CAAC,EAAE,KAAK,CAAC,IAAI,QAAQ;gBAAE,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC,CAAC;YACvF,MAAM,MAAM,GAAG,GAAG,EAAE;gBAClB,MAAM,IAAI,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE;oBACnC,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;oBAC/F,OAAO,GAAG,WAAW,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;gBACrC,CAAC,CAAC,CAAC;gBACH,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,SAAS,IAAI,KAAK,EAAE,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;YACjF,CAAC,CAAC;YACF,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,CAAC;YACtB,CAAC,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;YAClB,MAAM,cAAc,GAAG,cAAc,EAAE,CAAC;YACxC,MAAM,QAAQ,GAAa,EAAE,CAAC;YAC9B,IAAI,CAAC;gBACH,MAAM,OAAO,CAAC,GAAG,CACf,CAAC,GAAG,QAAQ,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,EAAE;oBACrC,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;wBACtB,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;wBAC3D,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;wBACf,OAAO,EAAE,CAAC;wBACV,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,CAAC,EAAE,GAAG,EAAE;4BACnC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,CAAC;4BACnB,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;4BACpB,OAAO,EAAE,CAAC;wBACZ,CAAC,CAAC,CAAC;wBACH,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;wBAClB,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;wBACjB,IAAI,MAAM,CAAC,KAAK;4BAAE,aAAa,EAAE,CAAC;wBAClC,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;wBAC9D,QAAQ,CAAC,IAAI,CAAC,GAAG,IAAI,IAAI,KAAK,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,KAAK,WAAW,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;wBAClG,OAAO,EAAE,CAAC;oBACZ,CAAC;gBACH,CAAC,CAAC,CACH,CAAC;YACJ,CAAC;oBAAS,CAAC;gBACT,cAAc,EAAE,CAAC;YACnB,CAAC;YACD,CAAC,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,SAAS,IAAI,KAAK,EAAE,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YACvE,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QACrC,CAAC;aAAM,CAAC;YACN,qEAAqE;YACrE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACrC,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;gBAClB,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,CAAC;gBACtB,MAAM,IAAI,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,WAAW,KAAK,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;gBACnJ,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAEd,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;gBAC3D,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBACf,OAAO,EAAE,CAAC;gBACV,IAAI,CAAC,GAAG,CAAC,CAAC;gBACV,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,CAC/C,OAAO,CAAC,CAAC,EAAE,GAAG,EAAE;oBACd,CAAC,EAAE,CAAC;oBACJ,IAAI,IAAI,GAAG,CAAC;wBAAE,CAAC,CAAC,OAAO,CAAC,GAAG,IAAI,KAAK,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;oBACvE,OAAO,EAAE,CAAC;gBACZ,CAAC,CAAC,CACH,CAAC;gBACF,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBAClB,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACjB,IAAI,MAAM,CAAC,KAAK;oBAAE,aAAa,EAAE,CAAC;gBAElC,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAC9D,CAAC,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,IAAI,WAAW,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;gBACvG,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;oBACjB,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;oBACrC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACpE,IAAI,IAAI;wBAAE,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;gBAC3C,CAAC;gBACD,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC;IACH,CAAC;YAAS,CAAC;QACT,cAAc,EAAE,CAAC;IACnB,CAAC;IAED,4CAA4C;IAC5C,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;IAC5B,cAAc,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;IACjC,MAAM,IAAI,GAAG,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,EAAE,GAAG,QAAQ,EAAE,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;IAEjG,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,eAAe,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjD,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,eAAe,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,qCAAqC,CAAC,CAAC;IAE1F,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,SAAS,UAAU,GAAG,CAAC,MAAM,YAAY,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,MAAM,OAAO,CAAC;IAClG,IAAI,aAAa,GAAG,CAAC,EAAE,CAAC;QACtB,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,GAAG,KAAK,aAAa,iBAAiB,aAAa,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,cAAc,CAAC,CAAC,CAAC;IACtH,CAAC;SAAM,CAAC;QACN,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,UAAU,GAAG,eAAe,IAAI,EAAE,CAAC,CAAC,CAAC;IAC3D,CAAC;IAED,uEAAuE;IACvE,OAAO,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACnC,CAAC;AAED,iEAAiE;AACjE,KAAK,UAAU,IAAI;IACjB,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC5B,IAAI,GAAG,KAAK,MAAM,EAAE,CAAC;QACnB,sDAAsD;QACtD,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAClC,CAAC;IACD,mEAAmE;IACnE,IAAI,GAAG,KAAK,KAAK;QAAE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC7C,OAAO,OAAO,EAAE,CAAC;AACnB,CAAC;AAED,IAAI,EAAE;KACH,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;KAClC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IACb,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACnB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}

package/dist/schema.js

@@ -0,0 +1,15 @@
+/**
+ * Eval data model — SWE-bench-compatible.
+ *
+ * A benchmark case is a {@link SweBenchInstance}. The core fields mirror the
+ * SWE-bench / SWE-bench Lite dataset schema field-for-field (`instance_id`,
+ * `base_commit`, `problem_statement`, gold `patch`, held-out `test_patch`,
+ * `FAIL_TO_PASS`, `PASS_TO_PASS`), so a case is interchangeable with a real
+ * SWE-bench row and our results export to SWE-bench's predictions format.
+ *
+ * The `Last Light extensions` block (ignored by real SWE-bench) carries the
+ * GitHub fixtures + behavioral expectations that let us drive — and grade —
+ * the REAL production workflow with a mocked GitHub.
+ */
+export {};
+//# sourceMappingURL=schema.js.map

package/dist/schema.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.js","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG"}

package/dist/seed.js

@@ -0,0 +1,49 @@
+/**
+ * Deterministic workspace seeding for the code-fix tier.
+ *
+ * Pre-populates the run's sandbox workspace (`<stateDir>/sandboxes/<taskId>`,
+ * the exact dir `setupTaskWorktree` would create) with a fixture repo checked
+ * out at a fixed base commit, and points its `origin` at a LOCAL bare repo —
+ * so the real workflow's `git push origin HEAD` succeeds fully offline with NO
+ * GitHub clone. Because the eval calls `runWorkflow` with no
+ * `ctx.prePopulateBranch`, the runner never triggers its own GitHub clone and
+ * the agent works directly in this seeded dir.
+ */
+import { execFileSync } from "node:child_process";
+import { mkdirSync, cpSync } from "node:fs";
+import { resolve } from "node:path";
+const FIXED = "2026-01-01T00:00:00 +0000";
+const GIT_ENV = {
+    ...process.env,
+    GIT_AUTHOR_NAME: "eval",
+    GIT_AUTHOR_EMAIL: "eval@example.com",
+    GIT_COMMITTER_NAME: "eval",
+    GIT_COMMITTER_EMAIL: "eval@example.com",
+    GIT_AUTHOR_DATE: FIXED,
+    GIT_COMMITTER_DATE: FIXED,
+};
+function git(cwd, args) {
+    return execFileSync("git", args, { cwd, env: GIT_ENV, stdio: ["ignore", "pipe", "pipe"] }).toString();
+}
+export function seedWorkspace(opts) {
+    const def = opts.defaultBranch ?? "main";
+    const sandboxBase = resolve(opts.stateDir, "sandboxes");
+    const workDir = resolve(sandboxBase, opts.taskId);
+    mkdirSync(workDir, { recursive: true });
+    cpSync(opts.fixtureDir, workDir, { recursive: true });
+    git(workDir, ["init", "-q", "-b", def]);
+    git(workDir, ["add", "-A"]);
+    git(workDir, ["commit", "-q", "-m", "base"]);
+    const baseCommit = git(workDir, ["rev-parse", "HEAD"]).trim();
+    const originsDir = resolve(opts.stateDir, "origins");
+    mkdirSync(originsDir, { recursive: true });
+    const originDir = resolve(originsDir, `${opts.taskId}.git`);
+    git(workDir, ["init", "--bare", "-q", originDir]);
+    git(workDir, ["remote", "add", "origin", `file://${originDir}`]);
+    git(workDir, ["push", "-q", "origin", `HEAD:refs/heads/${def}`]);
+    const branch = opts.branch ?? def;
+    if (branch !== def)
+        git(workDir, ["checkout", "-q", "-b", branch]);
+    return { workDir, originDir, baseCommit, branch };
+}
+//# sourceMappingURL=seed.js.map

package/dist/seed.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"seed.js","sourceRoot":"","sources":["../src/seed.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,MAAM,KAAK,GAAG,2BAA2B,CAAC;AAC1C,MAAM,OAAO,GAAsB;IACjC,GAAG,OAAO,CAAC,GAAG;IACd,eAAe,EAAE,MAAM;IACvB,gBAAgB,EAAE,kBAAkB;IACpC,kBAAkB,EAAE,MAAM;IAC1B,mBAAmB,EAAE,kBAAkB;IACvC,eAAe,EAAE,KAAK;IACtB,kBAAkB,EAAE,KAAK;CAC1B,CAAC;AAEF,SAAS,GAAG,CAAC,GAAW,EAAE,IAAc;IACtC,OAAO,YAAY,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC;AACxG,CAAC;AASD,MAAM,UAAU,aAAa,CAAC,IAQ7B;IACC,MAAM,GAAG,GAAG,IAAI,CAAC,aAAa,IAAI,MAAM,CAAC;IACzC,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;IACxD,MAAM,OAAO,GAAG,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IAClD,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACxC,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAEtD,GAAG,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC;IACxC,GAAG,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC;IAC5B,GAAG,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;IAC7C,MAAM,UAAU,GAAG,GAAG,CAAC,OAAO,EAAE,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAE9D,MAAM,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IACrD,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3C,MAAM,SAAS,GAAG,OAAO,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,MAAM,MAAM,CAAC,CAAC;IAC5D,GAAG,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC;IAClD,GAAG,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,UAAU,SAAS,EAAE,CAAC,CAAC,CAAC;IACjE,GAAG,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,mBAAmB,GAAG,EAAE,CAAC,CAAC,CAAC;IAEjE,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,GAAG,CAAC;IAClC,IAAI,MAAM,KAAK,GAAG;QAAE,GAAG,CAAC,OAAO,EAAE,CAAC,UAAU,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;IAEnE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC;AACpD,CAAC"}

package/models.json

@@ -0,0 +1,17 @@
+{
+  "_comment": "Models for the eval harness. `default` is what `npm run eval` uses (single model). `compare` is the cross-vendor set `npm run eval:compare` runs — each entry only runs if its `envKey` is present in the environment, so add/remove freely. `id` is the agentic-pi/pi-ai 'provider/model' spec; `label` is the display name on the scorecard.",
+  "default": "openai/gpt-5.4-mini",
+  "compare": [
+    { "id": "openai/gpt-5.5", "label": "GPT-5.5", "provider": "OpenAI", "envKey": "OPENAI_API_KEY" },
+    { "id": "openai/gpt-5.4-mini", "label": "GPT-5.4 mini", "provider": "OpenAI", "envKey": "OPENAI_API_KEY" },
+    { "id": "openai/gpt-5.3-codex", "label": "GPT-5.3 Codex", "provider": "OpenAI", "envKey": "OPENAI_API_KEY" },
+
+    { "id": "anthropic/claude-opus-4-8", "label": "Claude Opus 4.8", "provider": "Anthropic", "envKey": "ANTHROPIC_API_KEY" },
+    { "id": "anthropic/claude-sonnet-4-6", "label": "Claude Sonnet 4.6", "provider": "Anthropic", "envKey": "ANTHROPIC_API_KEY" },
+    { "id": "anthropic/claude-haiku-4-5", "label": "Claude Haiku 4.5", "provider": "Anthropic", "envKey": "ANTHROPIC_API_KEY" },
+
+    { "id": "fireworks/accounts/fireworks/models/glm-5p2", "label": "GLM-5.2", "provider": "Fireworks (open)", "envKey": "FIREWORKS_API_KEY" },
+    { "id": "fireworks/accounts/fireworks/models/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "provider": "Fireworks (open)", "envKey": "FIREWORKS_API_KEY" },
+    { "id": "fireworks/accounts/fireworks/models/gpt-oss-120b", "label": "GPT-OSS 120B", "provider": "Fireworks (open)", "envKey": "FIREWORKS_API_KEY" }
+  ]
+}