lastlight-evals 0.1.0

package/dist/metrics.js

@@ -0,0 +1,86 @@
+/**
+ * Pull per-run token/cost/turn metrics out of the session jsonl the event
+ * shim writes (`src/engine/event-shim.ts`). Each phase emits one `result`
+ * envelope carrying `total_cost_usd` / `total_input_tokens` /
+ * `total_output_tokens`. We sum them across every jsonl under the run's
+ * sessions dir. Best-effort: missing files / lines just contribute zero.
+ */
+import { readdirSync, readFileSync, existsSync, statSync } from "node:fs";
+import { join } from "node:path";
+/**
+ * The in-process executor flushes its FINAL session envelope (the `result`
+ * line that carries cost/tokens) fire-and-forget (`void shim.flush()`), so it
+ * can land just after `runWorkflow` resolves. Wait for the jsonl tree to stop
+ * growing before reading metrics, so we don't miss it (and don't delete the
+ * workspace out from under the pending write).
+ */
+export async function drainSessions(sessionsDir, maxMs = 4000, quietMs = 250) {
+    const sizeOf = () => {
+        const files = [];
+        walkJsonl(join(sessionsDir, "projects"), files);
+        return files.reduce((sum, f) => {
+            try {
+                return sum + statSync(f).size;
+            }
+            catch {
+                return sum;
+            }
+        }, 0);
+    };
+    const deadline = Date.now() + maxMs;
+    let last = -1;
+    let stableSince = Date.now();
+    while (Date.now() < deadline) {
+        const cur = sizeOf();
+        if (cur > 0 && cur === last) {
+            if (Date.now() - stableSince >= quietMs)
+                return;
+        }
+        else {
+            last = cur;
+            stableSince = Date.now();
+        }
+        await new Promise((r) => setTimeout(r, 80));
+    }
+}
+function walkJsonl(dir, out) {
+    if (!existsSync(dir))
+        return;
+    for (const name of readdirSync(dir, { withFileTypes: true })) {
+        const full = join(dir, name.name);
+        if (name.isDirectory())
+            walkJsonl(full, out);
+        else if (name.name.endsWith(".jsonl"))
+            out.push(full);
+    }
+}
+export function collectMetrics(sessionsDir) {
+    const files = [];
+    walkJsonl(join(sessionsDir, "projects"), files);
+    // Fallback: some shim configs write directly under sessionsDir.
+    walkJsonl(sessionsDir, files);
+    const seen = new Set();
+    const metrics = { inputTokens: 0, outputTokens: 0, costUsd: 0 };
+    for (const file of files) {
+        if (seen.has(file))
+            continue;
+        seen.add(file);
+        for (const line of readFileSync(file, "utf8").split("\n")) {
+            if (!line.trim() || !line.includes('"result"'))
+                continue;
+            try {
+                const env = JSON.parse(line);
+                if (env.type !== "result")
+                    continue;
+                metrics.inputTokens += env.total_input_tokens ?? 0;
+                metrics.outputTokens += env.total_output_tokens ?? 0;
+                metrics.costUsd += env.total_cost_usd ?? 0;
+            }
+            catch {
+                /* ignore malformed lines */
+            }
+        }
+    }
+    return metrics;
+}
+//# sourceMappingURL=metrics.js.map

package/dist/metrics.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../src/metrics.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAC1E,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAQjC;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,WAAmB,EAAE,KAAK,GAAG,IAAI,EAAE,OAAO,GAAG,GAAG;IAClF,MAAM,MAAM,GAAG,GAAW,EAAE;QAC1B,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,SAAS,CAAC,IAAI,CAAC,WAAW,EAAE,UAAU,CAAC,EAAE,KAAK,CAAC,CAAC;QAChD,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE;YAC7B,IAAI,CAAC;gBACH,OAAO,GAAG,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YAChC,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,GAAG,CAAC;YACb,CAAC;QACH,CAAC,EAAE,CAAC,CAAC,CAAC;IACR,CAAC,CAAC;IACF,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;IACpC,IAAI,IAAI,GAAG,CAAC,CAAC,CAAC;IACd,IAAI,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,EAAE,CAAC;QAC7B,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC;QACrB,IAAI,GAAG,GAAG,CAAC,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;YAC5B,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW,IAAI,OAAO;gBAAE,OAAO;QAClD,CAAC;aAAM,CAAC;YACN,IAAI,GAAG,GAAG,CAAC;YACX,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC3B,CAAC;QACD,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;IAC9C,CAAC;AACH,CAAC;AAED,SAAS,SAAS,CAAC,GAAW,EAAE,GAAa;IAC3C,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,OAAO;IAC7B,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;QAC7D,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;QAClC,IAAI,IAAI,CAAC,WAAW,EAAE;YAAE,SAAS,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;aACxC,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;YAAE,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACxD,CAAC;AACH,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,WAAmB;IAChD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,SAAS,CAAC,IAAI,CAAC,WAAW,EAAE,UAAU,CAAC,EAAE,KAAK,CAAC,CAAC;IAChD,gEAAgE;IAChE,SAAS,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;IAE9B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,OAAO,GAAe,EAAE,WAAW,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;IAC5E,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,SAAS;QAC7B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACf,KAAK,MAAM,IAAI,IAAI,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YAC1D,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC;gBAAE,SAAS;YACzD,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAK1B,CAAC;gBACF,IAAI,GAAG,CAAC,IAAI,KAAK,QAAQ;oBAAE,SAAS;gBACpC,OAAO,CAAC,WAAW,IAAI,GAAG,CAAC,kBAAkB,IAAI,CAAC,CAAC;gBACnD,OAAO,CAAC,YAAY,IAAI,GAAG,CAAC,mBAAmB,IAAI,CAAC,CAAC;gBACrD,OAAO,CAAC,OAAO,IAAI,GAAG,CAAC,cAAc,IAAI,CAAC,CAAC;YAC7C,CAAC;YAAC,MAAM,CAAC;gBACP,4BAA4B;YAC9B,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/dist/paths.js

@@ -0,0 +1,33 @@
+/**
+ * Filesystem anchors for the eval package.
+ *
+ * Built-in assets (the shipped sample `datasets/` + `models.json`) live at the
+ * PACKAGE ROOT — one level above this file's dir, which is `src/` under tsx in
+ * dev and `dist/` when built+installed. Resolving relative to `import.meta.url`
+ * (not `process.cwd()`) keeps them findable no matter where the CLI is invoked
+ * from — including out of `node_modules/lastlight-evals/`.
+ *
+ * Run OUTPUT, by contrast, is written under the caller's cwd (an installed
+ * package dir is read-only), overridable via `LASTLIGHT_EVALS_OUT`.
+ */
+import { dirname, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+/** Package root: the dir holding `datasets/`, `models.json`, `package.json`. */
+export function packageRoot() {
+    return resolve(dirname(fileURLToPath(import.meta.url)), "..");
+}
+/** Shipped sample datasets root (`<pkg>/datasets`). */
+export function builtinDatasetsRoot() {
+    return resolve(packageRoot(), "datasets");
+}
+/** Shipped default model registry (`<pkg>/models.json`). */
+export function builtinModelsPath() {
+    return resolve(packageRoot(), "models.json");
+}
+/** Where scorecards/artifacts are written (cwd-relative, NOT the package dir). */
+export function resultsRoot() {
+    return process.env.LASTLIGHT_EVALS_OUT
+        ? resolve(process.env.LASTLIGHT_EVALS_OUT)
+        : resolve(process.cwd(), "eval-results");
+}
+//# sourceMappingURL=paths.js.map

package/dist/paths.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"paths.js","sourceRoot":"","sources":["../src/paths.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,gFAAgF;AAChF,MAAM,UAAU,WAAW;IACzB,OAAO,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;AAChE,CAAC;AAED,uDAAuD;AACvD,MAAM,UAAU,mBAAmB;IACjC,OAAO,OAAO,CAAC,WAAW,EAAE,EAAE,UAAU,CAAC,CAAC;AAC5C,CAAC;AAED,4DAA4D;AAC5D,MAAM,UAAU,iBAAiB;IAC/B,OAAO,OAAO,CAAC,WAAW,EAAE,EAAE,aAAa,CAAC,CAAC;AAC/C,CAAC;AAED,kFAAkF;AAClF,MAAM,UAAU,WAAW;IACzB,OAAO,OAAO,CAAC,GAAG,CAAC,mBAAmB;QACpC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC;QAC1C,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,cAAc,CAAC,CAAC;AAC7C,CAAC"}

package/dist/report.js

@@ -0,0 +1,137 @@
+/**
+ * Scorecard rendering + SWE-bench-compatible artifacts.
+ *
+ *  - A stdout table comparing models on resolved% / triage-correct% / tokens /
+ *    cost / latency.
+ *  - `scorecard.json`  — the structured roll-up.
+ *  - `predictions.jsonl` — SWE-bench predictions shape
+ *    (`{ instance_id, model_name_or_path, model_patch }`), so the same artifact
+ *    is consumable by SWE-bench's own harness.
+ */
+import { mkdirSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+/**
+ * Fold N trials of ONE case (same model + instance) into a single result:
+ *   - binary verdicts (behavioral / resolved) are WORST-case — true only if
+ *     every non-errored trial passed (a reliability measure), with the pass
+ *     count kept alongside for variance.
+ *   - cost / tokens / latency are the MEAN across non-errored trials.
+ * A single trial is returned unchanged. If every trial errored, the aggregate
+ * carries that error.
+ */
+export function aggregateTrials(trials) {
+    if (trials.length === 1)
+        return trials[0];
+    const base = trials[0];
+    const ok = trials.filter((t) => !t.error);
+    if (!ok.length) {
+        return { ...base, trials: 0, trialErrors: trials.length, error: base.error ?? "all trials errored" };
+    }
+    const mean = (xs) => (xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : 0);
+    const out = {
+        ...base,
+        error: undefined,
+        inputTokens: Math.round(mean(ok.map((t) => t.inputTokens))),
+        outputTokens: Math.round(mean(ok.map((t) => t.outputTokens))),
+        costUsd: mean(ok.map((t) => t.costUsd)),
+        durationMs: Math.round(mean(ok.map((t) => t.durationMs))),
+        githubMutations: Math.round(mean(ok.map((t) => t.githubMutations ?? 0))),
+        trials: ok.length,
+        trialErrors: trials.length - ok.length,
+    };
+    // behavioral: worst-case ok, checks AND'd by name, keep a failing detail.
+    if (ok.some((t) => t.behavioral)) {
+        const passes = ok.filter((t) => t.behavioral?.ok).length;
+        const names = [...new Set(ok.flatMap((t) => t.behavioral?.checks.map((c) => c.name) ?? []))];
+        const checks = names.map((name) => {
+            const perTrial = ok.map((t) => t.behavioral?.checks.find((c) => c.name === name));
+            const failing = perTrial.find((c) => c && !c.ok);
+            return { name, ok: perTrial.every((c) => c?.ok), detail: failing?.detail };
+        });
+        out.behavioral = { ok: passes === ok.length, checks };
+        out.behavioralPass = passes;
+    }
+    // resolved: worst-case; keep a failing trial's test breakdown + a patch.
+    if (ok.some((t) => t.resolved !== undefined)) {
+        const passes = ok.filter((t) => t.resolved).length;
+        const rep = ok.find((t) => !t.resolved) ?? ok[0];
+        out.resolved = passes === ok.length;
+        out.resolvedPass = passes;
+        out.failToPass = rep.failToPass;
+        out.passToPass = rep.passToPass;
+        out.model_patch = (ok.find((t) => t.resolved) ?? ok[0]).model_patch;
+    }
+    return out;
+}
+/** Per-model aggregation over a set of results (one tier or all of them). */
+export function summarizeModels(results) {
+    const byModel = new Map();
+    for (const r of results) {
+        const list = byModel.get(r.model) ?? [];
+        list.push(r);
+        byModel.set(r.model, list);
+    }
+    const models = [];
+    for (const [model, list] of byModel) {
+        const codeFix = list.filter((r) => r.resolved !== undefined);
+        const behavioral = list.filter((r) => r.behavioral !== undefined && !r.error);
+        const durations = list.map((r) => r.durationMs).sort((a, b) => a - b);
+        models.push({
+            model,
+            total: list.length,
+            codeFixResolved: codeFix.filter((r) => r.resolved).length,
+            codeFixTotal: codeFix.length,
+            behavioralOk: behavioral.filter((r) => r.behavioral?.ok).length,
+            behavioralTotal: behavioral.length,
+            avgInputTokens: avg(list.map((r) => r.inputTokens)),
+            avgOutputTokens: avg(list.map((r) => r.outputTokens)),
+            totalCostUsd: list.reduce((s, r) => s + r.costUsd, 0),
+            p50DurationMs: durations[Math.floor(durations.length / 2)] ?? 0,
+            errors: list.filter((r) => r.error).length,
+        });
+    }
+    return models;
+}
+export function summarize(results) {
+    return { models: summarizeModels(results), results };
+}
+export function renderTable(card, labels = {}) {
+    const header = ["model", "code-fix", "behavioral", "in tok", "out tok", "cost $", "p50", "err"];
+    const rows = card.models.map((m) => [
+        labels[m.model] ?? m.model,
+        m.codeFixTotal ? `${m.codeFixResolved}/${m.codeFixTotal}` : "—",
+        m.behavioralTotal ? `${m.behavioralOk}/${m.behavioralTotal}` : "—",
+        String(Math.round(m.avgInputTokens)),
+        String(Math.round(m.avgOutputTokens)),
+        m.totalCostUsd.toFixed(4),
+        fmtMs(m.p50DurationMs),
+        String(m.errors),
+    ]);
+    return table([header, ...rows]);
+}
+export function writeArtifacts(dir, card) {
+    mkdirSync(dir, { recursive: true });
+    writeFileSync(join(dir, "scorecard.json"), JSON.stringify(card, null, 2));
+    const preds = card.results
+        .filter((r) => r.model_patch !== undefined)
+        .map((r) => JSON.stringify({ instance_id: r.instance_id, model_name_or_path: r.model, model_patch: r.model_patch ?? "" }))
+        .join("\n");
+    writeFileSync(join(dir, "predictions.jsonl"), preds ? preds + "\n" : "");
+}
+// ── helpers ─────────────────────────────────────────────────────────────────
+function avg(xs) {
+    return xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : 0;
+}
+function fmtMs(ms) {
+    if (ms < 1000)
+        return `${ms}ms`;
+    const s = Math.round(ms / 1000);
+    return s < 60 ? `${s}s` : `${Math.floor(s / 60)}m${String(s % 60).padStart(2, "0")}`;
+}
+function table(rows) {
+    const widths = rows[0].map((_, i) => Math.max(...rows.map((r) => (r[i] ?? "").length)));
+    return rows
+        .map((r) => r.map((c, i) => (c ?? "").padEnd(widths[i])).join("  "))
+        .join("\n");
+}
+//# sourceMappingURL=report.js.map

package/dist/report.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"report.js","sourceRoot":"","sources":["../src/report.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAuBjC;;;;;;;;GAQG;AACH,MAAM,UAAU,eAAe,CAAC,MAAwB;IACtD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;IAC1C,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;IACvB,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAC1C,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC;QACf,OAAO,EAAE,GAAG,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,oBAAoB,EAAE,CAAC;IACvG,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,EAAY,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3F,MAAM,GAAG,GAAmB;QAC1B,GAAG,IAAI;QACP,KAAK,EAAE,SAAS;QAChB,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC;QAC3D,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC;QAC7D,OAAO,EAAE,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QACvC,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;QACzD,eAAe,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,IAAI,CAAC,CAAC,CAAC,CAAC;QACxE,MAAM,EAAE,EAAE,CAAC,MAAM;QACjB,WAAW,EAAE,MAAM,CAAC,MAAM,GAAG,EAAE,CAAC,MAAM;KACvC,CAAC;IAEF,0EAA0E;IAC1E,IAAI,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,EAAE,CAAC;QACjC,MAAM,MAAM,GAAG,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC;QACzD,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7F,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YAChC,MAAM,QAAQ,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC;YAClF,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACjD,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;QAC7E,CAAC,CAAC,CAAC;QACH,GAAG,CAAC,UAAU,GAAG,EAAE,EAAE,EAAE,MAAM,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC;QACtD,GAAG,CAAC,cAAc,GAAG,MAAM,CAAC;IAC9B,CAAC;IAED,yEAAyE;IACzE,IAAI,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,EAAE,CAAC;QAC7C,MAAM,MAAM,GAAG,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;QACnD,MAAM,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QACjD,GAAG,CAAC,QAAQ,GAAG,MAAM,KAAK,EAAE,CAAC,MAAM,CAAC;QACpC,GAAG,CAAC,YAAY,GAAG,MAAM,CAAC;QAC1B,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC,UAAU,CAAC;QAChC,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC,UAAU,CAAC;QAChC,GAAG,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC;IACtE,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AAED,6EAA6E;AAC7E,MAAM,UAAU,eAAe,CAAC,OAAyB;IACvD,MAAM,OAAO,GAAG,IAAI,GAAG,EAA4B,CAAC;IACpD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QACxC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACb,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAC7B,CAAC;IAED,MAAM,MAAM,GAAmB,EAAE,CAAC;IAClC,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,OAAO,EAAE,CAAC;QACpC,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC;QAC7D,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,KAAK,SAAS,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QAC9E,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACtE,MAAM,CAAC,IAAI,CAAC;YACV,KAAK;YACL,KAAK,EAAE,IAAI,CAAC,MAAM;YAClB,eAAe,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM;YACzD,YAAY,EAAE,OAAO,CAAC,MAAM;YAC5B,YAAY,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,MAAM;YAC/D,eAAe,EAAE,UAAU,CAAC,MAAM;YAClC,cAAc,EAAE,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;YACnD,eAAe,EAAE,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;YACrD,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;YACrD,aAAa,EAAE,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;YAC/D,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM;SAC3C,CAAC,CAAC;IACL,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,OAAyB;IACjD,OAAO,EAAE,MAAM,EAAE,eAAe,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,CAAC;AACvD,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,IAAe,EAAE,SAAiC,EAAE;IAC9E,MAAM,MAAM,GAAG,CAAC,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IAChG,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QAClC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK;QAC1B,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,eAAe,IAAI,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,GAAG;QAC/D,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,GAAG;QAClE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC;QACpC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC;QACrC,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC;QACzB,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC;QACtB,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;KACjB,CAAC,CAAC;IACH,OAAO,KAAK,CAAC,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC;AAClC,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,GAAW,EAAE,IAAe;IACzD,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACpC,aAAa,CAAC,IAAI,CAAC,GAAG,EAAE,gBAAgB,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAC1E,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO;SACvB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,SAAS,CAAC;SAC1C,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,WAAW,EAAE,kBAAkB,EAAE,CAAC,CAAC,KAAK,EAAE,WAAW,EAAE,CAAC,CAAC,WAAW,IAAI,EAAE,EAAE,CAAC,CAAC;SACzH,IAAI,CAAC,IAAI,CAAC,CAAC;IACd,aAAa,CAAC,IAAI,CAAC,GAAG,EAAE,mBAAmB,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;AAC3E,CAAC;AAED,+EAA+E;AAE/E,SAAS,GAAG,CAAC,EAAY;IACvB,OAAO,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;AACnE,CAAC;AACD,SAAS,KAAK,CAAC,EAAU;IACvB,IAAI,EAAE,GAAG,IAAI;QAAE,OAAO,GAAG,EAAE,IAAI,CAAC;IAChC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC;IAChC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,CAAC,IAAI,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;AACvF,CAAC;AACD,SAAS,KAAK,CAAC,IAAgB;IAC7B,MAAM,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IACxF,OAAO,IAAI;SACR,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;SACnE,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC"}

package/dist/run-instance.js

@@ -0,0 +1,203 @@
+/**
+ * Run ONE eval instance against the REAL production workflow with GitHub mocked.
+ *
+ * Flow:
+ *   1. Start the fake GitHub (seeded from the instance's issue fixtures).
+ *   2. (code-fix) Deterministically seed the workspace: fixture repo @ base
+ *      commit + a local bare `origin` so `git push` works offline.
+ *   3. Load the REAL workflow YAML (build / issue-triage / …) via the loader.
+ *   4. runWorkflow with `sandbox: "none"`, `githubApiBaseUrl → fake GitHub`,
+ *      and an EMPTY approvalConfig so gates never pause. No real GitHub creds.
+ *   5. Grade deterministically (execution + behavioral) and collect metrics.
+ *
+ * The only deviations from production are the ones we can't do unattended:
+ * approvals are skipped and GitHub is mocked. Prompts, skills, phases, and the
+ * agent loop are exactly what ships.
+ */
+import { mkdtempSync, mkdirSync, rmSync, existsSync } from "node:fs";
+import { execFileSync } from "node:child_process";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import { getWorkflow, runWorkflow } from "lastlight/evals";
+import { startFakeGitHub } from "./fake-github.js";
+import { seedWorkspace } from "./seed.js";
+import { collectMetrics, drainSessions } from "./metrics.js";
+import { gradeBehavioral, gradeExecution, gradeTriage } from "./grade.js";
+const EVAL_ENV_KEYS = ["GITHUB_APP_ID", "GITHUB_APP_INSTALLATION_ID", "GITHUB_TOKEN", "GH_TOKEN", "GITHUB_API_URL"];
+/**
+ * Install the eval's static-token GitHub env and return a restore fn:
+ *   - unset the App creds so no real installation token is ever minted, and
+ *   - set a dummy `GITHUB_TOKEN`/`GH_TOKEN` so the GitHub extension loads in
+ *     static-token mode (its Octokit is pointed at the fake server via
+ *     `githubApiBaseUrl`).
+ * Every eval run wants the SAME values, so the parallel batch installs this
+ * once up front (stable baseline) and each `runInstance` skips its own env work
+ * via `manageEnv: false`.
+ */
+export function applyEvalEnv() {
+    const saved = snapshotEnv(EVAL_ENV_KEYS);
+    delete process.env.GITHUB_APP_ID;
+    delete process.env.GITHUB_APP_INSTALLATION_ID;
+    process.env.GITHUB_TOKEN = "eval-fake-token";
+    process.env.GH_TOKEN = "eval-fake-token";
+    return () => restoreEnv(saved);
+}
+function splitRepo(repo) {
+    const [owner, name] = repo.split("/");
+    return { owner: owner ?? "acme", name: name ?? "widget" };
+}
+export async function runInstance(inst, opts) {
+    const start = Date.now();
+    const { owner, name } = splitRepo(inst.repo);
+    const workflowName = inst.workflow ?? opts.defaultWorkflow ?? "issue-triage";
+    const isCodeFix = workflowName !== "issue-triage";
+    const stateDir = opts.stateDir ?? mkdtempSync(join(tmpdir(), "ll-eval-"));
+    const sessionsDir = join(stateDir, "agent-sessions");
+    // The shim appends per-phase jsonl under <sessionsDir>/projects/<slug>/ and
+    // does not create that parent recursively — pre-create it so token/cost
+    // metrics are captured (collectMetrics reads those jsonl files).
+    mkdirSync(join(sessionsDir, "projects"), { recursive: true });
+    const taskId = `${name}-${inst.issue?.number ?? 0}-${workflowName}-${slug(inst.instance_id)}`;
+    const issueNumber = inst.issue?.number ?? 1;
+    const branch = isCodeFix ? `lastlight/${slug(inst.instance_id)}` : "main";
+    // 1. Fake GitHub, seeded with the issue.
+    const fake = await startFakeGitHub({
+        owner,
+        repo: name,
+        issues: inst.issue ? [inst.issue] : [],
+        existingLabels: inst.issue?.labels ?? [],
+    });
+    // Static-token mode: no App creds (so no real mint), a dummy token so the
+    // GitHub extension loads, and point its Octokit at the fake. In a parallel
+    // batch the caller installs this once (manageEnv: false) so concurrent runs
+    // share one stable baseline instead of racing per-run env splices.
+    const restoreEvalEnv = opts.manageEnv === false ? () => { } : applyEvalEnv();
+    const result = {
+        instance_id: inst.instance_id,
+        model: opts.model,
+        workflowSucceeded: false,
+        inputTokens: 0,
+        outputTokens: 0,
+        costUsd: 0,
+        durationMs: 0,
+        phases: [],
+    };
+    try {
+        // 2. Seed the workspace for code-fix (triage needs no repo).
+        if (isCodeFix && opts.datasetDir) {
+            const fixtureDir = join(opts.datasetDir, "repos", inst.instance_id);
+            if (existsSync(fixtureDir)) {
+                seedWorkspace({ stateDir, taskId, fixtureDir, branch });
+            }
+        }
+        // 3. Real workflow definition + run context.
+        const def = getWorkflow(workflowName);
+        const ctx = {
+            owner,
+            repo: name,
+            issueNumber,
+            issueTitle: inst.issue?.title ?? inst.instance_id,
+            issueBody: inst.issue?.body ?? inst.problem_statement,
+            issueLabels: inst.issue?.labels ?? [],
+            commentBody: "",
+            sender: "eval",
+            branch,
+            taskId,
+            issueDir: `.lastlight/issue-${issueNumber}`,
+            bootstrapLabel: "lastlight:bootstrap",
+            // No prePopulateBranch → the runner never clones from GitHub; the agent
+            // works in the dir we seeded above (or an empty dir for triage).
+        };
+        const config = {
+            sandbox: "none",
+            stateDir,
+            sessionsDir,
+            model: opts.model,
+            githubApiBaseUrl: fake.url,
+            // Eval workflows shouldn't reach the network beyond the model + fake GH.
+            webSearch: false,
+        };
+        // 4. Run. Empty approvalConfig → every approval gate is disabled.
+        const wf = await runWorkflow(def, ctx, config, {}, undefined, undefined, {});
+        result.workflowSucceeded = wf.success;
+        result.phases = wf.phases.map((p) => ({ phase: p.phase, success: p.success }));
+        // A workflow-level failure (a phase erroring — provider auth/credit/rate
+        // errors, timeouts) is a RUN error, not a model "miss". Surface it so the
+        // scorecard counts it under errors instead of behavioral✗.
+        if (!wf.success) {
+            const failed = wf.phases.find((p) => !p.success && p.error);
+            result.error = failed?.error
+                ? `${failed.phase}: ${failed.error}`.slice(0, 300)
+                : "workflow failed";
+        }
+        // 5a. Behavioral grade (GitHub mutations).
+        const behavioralExpect = gradeBehavioral(inst.expect_github, fake, { issueNumber, branch });
+        const triage = gradeTriage(inst.triage_gold, fake, issueNumber);
+        result.behavioral = {
+            ok: behavioralExpect.ok && triage.ok,
+            checks: [...behavioralExpect.checks, ...triage.checks],
+        };
+        // 5b. Execution grade (code-fix only).
+        if (isCodeFix && (inst.FAIL_TO_PASS?.length || inst.test_patch)) {
+            const workDir = join(stateDir, "sandboxes", taskId);
+            const heldOutDir = opts.datasetDir ? join(opts.datasetDir, "tests", inst.instance_id) : undefined;
+            const exec = gradeExecution({
+                workDir,
+                heldOutDir,
+                testPatch: inst.test_patch,
+                failToPass: inst.FAIL_TO_PASS ?? [],
+                passToPass: inst.PASS_TO_PASS ?? [],
+            });
+            result.resolved = exec.resolved;
+            result.failToPass = exec.failToPass;
+            result.passToPass = exec.passToPass;
+            result.model_patch = gitDiff(workDir);
+        }
+        // 5c. Metrics. Drain the fire-and-forget session flush first so the final
+        // `result` envelope (cost/tokens) has landed before we read + clean up.
+        await drainSessions(sessionsDir);
+        const m = collectMetrics(sessionsDir);
+        result.inputTokens = m.inputTokens;
+        result.outputTokens = m.outputTokens;
+        result.costUsd = m.costUsd;
+        result.githubMutations = fake.calls.length;
+    }
+    catch (err) {
+        result.error = err instanceof Error ? err.message : String(err);
+    }
+    finally {
+        result.durationMs = Date.now() - start;
+        await fake.close();
+        restoreEvalEnv();
+        if (!opts.keepWorkspace && !opts.stateDir) {
+            rmSync(stateDir, { recursive: true, force: true });
+        }
+    }
+    return result;
+}
+function slug(s) {
+    return s.replace(/[^a-zA-Z0-9_-]+/g, "-").slice(0, 40);
+}
+function snapshotEnv(keys) {
+    const snap = {};
+    for (const k of keys)
+        snap[k] = process.env[k];
+    return snap;
+}
+function restoreEnv(snap) {
+    for (const [k, v] of Object.entries(snap)) {
+        if (v === undefined)
+            delete process.env[k];
+        else
+            process.env[k] = v;
+    }
+}
+function gitDiff(workDir) {
+    try {
+        return execFileSync("git", ["diff", "HEAD"], { cwd: workDir, stdio: ["ignore", "pipe", "ignore"] }).toString();
+    }
+    catch {
+        return undefined;
+    }
+}
+//# sourceMappingURL=run-instance.js.map

package/dist/run-instance.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-instance.js","sourceRoot":"","sources":["../src/run-instance.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrE,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AAEjC,OAAO,EAAE,WAAW,EAAE,WAAW,EAA6C,MAAM,iBAAiB,CAAC;AAGtG,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7D,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAsB1E,MAAM,aAAa,GAAG,CAAC,eAAe,EAAE,4BAA4B,EAAE,cAAc,EAAE,UAAU,EAAE,gBAAgB,CAAC,CAAC;AAEpH;;;;;;;;;GASG;AACH,MAAM,UAAU,YAAY;IAC1B,MAAM,KAAK,GAAG,WAAW,CAAC,aAAa,CAAC,CAAC;IACzC,OAAO,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;IACjC,OAAO,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC;IAC9C,OAAO,CAAC,GAAG,CAAC,YAAY,GAAG,iBAAiB,CAAC;IAC7C,OAAO,CAAC,GAAG,CAAC,QAAQ,GAAG,iBAAiB,CAAC;IACzC,OAAO,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;AACjC,CAAC;AAED,SAAS,SAAS,CAAC,IAAY;IAC7B,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACtC,OAAO,EAAE,KAAK,EAAE,KAAK,IAAI,MAAM,EAAE,IAAI,EAAE,IAAI,IAAI,QAAQ,EAAE,CAAC;AAC5D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAAsB,EAAE,IAAwB;IAChF,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7C,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,eAAe,IAAI,cAAc,CAAC;IAC7E,MAAM,SAAS,GAAG,YAAY,KAAK,cAAc,CAAC;IAElD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,UAAU,CAAC,CAAC,CAAC;IAC1E,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,EAAE,gBAAgB,CAAC,CAAC;IACrD,4EAA4E;IAC5E,wEAAwE;IACxE,iEAAiE;IACjE,SAAS,CAAC,IAAI,CAAC,WAAW,EAAE,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9D,MAAM,MAAM,GAAG,GAAG,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,MAAM,IAAI,CAAC,IAAI,YAAY,IAAI,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;IAC9F,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,EAAE,MAAM,IAAI,CAAC,CAAC;IAC5C,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,aAAa,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;IAE1E,yCAAyC;IACzC,MAAM,IAAI,GAAG,MAAM,eAAe,CAAC;QACjC,KAAK;QACL,IAAI,EAAE,IAAI;QACV,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE;QACtC,cAAc,EAAE,IAAI,CAAC,KAAK,EAAE,MAAM,IAAI,EAAE;KACzC,CAAC,CAAC;IAEH,0EAA0E;IAC1E,2EAA2E;IAC3E,4EAA4E;IAC5E,mEAAmE;IACnE,MAAM,cAAc,GAAG,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC,CAAC,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC,CAAC,YAAY,EAAE,CAAC;IAE5E,MAAM,MAAM,GAAmB;QAC7B,WAAW,EAAE,IAAI,CAAC,WAAW;QAC7B,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,iBAAiB,EAAE,KAAK;QACxB,WAAW,EAAE,CAAC;QACd,YAAY,EAAE,CAAC;QACf,OAAO,EAAE,CAAC;QACV,UAAU,EAAE,CAAC;QACb,MAAM,EAAE,EAAE;KACX,CAAC;IAEF,IAAI,CAAC;QACH,6DAA6D;QAC7D,IAAI,SAAS,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACjC,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,OAAO,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;YACpE,IAAI,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;gBAC3B,aAAa,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC,CAAC;YAC1D,CAAC;QACH,CAAC;QAED,6CAA6C;QAC7C,MAAM,GAAG,GAAG,WAAW,CAAC,YAAY,CAAC,CAAC;QACtC,MAAM,GAAG,GAAoB;YAC3B,KAAK;YACL,IAAI,EAAE,IAAI;YACV,WAAW;YACX,UAAU,EAAE,IAAI,CAAC,KAAK,EAAE,KAAK,IAAI,IAAI,CAAC,WAAW;YACjD,SAAS,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,IAAI,IAAI,CAAC,iBAAiB;YACrD,WAAW,EAAE,IAAI,CAAC,KAAK,EAAE,MAAM,IAAI,EAAE;YACrC,WAAW,EAAE,EAAE;YACf,MAAM,EAAE,MAAM;YACd,MAAM;YACN,MAAM;YACN,QAAQ,EAAE,oBAAoB,WAAW,EAAE;YAC3C,cAAc,EAAE,qBAAqB;YACrC,wEAAwE;YACxE,iEAAiE;SAClE,CAAC;QAEF,MAAM,MAAM,GAAmB;YAC7B,OAAO,EAAE,MAAM;YACf,QAAQ;YACR,WAAW;YACX,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,gBAAgB,EAAE,IAAI,CAAC,GAAG;YAC1B,yEAAyE;YACzE,SAAS,EAAE,KAAK;SACjB,CAAC;QAEF,kEAAkE;QAClE,MAAM,EAAE,GAAG,MAAM,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;QAE7E,MAAM,CAAC,iBAAiB,GAAG,EAAE,CAAC,OAAO,CAAC;QACtC,MAAM,CAAC,MAAM,GAAG,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QAE/E,yEAAyE;QACzE,0EAA0E;QAC1E,2DAA2D;QAC3D,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC;YAChB,MAAM,MAAM,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC;YAC5D,MAAM,CAAC,KAAK,GAAG,MAAM,EAAE,KAAK;gBAC1B,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,KAAK,MAAM,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;gBAClD,CAAC,CAAC,iBAAiB,CAAC;QACxB,CAAC;QAED,2CAA2C;QAC3C,MAAM,gBAAgB,GAAG,eAAe,CAAC,IAAI,CAAC,aAAa,EAAE,IAAI,EAAE,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,CAAC;QAC5F,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,EAAE,WAAW,CAAC,CAAC;QAChE,MAAM,CAAC,UAAU,GAAG;YAClB,EAAE,EAAE,gBAAgB,CAAC,EAAE,IAAI,MAAM,CAAC,EAAE;YACpC,MAAM,EAAE,CAAC,GAAG,gBAAgB,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC;SACvD,CAAC;QAEF,uCAAuC;QACvC,IAAI,SAAS,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,IAAI,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;YAChE,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;YACpD,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,OAAO,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YAClG,MAAM,IAAI,GAAG,cAAc,CAAC;gBAC1B,OAAO;gBACP,UAAU;gBACV,SAAS,EAAE,IAAI,CAAC,UAAU;gBAC1B,UAAU,EAAE,IAAI,CAAC,YAAY,IAAI,EAAE;gBACnC,UAAU,EAAE,IAAI,CAAC,YAAY,IAAI,EAAE;aACpC,CAAC,CAAC;YACH,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;YAChC,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;YACpC,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;YACpC,MAAM,CAAC,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;QACxC,CAAC;QAED,0EAA0E;QAC1E,wEAAwE;QACxE,MAAM,aAAa,CAAC,WAAW,CAAC,CAAC;QACjC,MAAM,CAAC,GAAG,cAAc,CAAC,WAAW,CAAC,CAAC;QACtC,MAAM,CAAC,WAAW,GAAG,CAAC,CAAC,WAAW,CAAC;QACnC,MAAM,CAAC,YAAY,GAAG,CAAC,CAAC,YAAY,CAAC;QACrC,MAAM,CAAC,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC;QAC3B,MAAM,CAAC,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;IAC7C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,CAAC,KAAK,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAClE,CAAC;YAAS,CAAC;QACT,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QACvC,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACnB,cAAc,EAAE,CAAC;QACjB,IAAI,CAAC,IAAI,CAAC,aAAa,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC1C,MAAM,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACrD,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,IAAI,CAAC,CAAS;IACrB,OAAO,CAAC,CAAC,OAAO,CAAC,kBAAkB,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AACzD,CAAC;AAED,SAAS,WAAW,CAAC,IAAc;IACjC,MAAM,IAAI,GAAuC,EAAE,CAAC;IACpD,KAAK,MAAM,CAAC,IAAI,IAAI;QAAE,IAAI,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAC/C,OAAO,IAAI,CAAC;AACd,CAAC;AACD,SAAS,UAAU,CAAC,IAAwC;IAC1D,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QAC1C,IAAI,CAAC,KAAK,SAAS;YAAE,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;;YACtC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC;AACH,CAAC;AAED,SAAS,OAAO,CAAC,OAAe;IAC9B,IAAI,CAAC;QACH,OAAO,YAAY,CAAC,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC;IACjH,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC"}