@vercel/agent-eval 2.4.0 → 2.5.0

package/dist/lib/reporters.js

@@ -0,0 +1,176 @@
+/**
+ * Reporters: first-class hooks that send results somewhere besides the local
+ * results/ tree.
+ *
+ * The runner calls {@link notifyRunComplete} once per finished run and
+ * {@link notifyExperimentComplete} once per experiment. Both isolate reporter
+ * failures: an error is caught, redacted, handed to `onFailure`, and the next
+ * reporter still runs. Nothing a reporter does can change an eval result.
+ */
+import { appendFile, mkdir } from 'node:fs/promises';
+import { dirname } from 'node:path';
+import { redactSecrets, redactValue } from './agents/redact.js';
+/** One line describing a failed reporter call, for the CLI and logs. */
+export function formatReporterError(event) {
+    const where = event.evalName
+        ? ` for ${event.evalName}${event.runNumber !== undefined ? ` run ${event.runNumber}` : ''}`
+        : '';
+    return `\u26a0 Reporter "${event.reporter}" failed in ${event.hook}${where}: ${event.error}`;
+}
+function errorMessage(error) {
+    return error instanceof Error ? error.message : String(error);
+}
+/**
+ * Deliver a run event to every reporter with an `onRunComplete` hook, in order.
+ * Returns the failures (also passed to `onFailure`).
+ */
+export async function notifyRunComplete(reporters, event, options) {
+    const listening = reporters.filter((reporter) => reporter.onRunComplete);
+    if (listening.length === 0)
+        return [];
+    const payload = redactValue(event, options.secrets);
+    const failures = [];
+    for (const reporter of listening) {
+        try {
+            await reporter.onRunComplete(payload);
+        }
+        catch (error) {
+            const failure = {
+                reporter: reporter.name,
+                hook: 'onRunComplete',
+                evalName: event.evalName,
+                runIndex: event.runIndex,
+                error: redactSecrets(errorMessage(error), options.secrets),
+            };
+            failures.push(failure);
+            options.onFailure?.(failure);
+        }
+    }
+    return failures;
+}
+/**
+ * Deliver an experiment event to every reporter with an
+ * `onExperimentComplete` hook, in order. Returns the failures (also passed to
+ * `onFailure`).
+ */
+export async function notifyExperimentComplete(reporters, event, options) {
+    const listening = reporters.filter((reporter) => reporter.onExperimentComplete);
+    if (listening.length === 0)
+        return [];
+    const payload = redactValue(event, options.secrets);
+    const failures = [];
+    for (const reporter of listening) {
+        try {
+            await reporter.onExperimentComplete(payload);
+        }
+        catch (error) {
+            const failure = {
+                reporter: reporter.name,
+                hook: 'onExperimentComplete',
+                error: redactSecrets(errorMessage(error), options.secrets),
+            };
+            failures.push(failure);
+            options.onFailure?.(failure);
+        }
+    }
+    return failures;
+}
+/**
+ * The JSON form of an experiment event used by the built-in reporters.
+ *
+ * `results` in a {@link ReporterExperimentEvent} carries everything the run
+ * produced: transcripts, script output, and generated files as raw bytes. A
+ * network payload doesn't need those (they're on disk under `outputDir`), so
+ * each run is reduced to its `result`. Config functions and the reporters
+ * themselves are left out too, since they aren't data.
+ */
+export function experimentEventToJson(event) {
+    const { results } = event;
+    const config = Object.fromEntries(Object.entries(results.config).filter(([key, value]) => key !== 'reporters' && typeof value !== 'function'));
+    const evals = results.evals.map(({ runs, ...summary }) => ({
+        ...summary,
+        runs: runs.map((run) => run.result),
+    }));
+    return {
+        ...event,
+        results: { startedAt: results.startedAt, completedAt: results.completedAt, config, evals },
+    };
+}
+/**
+ * Append one JSON line per finished run to a file. Each line is a
+ * {@link ReporterRunEvent}. Writes are serialized, so concurrent runs never
+ * interleave lines.
+ *
+ * @example
+ * ```ts
+ * reporters: [jsonlReporter({ path: 'results/runs.jsonl' })]
+ * ```
+ */
+export function jsonlReporter(options) {
+    let pending = Promise.resolve();
+    return {
+        name: 'jsonl',
+        onRunComplete(event) {
+            const line = `${JSON.stringify(event)}\n`;
+            const write = pending.then(async () => {
+                await mkdir(dirname(options.path), { recursive: true });
+                await appendFile(options.path, line, 'utf-8');
+            });
+            // A failed write fails only its own event; later events still write.
+            pending = write.catch(() => { });
+            return write;
+        },
+    };
+}
+/**
+ * POST each event as JSON. Run events are sent as
+ * `{ "type": "run", ...ReporterRunEvent }`; experiment events as
+ * `{ "type": "experiment", ...experimentEventToJson(event) }`. Network errors,
+ * 429, and 5xx responses are retried with exponential backoff; other non-2xx
+ * responses fail at once.
+ *
+ * @example
+ * ```ts
+ * reporters: [
+ *   httpReporter({
+ *     url: 'https://example.com/ingest',
+ *     headers: { authorization: `Bearer ${process.env.INGEST_TOKEN}` },
+ *   }),
+ * ]
+ * ```
+ */
+export function httpReporter(options) {
+    const retries = options.retries ?? 2;
+    const retryDelayMs = options.retryDelayMs ?? 500;
+    const post = async (body) => {
+        const payload = JSON.stringify(body);
+        for (let attempt = 0;; attempt++) {
+            let failure;
+            let retryable;
+            try {
+                const response = await fetch(options.url, {
+                    method: 'POST',
+                    headers: { 'content-type': 'application/json', ...options.headers },
+                    body: payload,
+                });
+                if (response.ok)
+                    return;
+                failure = `HTTP ${response.status} from ${options.url}`;
+                retryable = response.status === 429 || response.status >= 500;
+            }
+            catch (error) {
+                failure = `POST ${options.url} failed: ${errorMessage(error)}`;
+                retryable = true;
+            }
+            if (!retryable || attempt >= retries)
+                throw new Error(failure);
+            await new Promise((resolve) => setTimeout(resolve, retryDelayMs * 2 ** attempt));
+        }
+    };
+    return {
+        name: 'http',
+        onRunComplete: (event) => post({ type: 'run', ...event }),
+        onExperimentComplete: (event) => post({ type: 'experiment', ...experimentEventToJson(event) }),
+    };
+}
+//# sourceMappingURL=reporters.js.map

package/dist/lib/reporters.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"reporters.js","sourceRoot":"","sources":["../../src/lib/reporters.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAOpC,OAAO,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAwBhE,wEAAwE;AACxE,MAAM,UAAU,mBAAmB,CAAC,KAAyD;IAC3F,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ;QAC1B,CAAC,CAAC,QAAQ,KAAK,CAAC,QAAQ,GAAG,KAAK,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,QAAQ,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE;QAC3F,CAAC,CAAC,EAAE,CAAC;IACP,OAAO,oBAAoB,KAAK,CAAC,QAAQ,eAAe,KAAK,CAAC,IAAI,GAAG,KAAK,KAAK,KAAK,CAAC,KAAK,EAAE,CAAC;AAC/F,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAChE,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,SAA8B,EAC9B,KAAuB,EACvB,OAAgC;IAEhC,MAAM,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;IACzE,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEtC,MAAM,OAAO,GAAG,WAAW,CAAC,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;IACpD,MAAM,QAAQ,GAAsB,EAAE,CAAC;IACvC,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,IAAI,CAAC;YACH,MAAM,QAAQ,CAAC,aAAc,CAAC,OAAO,CAAC,CAAC;QACzC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,OAAO,GAAoB;gBAC/B,QAAQ,EAAE,QAAQ,CAAC,IAAI;gBACvB,IAAI,EAAE,eAAe;gBACrB,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,KAAK,EAAE,aAAa,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC;aAC3D,CAAC;YACF,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACvB,OAAO,CAAC,SAAS,EAAE,CAAC,OAAO,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,SAA8B,EAC9B,KAA8B,EAC9B,OAAgC;IAEhC,MAAM,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAC;IAChF,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEtC,MAAM,OAAO,GAAG,WAAW,CAAC,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;IACpD,MAAM,QAAQ,GAAsB,EAAE,CAAC;IACvC,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,IAAI,CAAC;YACH,MAAM,QAAQ,CAAC,oBAAqB,CAAC,OAAO,CAAC,CAAC;QAChD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,OAAO,GAAoB;gBAC/B,QAAQ,EAAE,QAAQ,CAAC,IAAI;gBACvB,IAAI,EAAE,sBAAsB;gBAC5B,KAAK,EAAE,aAAa,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC;aAC3D,CAAC;YACF,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACvB,OAAO,CAAC,SAAS,EAAE,CAAC,OAAO,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,qBAAqB,CAAC,KAA8B;IAClE,MAAM,EAAE,OAAO,EAAE,GAAG,KAAK,CAAC;IAC1B,MAAM,MAAM,GAAG,MAAM,CAAC,WAAW,CAC/B,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,CACnC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,GAAG,KAAK,WAAW,IAAI,OAAO,KAAK,KAAK,UAAU,CACrE,CACF,CAAC;IACF,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC;QACzD,GAAG,OAAO;QACV,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC;KACpC,CAAC,CAAC,CAAC;IACJ,OAAO;QACL,GAAG,KAAK;QACR,OAAO,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,WAAW,EAAE,OAAO,CAAC,WAAW,EAAE,MAAM,EAAE,KAAK,EAAE;KAC3F,CAAC;AACJ,CAAC;AAQD;;;;;;;;;GASG;AACH,MAAM,UAAU,aAAa,CAAC,OAA6B;IACzD,IAAI,OAAO,GAAkB,OAAO,CAAC,OAAO,EAAE,CAAC;IAC/C,OAAO;QACL,IAAI,EAAE,OAAO;QACb,aAAa,CAAC,KAAK;YACjB,MAAM,IAAI,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC;YAC1C,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;gBACpC,MAAM,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;gBACxD,MAAM,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;YAChD,CAAC,CAAC,CAAC;YACH,qEAAqE;YACrE,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;YAChC,OAAO,KAAK,CAAC;QACf,CAAC;KACF,CAAC;AACJ,CAAC;AAcD;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,UAAU,YAAY,CAAC,OAA4B;IACvD,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,CAAC,CAAC;IACrC,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,GAAG,CAAC;IAEjD,MAAM,IAAI,GAAG,KAAK,EAAE,IAAa,EAAiB,EAAE;QAClD,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QACrC,KAAK,IAAI,OAAO,GAAG,CAAC,GAAI,OAAO,EAAE,EAAE,CAAC;YAClC,IAAI,OAAe,CAAC;YACpB,IAAI,SAAkB,CAAC;YACvB,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,GAAG,EAAE;oBACxC,MAAM,EAAE,MAAM;oBACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE;oBACnE,IAAI,EAAE,OAAO;iBACd,CAAC,CAAC;gBACH,IAAI,QAAQ,CAAC,EAAE;oBAAE,OAAO;gBACxB,OAAO,GAAG,QAAQ,QAAQ,CAAC,MAAM,SAAS,OAAO,CAAC,GAAG,EAAE,CAAC;gBACxD,SAAS,GAAG,QAAQ,CAAC,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,CAAC;YAChE,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,GAAG,QAAQ,OAAO,CAAC,GAAG,YAAY,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC/D,SAAS,GAAG,IAAI,CAAC;YACnB,CAAC;YACD,IAAI,CAAC,SAAS,IAAI,OAAO,IAAI,OAAO;gBAAE,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC;YAC/D,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,YAAY,GAAG,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC;QACnF,CAAC;IACH,CAAC,CAAC;IAEF,OAAO;QACL,IAAI,EAAE,MAAM;QACZ,aAAa,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,KAAK,EAAE,CAAC;QACzD,oBAAoB,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,GAAG,qBAAqB,CAAC,KAAK,CAAC,EAAE,CAAC;KAC/F,CAAC;AACJ,CAAC"}

package/dist/lib/results.d.ts

@@ -1,12 +1,25 @@
 /**
  * Results storage and reporting for eval experiments.
  */
-import type { EvalRunResult, EvalRunData, EvalSummary, ExperimentResults, RunnableExperimentConfig } from './types.js';
+import type { EvalRunResult, EvalRunData, EvalSummary, ExperimentResults, RunnableExperimentConfig, UsageSummary } from './types.js';
 import type { AgentRunResult } from './agents/types.js';
 /**
  * Convert AgentRunResult to EvalRunData (result + transcript).
+ *
+ * `options.o11yAgentName` names the transcript parser used to read token usage
+ * when the agent did not report `usage` itself. Without it, usage is taken only
+ * from `agentResult.usage`.
+ */
+export declare function agentResultToEvalRunData(agentResult: AgentRunResult, options?: {
+    o11yAgentName?: string;
+}): EvalRunData;
+/**
+ * Aggregate token usage over the runs of one eval. Returns undefined when no
+ * run reported usage. Token fields cover the runs that reported usage and
+ * appear only when all of those runs reported the field; `costUsd` appears only
+ * when every run reported a cost.
  */
-export declare function agentResultToEvalRunData(agentResult: AgentRunResult): EvalRunData;
+export declare function summarizeUsage(runs: readonly EvalRunResult[]): UsageSummary | undefined;
 /**
  * Create a summary from multiple run data.
  */

package/dist/lib/results.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"results.d.ts","sourceRoot":"","sources":["../../src/lib/results.ts"],"names":[],"mappings":"AAAA;;GAEG;AAaH,OAAO,KAAK,EACX,aAAa,EACb,WAAW,EACX,WAAW,EACX,iBAAiB,EACjB,wBAAwB,EACxB,MAAM,YAAY,CAAC;AACpB,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAMxD;;GAEG;AACH,wBAAgB,wBAAwB,CACvC,WAAW,EAAE,cAAc,GACzB,WAAW,CA4Cb;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAChC,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,WAAW,EAAE,GACpB,WAAW,CAab;AAED;;GAEG;AACH,wBAAgB,uBAAuB,CACtC,MAAM,EAAE,wBAAwB,EAChC,KAAK,EAAE,WAAW,EAAE,EACpB,SAAS,EAAE,IAAI,EACf,WAAW,EAAE,IAAI,GACf,iBAAiB,CAOnB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAClC,iCAAiC;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,8CAA8C;IAC9C,cAAc,EAAE,MAAM,CAAC;IACvB,4DAA4D;IAC5D,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtC;;qFAEiF;IACjF,mBAAmB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7C,sEAAsE;IACtE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,+EAA+E;IAC/E,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,6FAA6F;IAC7F,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACtC;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,WAAW,CAC1B,OAAO,EAAE,iBAAiB,EAC1B,OAAO,EAAE,kBAAkB,GACzB,MAAM,CAmMR;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,iBAAiB,GAAG,MAAM,CA8DrE;AAED;;GAEG;AACH,wBAAgB,eAAe,CAC9B,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,aAAa,EACrB,OAAO,CAAC,EAAE;IAAE,cAAc,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,GACnE,MAAM,CAwBR;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CACpC,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EACjB,OAAO,CAAC,EAAE;IAAE,cAAc,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,GACnE,MAAM,CASR;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,gBAAgB;IAChC,mFAAmF;IACnF,yBAAyB,CAAC,EAAE,OAAO,CAAC;IACpC,6BAA6B,CAAC,EAAE,MAAM,CAAC;CACvC;AAED;;;;;;;;;GASG;AACH,wBAAgB,mBAAmB,CAClC,UAAU,EAAE,MAAM,EAClB,cAAc,EAAE,MAAM,EACtB,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EACpC,OAAO,GAAE,gBAAqB,GAC5B,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,CAkF7B"}
+{"version":3,"file":"results.d.ts","sourceRoot":"","sources":["../../src/lib/results.ts"],"names":[],"mappings":"AAAA;;GAEG;AAaH,OAAO,KAAK,EACX,aAAa,EACb,WAAW,EACX,WAAW,EACX,iBAAiB,EACjB,wBAAwB,EACxB,YAAY,EACZ,MAAM,YAAY,CAAC;AACpB,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAOxD;;;;;;GAMG;AACH,wBAAgB,wBAAwB,CACvC,WAAW,EAAE,cAAc,EAC3B,OAAO,GAAE;IAAE,aAAa,CAAC,EAAE,MAAM,CAAA;CAAO,GACtC,WAAW,CAgEb;AAWD;;;;;GAKG;AACH,wBAAgB,cAAc,CAC7B,IAAI,EAAE,SAAS,aAAa,EAAE,GAC5B,YAAY,GAAG,SAAS,CAiB1B;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAChC,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,WAAW,EAAE,GACpB,WAAW,CAeb;AAED;;GAEG;AACH,wBAAgB,uBAAuB,CACtC,MAAM,EAAE,wBAAwB,EAChC,KAAK,EAAE,WAAW,EAAE,EACpB,SAAS,EAAE,IAAI,EACf,WAAW,EAAE,IAAI,GACf,iBAAiB,CAOnB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAClC,iCAAiC;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,8CAA8C;IAC9C,cAAc,EAAE,MAAM,CAAC;IACvB,4DAA4D;IAC5D,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtC;;qFAEiF;IACjF,mBAAmB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7C,sEAAsE;IACtE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,+EAA+E;IAC/E,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,6FAA6F;IAC7F,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACtC;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,WAAW,CAC1B,OAAO,EAAE,iBAAiB,EAC1B,OAAO,EAAE,kBAAkB,GACzB,MAAM,CA8MR;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,iBAAiB,GAAG,MAAM,CA8DrE;AAED;;GAEG;AACH,wBAAgB,eAAe,CAC9B,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,aAAa,EACrB,OAAO,CAAC,EAAE;IAAE,cAAc,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,GACnE,MAAM,CAwBR;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CACpC,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EACjB,OAAO,CAAC,EAAE;IAAE,cAAc,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,GACnE,MAAM,CASR;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,gBAAgB;IAChC,mFAAmF;IACnF,yBAAyB,CAAC,EAAE,OAAO,CAAC;IACpC,6BAA6B,CAAC,EAAE,MAAM,CAAC;CACvC;AAED;;;;;;;;;GASG;AACH,wBAAgB,mBAAmB,CAClC,UAAU,EAAE,MAAM,EAClB,cAAc,EAAE,MAAM,EACtB,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EACpC,OAAO,GAAE,gBAAqB,GAC5B,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,CAkF7B"}

package/dist/lib/results.js

@@ -4,14 +4,19 @@
 import { mkdirSync, writeFileSync, readdirSync, readFileSync, existsSync, statSync, unlinkSync, } from 'fs';
 import { join, dirname } from 'path';
 import chalk from 'chalk';
-import { parseTranscript } from './o11y/index.js';
+import { isWorkspacePath } from './agents/shared.js';
+import { parseTranscript, parseTranscriptUsage } from './o11y/index.js';
 import { isNonModelFailure } from './classifier.js';
 import { copyFixtureFiles } from './fixture.js';
 import { computeReuseCompatibilityFingerprint } from './fingerprint.js';
 /**
  * Convert AgentRunResult to EvalRunData (result + transcript).
+ *
+ * `options.o11yAgentName` names the transcript parser used to read token usage
+ * when the agent did not report `usage` itself. Without it, usage is taken only
+ * from `agentResult.usage`.
  */
-export function agentResultToEvalRunData(agentResult) {
+export function agentResultToEvalRunData(agentResult, options = {}) {
     // Collect output content from scripts and tests
     const outputContent = {};
     // Add EVAL.ts test output
@@ -39,6 +44,25 @@ export function agentResultToEvalRunData(agentResult) {
     if (agentResult.modelRepair) {
         result.modelRepair = agentResult.modelRepair;
     }
+    if (agentResult.provenance) {
+        result.provenance = agentResult.provenance;
+    }
+    const usage = agentResult.usage ??
+        (options.o11yAgentName
+            ? parseTranscriptUsage(agentResult.transcript, options.o11yAgentName)
+            : undefined);
+    if (usage) {
+        result.usage = usage;
+    }
+    if (agentResult.verifier) {
+        result.verifier = agentResult.verifier;
+    }
+    if (agentResult.verifierSandboxId) {
+        result.verifierSandboxId = agentResult.verifierSandboxId;
+    }
+    if (agentResult.tampering) {
+        result.tampering = agentResult.tampering;
+    }
     return {
         result,
         transcript: agentResult.transcript,
@@ -47,6 +71,38 @@ export function agentResultToEvalRunData(agentResult) {
         deletedFiles: agentResult.deletedFiles,
     };
 }
+const USAGE_TOKEN_FIELDS = [
+    'totalTokens',
+    'inputTokens',
+    'outputTokens',
+    'cacheReadTokens',
+    'cacheWriteTokens',
+    'reasoningTokens',
+];
+/**
+ * Aggregate token usage over the runs of one eval. Returns undefined when no
+ * run reported usage. Token fields cover the runs that reported usage and
+ * appear only when all of those runs reported the field; `costUsd` appears only
+ * when every run reported a cost.
+ */
+export function summarizeUsage(runs) {
+    const reported = runs.flatMap((run) => (run.usage ? [run.usage] : []));
+    if (reported.length === 0)
+        return undefined;
+    const summary = { runsWithUsage: reported.length };
+    for (const field of USAGE_TOKEN_FIELDS) {
+        if (reported.every((usage) => usage[field] !== undefined)) {
+            summary[field] = reported.reduce((sum, usage) => sum + usage[field], 0);
+        }
+    }
+    if (summary.totalTokens !== undefined) {
+        summary.meanTotalTokens = summary.totalTokens / reported.length;
+    }
+    if (runs.every((run) => run.usage?.costUsd !== undefined)) {
+        summary.costUsd = runs.reduce((sum, run) => sum + run.usage.costUsd, 0);
+    }
+    return summary;
+}
 /**
  * Create a summary from multiple run data.
  */
@@ -54,12 +110,14 @@ export function createEvalSummary(name, runData) {
     const runs = runData.map((r) => r.result);
     const passedRuns = runs.filter((r) => r.status === 'passed').length;
     const totalDuration = runs.reduce((sum, r) => sum + r.duration, 0);
+    const usage = summarizeUsage(runs);
     return {
         name,
         totalRuns: runs.length,
         passedRuns,
         passRate: runs.length > 0 ? (passedRuns / runs.length) * 100 : 0,
         meanDuration: runs.length > 0 ? totalDuration / runs.length : 0,
+        ...(usage ? { usage } : {}),
         runs: runData,
     };
 }
@@ -109,6 +167,10 @@ export function saveResults(results, options) {
             passRate: `${evalSummary.passRate.toFixed(0)}%`,
             meanDuration: evalSummary.meanDuration,
         };
+        const usage = summarizeUsage(evalSummary.runs.map((run) => run.result));
+        if (usage) {
+            summaryForFile.usage = usage;
+        }
         if (fingerprint) {
             summaryForFile.fingerprint = fingerprint;
         }
@@ -154,8 +216,11 @@ export function saveResults(results, options) {
                 // Save raw transcript for debugging (transcript-raw.jsonl)
                 writeFileSync(join(runDir, 'transcript-raw.jsonl'), runData.transcript);
                 resultWithPaths.transcriptRawPath = './transcript-raw.jsonl';
-                // Include summary in result.json for quick access
-                resultWithPaths.o11y = transcript.summary;
+                // Include summary in result.json for quick access. Usage the agent
+                // reported directly (outside its transcript) wins, as in the result.
+                resultWithPaths.o11y = runData.result.usage
+                    ? { ...transcript.summary, usage: runData.result.usage }
+                    : transcript.summary;
             }
             // Save script/test outputs to outputs/
             const outputsDir = join(runDir, 'outputs');
@@ -205,6 +270,10 @@ export function saveResults(results, options) {
                     // Remove deleted files from the copied project
                     if (runData.deletedFiles) {
                         for (const filePath of runData.deletedFiles) {
+                            // Captured paths come from the agent's sandbox; never let one
+                            // reach outside the project directory on this machine.
+                            if (!isWorkspacePath(filePath))
+                                continue;
                             const fullPath = join(projectDir, filePath);
                             if (existsSync(fullPath)) {
                                 unlinkSync(fullPath);
@@ -215,6 +284,8 @@ export function saveResults(results, options) {
                 // Write generated files (overwrites originals in 'all' mode where paths match)
                 if (runData.generatedFiles) {
                     for (const [filePath, content] of Object.entries(runData.generatedFiles)) {
+                        if (!isWorkspacePath(filePath))
+                            continue;
                         const fullPath = join(projectDir, filePath);
                         mkdirSync(dirname(fullPath), { recursive: true });
                         writeFileSync(fullPath, content);

package/dist/lib/results.js.map

@@ -1 +1 @@
-{"version":3,"file":"results.js","sourceRoot":"","sources":["../../src/lib/results.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EACN,SAAS,EACT,aAAa,EACb,WAAW,EACX,YAAY,EACZ,UAAU,EACV,QAAQ,EACR,UAAU,GACV,MAAM,IAAI,CAAC;AACZ,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AACrC,OAAO,KAAK,MAAM,OAAO,CAAC;AAS1B,OAAO,EAAE,eAAe,EAAmB,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACpD,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAChD,OAAO,EAAE,oCAAoC,EAAE,MAAM,kBAAkB,CAAC;AAExE;;GAEG;AACH,MAAM,UAAU,wBAAwB,CACvC,WAA2B;IAE3B,gDAAgD;IAChD,MAAM,aAAa,GAAiC,EAAE,CAAC;IAEvD,0BAA0B;IAC1B,IAAI,WAAW,CAAC,UAAU,EAAE,MAAM,EAAE,CAAC;QACpC,aAAa,CAAC,IAAI,GAAG,WAAW,CAAC,UAAU,CAAC,MAAM,CAAC;IACpD,CAAC;IAED,qEAAqE;IACrE,IACC,WAAW,CAAC,cAAc;QAC1B,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC,MAAM,GAAG,CAAC,EACjD,CAAC;QACF,aAAa,CAAC,OAAO,GAAG,EAAE,CAAC;QAC3B,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAC1C,WAAW,CAAC,cAAc,CAC1B,EAAE,CAAC;YACH,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;gBACnB,aAAa,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;YAC7C,CAAC;QACF,CAAC;IACF,CAAC;IAED,MAAM,MAAM,GAAkB;QAC7B,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;QACjD,KAAK,EAAE,WAAW,CAAC,KAAK;QACxB,QAAQ,EAAE,WAAW,CAAC,QAAQ,GAAG,IAAI,EAAE,qBAAqB;KAC5D,CAAC;IACF,IAAI,WAAW,CAAC,aAAa,EAAE,CAAC;QAC/B,MAAM,CAAC,aAAa,GAAG,WAAW,CAAC,aAAa,CAAC;IAClD,CAAC;IACD,IAAI,WAAW,CAAC,WAAW,EAAE,CAAC;QAC7B,MAAM,CAAC,WAAW,GAAG,WAAW,CAAC,WAAW,CAAC;IAC9C,CAAC;IAED,OAAO;QACN,MAAM;QACN,UAAU,EAAE,WAAW,CAAC,UAAU;QAClC,aAAa,EACZ,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS;QAClE,cAAc,EAAE,WAAW,CAAC,cAAc;QAC1C,YAAY,EAAE,WAAW,CAAC,YAAY;KACtC,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAChC,IAAY,EACZ,OAAsB;IAEtB,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAC1C,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAC;IACpE,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;IAEnE,OAAO;QACN,IAAI;QACJ,SAAS,EAAE,IAAI,CAAC,MAAM;QACtB,UAAU;QACV,QAAQ,EAAE,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;QAChE,YAAY,EAAE,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC/D,IAAI,EAAE,OAAO;KACb,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,uBAAuB,CACtC,MAAgC,EAChC,KAAoB,EACpB,SAAe,EACf,WAAiB;IAEjB,OAAO;QACN,SAAS,EAAE,SAAS,CAAC,WAAW,EAAE;QAClC,WAAW,EAAE,WAAW,CAAC,WAAW,EAAE;QACtC,MAAM;QACN,KAAK;KACL,CAAC;AACH,CAAC;AAwBD;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,WAAW,CAC1B,OAA0B,EAC1B,OAA2B;IAE3B,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACvD,MAAM,aAAa,GAAG,IAAI,CACzB,OAAO,CAAC,UAAU,EAClB,OAAO,CAAC,cAAc,EACtB,SAAS,CACT,CAAC;IAEF,8BAA8B;IAC9B,SAAS,CAAC,aAAa,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE9C,wBAAwB;IACxB,KAAK,MAAM,WAAW,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACzC,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,EAAE,WAAW,CAAC,IAAI,CAAC,CAAC;QACtD,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAExC,8CAA8C;QAC9C,MAAM,WAAW,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QAC7D,MAAM,kBAAkB,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QAC3E,MAAM,6BAA6B,GAAG,oCAAoC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC3F,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QACnD,MAAM,cAAc,GAA4B;YAC/C,SAAS,EAAE,WAAW,CAAC,SAAS;YAChC,UAAU,EAAE,WAAW,CAAC,UAAU;YAClC,QAAQ,EAAE,GAAG,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;YAC/C,YAAY,EAAE,WAAW,CAAC,YAAY;SACtC,CAAC;QACF,IAAI,WAAW,EAAE,CAAC;YACjB,cAAc,CAAC,WAAW,GAAG,WAAW,CAAC;QAC1C,CAAC;QACD,IAAI,kBAAkB,EAAE,CAAC;YACxB,cAAc,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QACxD,CAAC;QACD,IAAI,6BAA6B,EAAE,CAAC;YACnC,cAAc,CAAC,6BAA6B,GAAG,6BAA6B,CAAC;QAC9E,CAAC;QACD,IAAI,KAAK,KAAK,KAAK,EAAE,CAAC;YACrB,cAAc,CAAC,KAAK,GAAG,KAAK,CAAC;QAC9B,CAAC;QACD,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;YACnB,cAAc,CAAC,KAAK,GAAG,IAAI,CAAC;QAC7B,CAAC;QACD,aAAa,CACZ,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,EAC7B,IAAI,CAAC,SAAS,CAAC,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC,CACvC,CAAC;QAEF,8BAA8B;QAC9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAClD,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACpC,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC7C,SAAS,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAEvC,+CAA+C;YAC/C,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,WAAW,IAAI,eAAe,CAAC;YAClE,MAAM,cAAc,GAAG,WAAW,KAAK,gBAAgB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC;YAC3F,MAAM,EAAE,aAAa,EAAE,GAAG,UAAU,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;YACxD,MAAM,KAAK,GAAG,WAAW,KAAK,gBAAgB,CAAC,CAAC,CAAC,CAAC,aAAa,IAAI,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC;YAChH,MAAM,eAAe,GAEjB;gBACH,GAAG,UAAU;gBACb,KAAK;aACL,CAAC;YACF,IAAI,WAAW,KAAK,gBAAgB,EAAE,CAAC;gBACtC,eAAe,CAAC,WAAW,GAAG,WAAW,CAAC;gBAC1C,eAAe,CAAC,cAAc,GAAG,cAAc,CAAC;gBAChD,eAAe,CAAC,aAAa,GAAG,aAAa,CAAC;YAC/C,CAAC;YAED,gCAAgC;YAChC,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;gBACxB,2BAA2B;gBAC3B,MAAM,UAAU,GAAG,eAAe,CACjC,OAAO,CAAC,UAAU,EAClB,OAAO,CAAC,MAAM,CAAC,KAAK,EACpB,KAAK,CACL,CAAC;gBAEF,6DAA6D;gBAC7D,aAAa,CACZ,IAAI,CAAC,MAAM,EAAE,iBAAiB,CAAC,EAC/B,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CACnC,CAAC;gBACF,eAAe,CAAC,cAAc,GAAG,mBAAmB,CAAC;gBAErD,2DAA2D;gBAC3D,aAAa,CACZ,IAAI,CAAC,MAAM,EAAE,sBAAsB,CAAC,EACpC,OAAO,CAAC,UAAU,CAClB,CAAC;gBACF,eAAe,CAAC,iBAAiB,GAAG,wBAAwB,CAAC;gBAE7D,kDAAkD;gBAClD,eAAe,CAAC,IAAI,GAAG,UAAU,CAAC,OAAO,CAAC;YAC3C,CAAC;YAED,uCAAuC;YACvC,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YAC3C,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAE3C,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;gBAC3B,MAAM,WAAW,GAAiC,EAAE,CAAC;gBAErD,2BAA2B;gBAC3B,IAAI,OAAO,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;oBAChC,aAAa,CACZ,IAAI,CAAC,UAAU,EAAE,UAAU,CAAC,EAC5B,OAAO,CAAC,aAAa,CAAC,IAAI,CAC1B,CAAC;oBACF,WAAW,CAAC,IAAI,GAAG,oBAAoB,CAAC;gBACzC,CAAC;gBAED,kFAAkF;gBAClF,IAAI,OAAO,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC;oBACnC,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;oBAC/C,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;oBAC3C,WAAW,CAAC,OAAO,GAAG,EAAE,CAAC;oBACzB,KAAK,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAC3C,OAAO,CAAC,aAAa,CAAC,OAAO,CAC7B,EAAE,CAAC;wBACH,IAAI,OAAO,EAAE,CAAC;4BACb,MAAM,QAAQ,GAAG,GAAG,IAAI,MAAM,CAAC;4BAC/B,aAAa,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,EAAE,OAAO,CAAC,CAAC;4BACnD,WAAW,CAAC,OAAO,CAClB,IAAI,CACJ,GAAG,qBAAqB,QAAQ,EAAE,CAAC;wBACrC,CAAC;oBACF,CAAC;gBACF,CAAC;gBAED,IACC,WAAW,CAAC,IAAI;oBAChB,CAAC,WAAW,CAAC,OAAO;wBACnB,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,EAC5C,CAAC;oBACF,eAAe,CAAC,WAAW,GAAG,WAAW,CAAC;gBAC3C,CAAC;YACF,CAAC;YAED,+CAA+C;YAC/C,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC;YAC3C,MAAM,iBAAiB,GACtB,OAAO,CAAC,cAAc;gBACtB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;YAChD,MAAM,eAAe,GACpB,OAAO,CAAC,YAAY,IAAI,OAAO,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC;YACzD,IACC,SAAS;gBACT,SAAS,KAAK,MAAM;gBACpB,CAAC,iBAAiB,IAAI,eAAe,CAAC,EACrC,CAAC;gBACF,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;gBAC3C,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;gBAE3C,oDAAoD;gBACpD,IAAI,SAAS,KAAK,KAAK,EAAE,CAAC;oBACzB,MAAM,WAAW,GAChB,OAAO,CAAC,YAAY,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;oBAC1C,IAAI,WAAW,EAAE,CAAC;wBACjB,gBAAgB,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC;oBAC3C,CAAC;oBAED,+CAA+C;oBAC/C,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;wBAC1B,KAAK,MAAM,QAAQ,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;4BAC7C,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;4BAC5C,IAAI,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;gCAC1B,UAAU,CAAC,QAAQ,CAAC,CAAC;4BACtB,CAAC;wBACF,CAAC;oBACF,CAAC;gBACF,CAAC;gBAED,+EAA+E;gBAC/E,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;oBAC5B,KAAK,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAC/C,OAAO,CAAC,cAAc,CACtB,EAAE,CAAC;wBACH,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;wBAC5C,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;wBAClD,aAAa,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;oBAClC,CAAC;gBACF,CAAC;YACF,CAAC;YAED,+CAA+C;YAC/C,aAAa,CACZ,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,EAC3B,IAAI,CAAC,SAAS,CAAC,eAAe,EAAE,IAAI,EAAE,CAAC,CAAC,CACxC,CAAC;QACH,CAAC;IACF,CAAC;IAED,OAAO,aAAa,CAAC;AACtB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,OAA0B;IAC5D,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,SAAS,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAEjC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC;IAC7C,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;IAClC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,0BAA0B;IAC1B,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;IACzE,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;IAC5E,MAAM,eAAe,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,SAAS,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAE5E,KAAK,MAAM,WAAW,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACzC,MAAM,QAAQ,GACb,WAAW,CAAC,UAAU,KAAK,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;QAC9D,MAAM,SAAS,GACd,WAAW,CAAC,UAAU,KAAK,WAAW,CAAC,SAAS;YAC/C,CAAC,CAAC,KAAK,CAAC,KAAK;YACb,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC;QAEd,KAAK,CAAC,IAAI,CACT,SAAS,CACR,GAAG,QAAQ,IAAI,WAAW,CAAC,IAAI,KAAK,WAAW,CAAC,UAAU,IACzD,WAAW,CAAC,SACb,YAAY,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAC/C,CACD,CAAC;QACF,KAAK,CAAC,IAAI,CACT,KAAK,CAAC,IAAI,CACT,oBAAoB,WAAW,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAC1D,CACD,CAAC;QACF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;IAClC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,MAAM,YAAY,GACjB,eAAe,KAAK,GAAG;QACtB,CAAC,CAAC,KAAK,CAAC,KAAK;QACb,CAAC,CAAC,eAAe,IAAI,EAAE;YACvB,CAAC,CAAC,KAAK,CAAC,MAAM;YACd,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC;IACd,KAAK,CAAC,IAAI,CACT,YAAY,CACX,YAAY,WAAW,IAAI,SAAS,YAAY,eAAe,CAAC,OAAO,CACtE,CAAC,CACD,IAAI,CACL,CACD,CAAC;IAEF,MAAM,QAAQ,GACb,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,OAAO,EAAE;QACvC,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC;QACvC,IAAI,CAAC;IACN,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,eAAe,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC9D,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACzB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAC9B,QAAgB,EAChB,SAAiB,EACjB,SAAiB,EACjB,MAAqB,EACrB,OAAqE;IAErE,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IACpD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC;IACnE,MAAM,MAAM,GAAG,OAAO,EAAE,cAAc;QACrC,CAAC,CAAC,GAAG,OAAO,CAAC,cAAc,IAAI,QAAQ,EAAE;QACzC,CAAC,CAAC,QAAQ,CAAC;IAEZ,IAAI,IAAI,GAAG,KAAK,CAAC,GAAG,IAAI,IAAI,MAAM,KAAK,SAAS,IAAI,SAAS,GAAG,CAAC,CAAC;IAClE,IAAI,OAAO,EAAE,KAAK,IAAI,OAAO,EAAE,KAAK,EAAE,CAAC;QACtC,IAAI,IAAI,KAAK,CAAC,IAAI,CACjB,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAClE,CAAC;IACH,CAAC;IACD,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAExD,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QAClB,IAAI,IAAI,KAAK,CAAC,GAAG,CAChB,MAAM,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAC/B,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EACrC,EAAE,CACF,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACb,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB,CACpC,QAAgB,EAChB,SAAiB,EACjB,SAAiB,EACjB,OAAqE;IAErE,MAAM,MAAM,GAAG,OAAO,EAAE,cAAc;QACrC,CAAC,CAAC,GAAG,OAAO,CAAC,cAAc,IAAI,QAAQ,EAAE;QACzC,CAAC,CAAC,QAAQ,CAAC;IACZ,MAAM,IAAI,GAAG,CAAC,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC1E,MAAM,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IACxC,OAAO,KAAK,CAAC,IAAI,CAChB,WAAW,MAAM,KAAK,SAAS,IAAI,SAAS,IAAI,MAAM,KAAK,CAC3D,CAAC;AACH,CAAC;AAkBD;;;;;;;;;GASG;AACH,MAAM,UAAU,mBAAmB,CAClC,UAAkB,EAClB,cAAsB,EACtB,YAAoC,EACpC,UAA4B,EAAE;IAE9B,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA0B,CAAC;IACnD,MAAM,aAAa,GAAG,IAAI,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;IAEvD,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC;QAAE,OAAO,QAAQ,CAAC;IAEhD,0CAA0C;IAC1C,IAAI,UAAoB,CAAC;IACzB,IAAI,CAAC;QACJ,UAAU,GAAG,WAAW,CAAC,aAAa,CAAC;aACrC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;aACjC,IAAI,EAAE;aACN,OAAO,EAAE,CAAC;IACb,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,QAAQ,CAAC;IACjB,CAAC;IAED,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACpC,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;QAC7C,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE;YAAE,SAAS;QAE7C,IAAI,QAAkB,CAAC;QACvB,IAAI,CAAC;YACJ,QAAQ,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;QACjE,CAAC;QAAC,MAAM,CAAC;YACR,SAAS;QACV,CAAC;QAED,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAChC,gDAAgD;YAChD,IAAI,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC;gBAAE,SAAS;YAEpC,+CAA+C;YAC/C,MAAM,mBAAmB,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;YAClD,IAAI,CAAC,mBAAmB;gBAAE,SAAS;YAEnC,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,EAAE,OAAO,EAAE,cAAc,CAAC,CAAC;YACzD,IAAI,CAAC;gBACJ,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC;gBAE/D,yEAAyE;gBACzE,wEAAwE;gBACxE,0DAA0D;gBAC1D,IAAI,OAAO,CAAC,WAAW,KAAK,mBAAmB;oBAAE,SAAS;gBAC1D,IACC,OAAO,CAAC,yBAAyB;oBACjC,OAAO,CAAC,6BAA6B,KAAK,OAAO,CAAC,6BAA6B;oBAC9E,SAAS;gBAEX,yEAAyE;gBACzE,IAAI,OAAO,CAAC,KAAK,KAAK,KAAK;oBAAE,SAAS;gBAEtC,0BAA0B;gBAC1B,IAAI,OAAO,CAAC,KAAK,KAAK,IAAI;oBAAE,SAAS;gBAErC,kEAAkE;gBAClE,IAAI,iBAAiB,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;oBAAE,SAAS;gBAEtD,oEAAoE;gBACpE,IAAI,OAAO,CAAC,SAAS,IAAI,CAAC;oBAAE,SAAS;gBAErC,4EAA4E;gBAC5E,iFAAiF;gBACjF,IACC,OAAO,CAAC,UAAU,KAAK,CAAC;oBACxB,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,EAAE,qBAAqB,CAAC,CAAC;oBAExD,SAAS;gBAEV,QAAQ,CAAC,GAAG,CAAC,OAAO,EAAE;oBACrB,QAAQ,EAAE,OAAO;oBACjB,WAAW,EAAE,OAAO,CAAC,WAAW;oBAChC,QAAQ,EAAE,OAAO,CAAC,QAAQ;oBAC1B,SAAS;iBACT,CAAC,CAAC;YACJ,CAAC;YAAC,MAAM,CAAC;gBACR,yBAAyB;YAC1B,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,QAAQ,CAAC;AACjB,CAAC"}
+{"version":3,"file":"results.js","sourceRoot":"","sources":["../../src/lib/results.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EACN,SAAS,EACT,aAAa,EACb,WAAW,EACX,YAAY,EACZ,UAAU,EACV,QAAQ,EACR,UAAU,GACV,MAAM,IAAI,CAAC;AACZ,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AACrC,OAAO,KAAK,MAAM,OAAO,CAAC;AAU1B,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,eAAe,EAAE,oBAAoB,EAAmB,MAAM,iBAAiB,CAAC;AACzF,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACpD,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAChD,OAAO,EAAE,oCAAoC,EAAE,MAAM,kBAAkB,CAAC;AAExE;;;;;;GAMG;AACH,MAAM,UAAU,wBAAwB,CACvC,WAA2B,EAC3B,UAAsC,EAAE;IAExC,gDAAgD;IAChD,MAAM,aAAa,GAAiC,EAAE,CAAC;IAEvD,0BAA0B;IAC1B,IAAI,WAAW,CAAC,UAAU,EAAE,MAAM,EAAE,CAAC;QACpC,aAAa,CAAC,IAAI,GAAG,WAAW,CAAC,UAAU,CAAC,MAAM,CAAC;IACpD,CAAC;IAED,qEAAqE;IACrE,IACC,WAAW,CAAC,cAAc;QAC1B,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC,MAAM,GAAG,CAAC,EACjD,CAAC;QACF,aAAa,CAAC,OAAO,GAAG,EAAE,CAAC;QAC3B,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAC1C,WAAW,CAAC,cAAc,CAC1B,EAAE,CAAC;YACH,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;gBACnB,aAAa,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;YAC7C,CAAC;QACF,CAAC;IACF,CAAC;IAED,MAAM,MAAM,GAAkB;QAC7B,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;QACjD,KAAK,EAAE,WAAW,CAAC,KAAK;QACxB,QAAQ,EAAE,WAAW,CAAC,QAAQ,GAAG,IAAI,EAAE,qBAAqB;KAC5D,CAAC;IACF,IAAI,WAAW,CAAC,aAAa,EAAE,CAAC;QAC/B,MAAM,CAAC,aAAa,GAAG,WAAW,CAAC,aAAa,CAAC;IAClD,CAAC;IACD,IAAI,WAAW,CAAC,WAAW,EAAE,CAAC;QAC7B,MAAM,CAAC,WAAW,GAAG,WAAW,CAAC,WAAW,CAAC;IAC9C,CAAC;IACD,IAAI,WAAW,CAAC,UAAU,EAAE,CAAC;QAC5B,MAAM,CAAC,UAAU,GAAG,WAAW,CAAC,UAAU,CAAC;IAC5C,CAAC;IACD,MAAM,KAAK,GACV,WAAW,CAAC,KAAK;QACjB,CAAC,OAAO,CAAC,aAAa;YACrB,CAAC,CAAC,oBAAoB,CAAC,WAAW,CAAC,UAAU,EAAE,OAAO,CAAC,aAAa,CAAC;YACrE,CAAC,CAAC,SAAS,CAAC,CAAC;IACf,IAAI,KAAK,EAAE,CAAC;QACX,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC;IACtB,CAAC;IACD,IAAI,WAAW,CAAC,QAAQ,EAAE,CAAC;QAC1B,MAAM,CAAC,QAAQ,GAAG,WAAW,CAAC,QAAQ,CAAC;IACxC,CAAC;IACD,IAAI,WAAW,CAAC,iBAAiB,EAAE,CAAC;QACnC,MAAM,CAAC,iBAAiB,GAAG,WAAW,CAAC,iBAAiB,CAAC;IAC1D,CAAC;IACD,IAAI,WAAW,CAAC,SAAS,EAAE,CAAC;QAC3B,MAAM,CAAC,SAAS,GAAG,WAAW,CAAC,SAAS,CAAC;IAC1C,CAAC;IAED,OAAO;QACN,MAAM;QACN,UAAU,EAAE,WAAW,CAAC,UAAU;QAClC,aAAa,EACZ,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS;QAClE,cAAc,EAAE,WAAW,CAAC,cAAc;QAC1C,YAAY,EAAE,WAAW,CAAC,YAAY;KACtC,CAAC;AACH,CAAC;AAED,MAAM,kBAAkB,GAAG;IAC1B,aAAa;IACb,aAAa;IACb,cAAc;IACd,iBAAiB;IACjB,kBAAkB;IAClB,iBAAiB;CACR,CAAC;AAEX;;;;;GAKG;AACH,MAAM,UAAU,cAAc,CAC7B,IAA8B;IAE9B,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACvE,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAE5C,MAAM,OAAO,GAAiB,EAAE,aAAa,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC;IACjE,KAAK,MAAM,KAAK,IAAI,kBAAkB,EAAE,CAAC;QACxC,IAAI,QAAQ,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,SAAS,CAAC,EAAE,CAAC;YAC3D,OAAO,CAAC,KAAK,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,CAAC,KAAK,CAAE,EAAE,CAAC,CAAC,CAAC;QAC1E,CAAC;IACF,CAAC;IACD,IAAI,OAAO,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;QACvC,OAAO,CAAC,eAAe,GAAG,OAAO,CAAC,WAAW,GAAG,QAAQ,CAAC,MAAM,CAAC;IACjE,CAAC;IACD,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,KAAK,SAAS,CAAC,EAAE,CAAC;QAC3D,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,CAAC,KAAM,CAAC,OAAQ,EAAE,CAAC,CAAC,CAAC;IAC3E,CAAC;IACD,OAAO,OAAO,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAChC,IAAY,EACZ,OAAsB;IAEtB,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAC1C,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAC;IACpE,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;IACnE,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;IAEnC,OAAO;QACN,IAAI;QACJ,SAAS,EAAE,IAAI,CAAC,MAAM;QACtB,UAAU;QACV,QAAQ,EAAE,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;QAChE,YAAY,EAAE,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC/D,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3B,IAAI,EAAE,OAAO;KACb,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,uBAAuB,CACtC,MAAgC,EAChC,KAAoB,EACpB,SAAe,EACf,WAAiB;IAEjB,OAAO;QACN,SAAS,EAAE,SAAS,CAAC,WAAW,EAAE;QAClC,WAAW,EAAE,WAAW,CAAC,WAAW,EAAE;QACtC,MAAM;QACN,KAAK;KACL,CAAC;AACH,CAAC;AAwBD;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,WAAW,CAC1B,OAA0B,EAC1B,OAA2B;IAE3B,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACvD,MAAM,aAAa,GAAG,IAAI,CACzB,OAAO,CAAC,UAAU,EAClB,OAAO,CAAC,cAAc,EACtB,SAAS,CACT,CAAC;IAEF,8BAA8B;IAC9B,SAAS,CAAC,aAAa,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE9C,wBAAwB;IACxB,KAAK,MAAM,WAAW,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACzC,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,EAAE,WAAW,CAAC,IAAI,CAAC,CAAC;QACtD,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAExC,8CAA8C;QAC9C,MAAM,WAAW,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QAC7D,MAAM,kBAAkB,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QAC3E,MAAM,6BAA6B,GAAG,oCAAoC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC3F,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QACnD,MAAM,cAAc,GAA4B;YAC/C,SAAS,EAAE,WAAW,CAAC,SAAS;YAChC,UAAU,EAAE,WAAW,CAAC,UAAU;YAClC,QAAQ,EAAE,GAAG,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;YAC/C,YAAY,EAAE,WAAW,CAAC,YAAY;SACtC,CAAC;QACF,MAAM,KAAK,GAAG,cAAc,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;QACxE,IAAI,KAAK,EAAE,CAAC;YACX,cAAc,CAAC,KAAK,GAAG,KAAK,CAAC;QAC9B,CAAC;QACD,IAAI,WAAW,EAAE,CAAC;YACjB,cAAc,CAAC,WAAW,GAAG,WAAW,CAAC;QAC1C,CAAC;QACD,IAAI,kBAAkB,EAAE,CAAC;YACxB,cAAc,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QACxD,CAAC;QACD,IAAI,6BAA6B,EAAE,CAAC;YACnC,cAAc,CAAC,6BAA6B,GAAG,6BAA6B,CAAC;QAC9E,CAAC;QACD,IAAI,KAAK,KAAK,KAAK,EAAE,CAAC;YACrB,cAAc,CAAC,KAAK,GAAG,KAAK,CAAC;QAC9B,CAAC;QACD,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;YACnB,cAAc,CAAC,KAAK,GAAG,IAAI,CAAC;QAC7B,CAAC;QACD,aAAa,CACZ,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,EAC7B,IAAI,CAAC,SAAS,CAAC,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC,CACvC,CAAC;QAEF,8BAA8B;QAC9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAClD,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACpC,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC7C,SAAS,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAEvC,+CAA+C;YAC/C,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,WAAW,IAAI,eAAe,CAAC;YAClE,MAAM,cAAc,GAAG,WAAW,KAAK,gBAAgB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC;YAC3F,MAAM,EAAE,aAAa,EAAE,GAAG,UAAU,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;YACxD,MAAM,KAAK,GAAG,WAAW,KAAK,gBAAgB,CAAC,CAAC,CAAC,CAAC,aAAa,IAAI,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC;YAChH,MAAM,eAAe,GAEjB;gBACH,GAAG,UAAU;gBACb,KAAK;aACL,CAAC;YACF,IAAI,WAAW,KAAK,gBAAgB,EAAE,CAAC;gBACtC,eAAe,CAAC,WAAW,GAAG,WAAW,CAAC;gBAC1C,eAAe,CAAC,cAAc,GAAG,cAAc,CAAC;gBAChD,eAAe,CAAC,aAAa,GAAG,aAAa,CAAC;YAC/C,CAAC;YAED,gCAAgC;YAChC,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;gBACxB,2BAA2B;gBAC3B,MAAM,UAAU,GAAG,eAAe,CACjC,OAAO,CAAC,UAAU,EAClB,OAAO,CAAC,MAAM,CAAC,KAAK,EACpB,KAAK,CACL,CAAC;gBAEF,6DAA6D;gBAC7D,aAAa,CACZ,IAAI,CAAC,MAAM,EAAE,iBAAiB,CAAC,EAC/B,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CACnC,CAAC;gBACF,eAAe,CAAC,cAAc,GAAG,mBAAmB,CAAC;gBAErD,2DAA2D;gBAC3D,aAAa,CACZ,IAAI,CAAC,MAAM,EAAE,sBAAsB,CAAC,EACpC,OAAO,CAAC,UAAU,CAClB,CAAC;gBACF,eAAe,CAAC,iBAAiB,GAAG,wBAAwB,CAAC;gBAE7D,mEAAmE;gBACnE,qEAAqE;gBACrE,eAAe,CAAC,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK;oBAC1C,CAAC,CAAC,EAAE,GAAG,UAAU,CAAC,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE;oBACxD,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC;YACvB,CAAC;YAED,uCAAuC;YACvC,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YAC3C,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAE3C,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;gBAC3B,MAAM,WAAW,GAAiC,EAAE,CAAC;gBAErD,2BAA2B;gBAC3B,IAAI,OAAO,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;oBAChC,aAAa,CACZ,IAAI,CAAC,UAAU,EAAE,UAAU,CAAC,EAC5B,OAAO,CAAC,aAAa,CAAC,IAAI,CAC1B,CAAC;oBACF,WAAW,CAAC,IAAI,GAAG,oBAAoB,CAAC;gBACzC,CAAC;gBAED,kFAAkF;gBAClF,IAAI,OAAO,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC;oBACnC,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;oBAC/C,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;oBAC3C,WAAW,CAAC,OAAO,GAAG,EAAE,CAAC;oBACzB,KAAK,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAC3C,OAAO,CAAC,aAAa,CAAC,OAAO,CAC7B,EAAE,CAAC;wBACH,IAAI,OAAO,EAAE,CAAC;4BACb,MAAM,QAAQ,GAAG,GAAG,IAAI,MAAM,CAAC;4BAC/B,aAAa,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,EAAE,OAAO,CAAC,CAAC;4BACnD,WAAW,CAAC,OAAO,CAClB,IAAI,CACJ,GAAG,qBAAqB,QAAQ,EAAE,CAAC;wBACrC,CAAC;oBACF,CAAC;gBACF,CAAC;gBAED,IACC,WAAW,CAAC,IAAI;oBAChB,CAAC,WAAW,CAAC,OAAO;wBACnB,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,EAC5C,CAAC;oBACF,eAAe,CAAC,WAAW,GAAG,WAAW,CAAC;gBAC3C,CAAC;YACF,CAAC;YAED,+CAA+C;YAC/C,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC;YAC3C,MAAM,iBAAiB,GACtB,OAAO,CAAC,cAAc;gBACtB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;YAChD,MAAM,eAAe,GACpB,OAAO,CAAC,YAAY,IAAI,OAAO,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC;YACzD,IACC,SAAS;gBACT,SAAS,KAAK,MAAM;gBACpB,CAAC,iBAAiB,IAAI,eAAe,CAAC,EACrC,CAAC;gBACF,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;gBAC3C,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;gBAE3C,oDAAoD;gBACpD,IAAI,SAAS,KAAK,KAAK,EAAE,CAAC;oBACzB,MAAM,WAAW,GAChB,OAAO,CAAC,YAAY,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;oBAC1C,IAAI,WAAW,EAAE,CAAC;wBACjB,gBAAgB,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC;oBAC3C,CAAC;oBAED,+CAA+C;oBAC/C,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;wBAC1B,KAAK,MAAM,QAAQ,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;4BAC7C,8DAA8D;4BAC9D,uDAAuD;4BACvD,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC;gCAAE,SAAS;4BACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;4BAC5C,IAAI,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;gCAC1B,UAAU,CAAC,QAAQ,CAAC,CAAC;4BACtB,CAAC;wBACF,CAAC;oBACF,CAAC;gBACF,CAAC;gBAED,+EAA+E;gBAC/E,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;oBAC5B,KAAK,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAC/C,OAAO,CAAC,cAAc,CACtB,EAAE,CAAC;wBACH,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC;4BAAE,SAAS;wBACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;wBAC5C,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;wBAClD,aAAa,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;oBAClC,CAAC;gBACF,CAAC;YACF,CAAC;YAED,+CAA+C;YAC/C,aAAa,CACZ,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,EAC3B,IAAI,CAAC,SAAS,CAAC,eAAe,EAAE,IAAI,EAAE,CAAC,CAAC,CACxC,CAAC;QACH,CAAC;IACF,CAAC;IAED,OAAO,aAAa,CAAC;AACtB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,OAA0B;IAC5D,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,SAAS,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAEjC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC;IAC7C,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;IAClC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,0BAA0B;IAC1B,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;IACzE,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;IAC5E,MAAM,eAAe,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,SAAS,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAE5E,KAAK,MAAM,WAAW,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACzC,MAAM,QAAQ,GACb,WAAW,CAAC,UAAU,KAAK,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;QAC9D,MAAM,SAAS,GACd,WAAW,CAAC,UAAU,KAAK,WAAW,CAAC,SAAS;YAC/C,CAAC,CAAC,KAAK,CAAC,KAAK;YACb,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC;QAEd,KAAK,CAAC,IAAI,CACT,SAAS,CACR,GAAG,QAAQ,IAAI,WAAW,CAAC,IAAI,KAAK,WAAW,CAAC,UAAU,IACzD,WAAW,CAAC,SACb,YAAY,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAC/C,CACD,CAAC;QACF,KAAK,CAAC,IAAI,CACT,KAAK,CAAC,IAAI,CACT,oBAAoB,WAAW,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAC1D,CACD,CAAC;QACF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;IAClC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,MAAM,YAAY,GACjB,eAAe,KAAK,GAAG;QACtB,CAAC,CAAC,KAAK,CAAC,KAAK;QACb,CAAC,CAAC,eAAe,IAAI,EAAE;YACvB,CAAC,CAAC,KAAK,CAAC,MAAM;YACd,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC;IACd,KAAK,CAAC,IAAI,CACT,YAAY,CACX,YAAY,WAAW,IAAI,SAAS,YAAY,eAAe,CAAC,OAAO,CACtE,CAAC,CACD,IAAI,CACL,CACD,CAAC;IAEF,MAAM,QAAQ,GACb,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,OAAO,EAAE;QACvC,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC;QACvC,IAAI,CAAC;IACN,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,eAAe,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC9D,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACzB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAC9B,QAAgB,EAChB,SAAiB,EACjB,SAAiB,EACjB,MAAqB,EACrB,OAAqE;IAErE,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IACpD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC;IACnE,MAAM,MAAM,GAAG,OAAO,EAAE,cAAc;QACrC,CAAC,CAAC,GAAG,OAAO,CAAC,cAAc,IAAI,QAAQ,EAAE;QACzC,CAAC,CAAC,QAAQ,CAAC;IAEZ,IAAI,IAAI,GAAG,KAAK,CAAC,GAAG,IAAI,IAAI,MAAM,KAAK,SAAS,IAAI,SAAS,GAAG,CAAC,CAAC;IAClE,IAAI,OAAO,EAAE,KAAK,IAAI,OAAO,EAAE,KAAK,EAAE,CAAC;QACtC,IAAI,IAAI,KAAK,CAAC,IAAI,CACjB,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAClE,CAAC;IACH,CAAC;IACD,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAExD,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QAClB,IAAI,IAAI,KAAK,CAAC,GAAG,CAChB,MAAM,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAC/B,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EACrC,EAAE,CACF,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACb,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB,CACpC,QAAgB,EAChB,SAAiB,EACjB,SAAiB,EACjB,OAAqE;IAErE,MAAM,MAAM,GAAG,OAAO,EAAE,cAAc;QACrC,CAAC,CAAC,GAAG,OAAO,CAAC,cAAc,IAAI,QAAQ,EAAE;QACzC,CAAC,CAAC,QAAQ,CAAC;IACZ,MAAM,IAAI,GAAG,CAAC,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC1E,MAAM,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IACxC,OAAO,KAAK,CAAC,IAAI,CAChB,WAAW,MAAM,KAAK,SAAS,IAAI,SAAS,IAAI,MAAM,KAAK,CAC3D,CAAC;AACH,CAAC;AAkBD;;;;;;;;;GASG;AACH,MAAM,UAAU,mBAAmB,CAClC,UAAkB,EAClB,cAAsB,EACtB,YAAoC,EACpC,UAA4B,EAAE;IAE9B,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA0B,CAAC;IACnD,MAAM,aAAa,GAAG,IAAI,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;IAEvD,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC;QAAE,OAAO,QAAQ,CAAC;IAEhD,0CAA0C;IAC1C,IAAI,UAAoB,CAAC;IACzB,IAAI,CAAC;QACJ,UAAU,GAAG,WAAW,CAAC,aAAa,CAAC;aACrC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;aACjC,IAAI,EAAE;aACN,OAAO,EAAE,CAAC;IACb,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,QAAQ,CAAC;IACjB,CAAC;IAED,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACpC,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;QAC7C,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE;YAAE,SAAS;QAE7C,IAAI,QAAkB,CAAC;QACvB,IAAI,CAAC;YACJ,QAAQ,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;QACjE,CAAC;QAAC,MAAM,CAAC;YACR,SAAS;QACV,CAAC;QAED,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAChC,gDAAgD;YAChD,IAAI,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC;gBAAE,SAAS;YAEpC,+CAA+C;YAC/C,MAAM,mBAAmB,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;YAClD,IAAI,CAAC,mBAAmB;gBAAE,SAAS;YAEnC,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,EAAE,OAAO,EAAE,cAAc,CAAC,CAAC;YACzD,IAAI,CAAC;gBACJ,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC;gBAE/D,yEAAyE;gBACzE,wEAAwE;gBACxE,0DAA0D;gBAC1D,IAAI,OAAO,CAAC,WAAW,KAAK,mBAAmB;oBAAE,SAAS;gBAC1D,IACC,OAAO,CAAC,yBAAyB;oBACjC,OAAO,CAAC,6BAA6B,KAAK,OAAO,CAAC,6BAA6B;oBAC9E,SAAS;gBAEX,yEAAyE;gBACzE,IAAI,OAAO,CAAC,KAAK,KAAK,KAAK;oBAAE,SAAS;gBAEtC,0BAA0B;gBAC1B,IAAI,OAAO,CAAC,KAAK,KAAK,IAAI;oBAAE,SAAS;gBAErC,kEAAkE;gBAClE,IAAI,iBAAiB,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;oBAAE,SAAS;gBAEtD,oEAAoE;gBACpE,IAAI,OAAO,CAAC,SAAS,IAAI,CAAC;oBAAE,SAAS;gBAErC,4EAA4E;gBAC5E,iFAAiF;gBACjF,IACC,OAAO,CAAC,UAAU,KAAK,CAAC;oBACxB,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,EAAE,qBAAqB,CAAC,CAAC;oBAExD,SAAS;gBAEV,QAAQ,CAAC,GAAG,CAAC,OAAO,EAAE;oBACrB,QAAQ,EAAE,OAAO;oBACjB,WAAW,EAAE,OAAO,CAAC,WAAW;oBAChC,QAAQ,EAAE,OAAO,CAAC,QAAQ;oBAC1B,SAAS;iBACT,CAAC,CAAC;YACJ,CAAC;YAAC,MAAM,CAAC;gBACR,yBAAyB;YAC1B,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,QAAQ,CAAC;AACjB,CAAC"}

package/dist/lib/runner.d.ts

@@ -4,7 +4,7 @@
  * 429 rate-limit errors are retried with exponential backoff.
  * With earlyExit, in-flight attempts are aborted when one passes.
  */
-import type { ResolvedExperimentConfig, EvalFixture, EvalRunData, ExperimentResults, RunnableExperimentConfig, ProgressEvent } from './types.js';
+import type { ResolvedExperimentConfig, Classification, EvalFixture, EvalRunData, ExperimentResults, RunnableExperimentConfig, ProgressEvent, Reporter } from './types.js';
 /**
  * Rate-limits how many operations can START within a time window.
  * Once started, operations run freely with no concurrency limit.
@@ -50,6 +50,18 @@ export interface RunExperimentOptions {
     smoke?: boolean;
     /** Shared rate limiter to control how many sandbox runs start per time window */
     rateLimiter?: StartRateLimiter;
+    /** Reporters to call in addition to `config.reporters`. @default undefined */
+    reporters?: Reporter[];
+    /** Evals the caller skipped because a cached result was reused. Listed in the
+     * reporters' experiment event; they get no run events. @default undefined */
+    reusedEvals?: string[];
+    /**
+     * Optional step that runs after results are saved and before reporters'
+     * `onExperimentComplete`. Classifications it returns are included in the
+     * reporter event. The CLI uses this for failure classification, which can
+     * remove non-model failures from the results directory. @default undefined
+     */
+    classify?: (results: ExperimentResults, outputDir: string) => Promise<Map<string, Classification> | void>;
 }
 /**
  * Run an experiment - execute all evals with configured runs concurrently.
@@ -76,5 +88,8 @@ export declare function runSingleEval<T extends ResolvedExperimentConfig['model'
     webResearch?: ResolvedExperimentConfig['webResearch'];
     disableBundledSkills?: ResolvedExperimentConfig['disableBundledSkills'];
     judge?: ResolvedExperimentConfig['judge'];
+    agentEnv?: ResolvedExperimentConfig['agentEnv'];
+    verifier?: ResolvedExperimentConfig['verifier'];
+    protectedPaths?: ResolvedExperimentConfig['protectedPaths'];
 }): Promise<T extends Array<unknown> ? EvalRunData[] : EvalRunData>;
 //# sourceMappingURL=runner.d.ts.map

package/dist/lib/runner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/lib/runner.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EACV,wBAAwB,EACxB,WAAW,EACX,WAAW,EAEX,iBAAiB,EACjB,wBAAwB,EACxB,aAAa,EACd,MAAM,YAAY,CAAC;AASpB;;;;GAIG;AACH,qBAAa,gBAAgB;IAMzB,OAAO,CAAC,QAAQ,CAAC,eAAe;IAChC,OAAO,CAAC,QAAQ,CAAC,QAAQ;IAN3B,OAAO,CAAC,KAAK,CAAsB;IACnC,OAAO,CAAC,OAAO,CAAK;IACpB,OAAO,CAAC,KAAK,CAA+C;gBAGzC,eAAe,EAAE,MAAM,EACvB,QAAQ,EAAE,MAAM;IAGnC;;;OAGG;IACG,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAWlC,OAAO,CAAC,WAAW;CAgBpB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,wCAAwC;IACxC,MAAM,EAAE,wBAAwB,CAAC;IACjC,sBAAsB;IACtB,QAAQ,EAAE,WAAW,EAAE,CAAC;IACxB,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,gCAAgC;IAChC,UAAU,EAAE,MAAM,CAAC;IACnB,sBAAsB;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,iEAAiE;IACjE,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtC,wEAAwE;IACxE,mBAAmB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7C,oCAAoC;IACpC,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,aAAa,KAAK,IAAI,CAAC;IAC5C,qCAAqC;IACrC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,uCAAuC;IACvC,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,iFAAiF;IACjF,WAAW,CAAC,EAAE,gBAAgB,CAAC;CAChC;AAoBD;;;GAGG;AACH,wBAAsB,aAAa,CACjC,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,iBAAiB,CAAC,CAsQ5B;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,CAAC,SAAS,wBAAwB,CAAC,OAAO,CAAC,EAC7E,OAAO,EAAE,WAAW,EACpB,OAAO,EAAE;IACP,KAAK,CAAC,EAAE,wBAAwB,CAAC,OAAO,CAAC,CAAC;IAC1C,KAAK,EAAE,CAAC,CAAC;IACT,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,wBAAwB,CAAC,OAAO,CAAC,CAAC;IAC1C,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,UAAU,CAAC,EAAE,wBAAwB,CAAC,YAAY,CAAC,CAAC;IACpD,OAAO,CAAC,EAAE,wBAAwB,CAAC,SAAS,CAAC,CAAC;IAC9C,YAAY,CAAC,EAAE,wBAAwB,CAAC,cAAc,CAAC,CAAC;IACxD,WAAW,CAAC,EAAE,wBAAwB,CAAC,aAAa,CAAC,CAAC;IACtD,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC;IACxC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,YAAY,CAAC,EAAE,wBAAwB,CAAC,cAAc,CAAC,CAAC;IACxD,WAAW,CAAC,EAAE,wBAAwB,CAAC,aAAa,CAAC,CAAC;IACtD,oBAAoB,CAAC,EAAE,wBAAwB,CAAC,sBAAsB,CAAC,CAAC;IACxE,KAAK,CAAC,EAAE,wBAAwB,CAAC,OAAO,CAAC,CAAC;CAC3C,GACA,OAAO,CAAC,CAAC,SAAS,KAAK,CAAC,OAAO,CAAC,GAAG,WAAW,EAAE,GAAG,WAAW,CAAC,CA8CjE"}
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/lib/runner.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EACV,wBAAwB,EACxB,cAAc,EACd,WAAW,EACX,WAAW,EAEX,iBAAiB,EACjB,wBAAwB,EACxB,aAAa,EACb,QAAQ,EACT,MAAM,YAAY,CAAC;AAgBpB;;;;GAIG;AACH,qBAAa,gBAAgB;IAMzB,OAAO,CAAC,QAAQ,CAAC,eAAe;IAChC,OAAO,CAAC,QAAQ,CAAC,QAAQ;IAN3B,OAAO,CAAC,KAAK,CAAsB;IACnC,OAAO,CAAC,OAAO,CAAK;IACpB,OAAO,CAAC,KAAK,CAA+C;gBAGzC,eAAe,EAAE,MAAM,EACvB,QAAQ,EAAE,MAAM;IAGnC;;;OAGG;IACG,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAWlC,OAAO,CAAC,WAAW;CAgBpB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,wCAAwC;IACxC,MAAM,EAAE,wBAAwB,CAAC;IACjC,sBAAsB;IACtB,QAAQ,EAAE,WAAW,EAAE,CAAC;IACxB,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,gCAAgC;IAChC,UAAU,EAAE,MAAM,CAAC;IACnB,sBAAsB;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,iEAAiE;IACjE,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtC,wEAAwE;IACxE,mBAAmB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7C,oCAAoC;IACpC,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,aAAa,KAAK,IAAI,CAAC;IAC5C,qCAAqC;IACrC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,uCAAuC;IACvC,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,iFAAiF;IACjF,WAAW,CAAC,EAAE,gBAAgB,CAAC;IAC/B,8EAA8E;IAC9E,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IACvB;iFAC6E;IAC7E,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,CACT,OAAO,EAAE,iBAAiB,EAC1B,SAAS,EAAE,MAAM,KACd,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,GAAG,IAAI,CAAC,CAAC;CAClD;AAoBD;;;GAGG;AACH,wBAAsB,aAAa,CACjC,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,iBAAiB,CAAC,CAwU5B;AA2BD;;GAEG;AACH,wBAAsB,aAAa,CAAC,CAAC,SAAS,wBAAwB,CAAC,OAAO,CAAC,EAC7E,OAAO,EAAE,WAAW,EACpB,OAAO,EAAE;IACP,KAAK,CAAC,EAAE,wBAAwB,CAAC,OAAO,CAAC,CAAC;IAC1C,KAAK,EAAE,CAAC,CAAC;IACT,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,wBAAwB,CAAC,OAAO,CAAC,CAAC;IAC1C,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,UAAU,CAAC,EAAE,wBAAwB,CAAC,YAAY,CAAC,CAAC;IACpD,OAAO,CAAC,EAAE,wBAAwB,CAAC,SAAS,CAAC,CAAC;IAC9C,YAAY,CAAC,EAAE,wBAAwB,CAAC,cAAc,CAAC,CAAC;IACxD,WAAW,CAAC,EAAE,wBAAwB,CAAC,aAAa,CAAC,CAAC;IACtD,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC;IACxC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,YAAY,CAAC,EAAE,wBAAwB,CAAC,cAAc,CAAC,CAAC;IACxD,WAAW,CAAC,EAAE,wBAAwB,CAAC,aAAa,CAAC,CAAC;IACtD,oBAAoB,CAAC,EAAE,wBAAwB,CAAC,sBAAsB,CAAC,CAAC;IACxE,KAAK,CAAC,EAAE,wBAAwB,CAAC,OAAO,CAAC,CAAC;IAC1C,QAAQ,CAAC,EAAE,wBAAwB,CAAC,UAAU,CAAC,CAAC;IAChD,QAAQ,CAAC,EAAE,wBAAwB,CAAC,UAAU,CAAC,CAAC;IAChD,cAAc,CAAC,EAAE,wBAAwB,CAAC,gBAAgB,CAAC,CAAC;CAC7D,GACA,OAAO,CAAC,CAAC,SAAS,KAAK,CAAC,OAAO,CAAC,GAAG,WAAW,EAAE,GAAG,WAAW,CAAC,CAkDjE"}

package/dist/lib/runner.js

@@ -4,7 +4,8 @@
  * 429 rate-limit errors are retried with exponential backoff.
  * With earlyExit, in-flight attempts are aborted when one passes.
  */
-import { assertRunRuntimeControls, getAgent } from './agents/index.js';
+import { assertRunRuntimeControls, getAgent, runCredentials } from './agents/index.js';
+import { formatReporterError, notifyExperimentComplete, notifyRunComplete, } from './reporters.js';
 import { agentResultToEvalRunData, createEvalSummary, createExperimentResults, saveResults, } from './results.js';
 /**
  * Rate-limits how many operations can START within a time window.
@@ -60,17 +61,40 @@ export class StartRateLimiter {
 export async function runExperiment(options) {
     const { config, fixtures, apiKey, resultsDir, experimentName, fingerprints, contentFingerprints, onProgress, smoke, rateLimiter } = options;
     const startedAt = new Date();
+    const reporters = [...(config.reporters ?? []), ...(options.reporters ?? [])];
     assertRunRuntimeControls(config.agent, {
         disableBundledSkills: config.disableBundledSkills,
         webResearch: config.webResearch,
     }, config.judge?.agent);
     // Get the agent from registry
     const agent = getAgent(config.agent);
+    // Transcript parser for reading token usage (falls back to the agent id, which
+    // is what saveResults parses transcripts with).
+    const o11yAgentName = agent.definition?.o11yAgentName ?? config.agent;
     const emit = (event) => {
         if (onProgress) {
             onProgress(event);
         }
     };
+    // Reporter payloads are redacted with the same credentials the agent's own
+    // result is (the codegen key and a pinned judge's key).
+    const reporterSecrets = reporters.length > 0 ? reporterCredentials(agent, config, apiKey) : [];
+    const onReporterFailure = (failure) => {
+        const event = {
+            type: 'reporter:error',
+            reporter: failure.reporter,
+            hook: failure.hook,
+            error: failure.error,
+            ...(failure.evalName !== undefined ? { evalName: failure.evalName } : {}),
+            ...(failure.runIndex !== undefined ? { runNumber: failure.runIndex + 1 } : {}),
+        };
+        // Never swallow a reporter failure silently: without a progress handler,
+        // log it directly.
+        if (onProgress)
+            emit(event);
+        else
+            console.warn(formatReporterError(event));
+    };
     // Create AbortController per fixture for earlyExit
     const abortControllers = new Map();
     for (const fixture of fixtures) {
@@ -128,6 +152,9 @@ export async function runExperiment(options) {
                 webResearch: config.webResearch,
                 disableBundledSkills: config.disableBundledSkills,
                 judge: config.judge,
+                agentEnv: config.agentEnv,
+                verifier: config.verifier,
+                protectedPaths: config.protectedPaths,
             }),
             new Promise((_, reject) => {
                 timeoutId = setTimeout(() => {
@@ -161,7 +188,7 @@ export async function runExperiment(options) {
                 aborted: true,
             };
         }
-        let runData = agentResultToEvalRunData(agentResult);
+        let runData = agentResultToEvalRunData(agentResult, { o11yAgentName });
         if (config.onRunComplete) {
             try {
                 const updatedRunData = await config.onRunComplete({
@@ -226,6 +253,23 @@ export async function runExperiment(options) {
                 emit({ type: 'experiment:earlyExit', evalName: attempt.fixture.name, runNumber: attempt.runIndex + 1 });
                 abortControllers.get(attempt.fixture.name).abort();
             }
+            // Reporters see the final attempt only (after any retry) and the result
+            // after the user's onRunComplete hook, which ran inside runAttempt.
+            if (reporters.length > 0) {
+                const fingerprint = fingerprints?.[attempt.fixture.name];
+                const contentFingerprint = contentFingerprints?.[attempt.fixture.name];
+                await notifyRunComplete(reporters, {
+                    schemaVersion: 1,
+                    experimentName,
+                    evalName: attempt.fixture.name,
+                    runIndex: attempt.runIndex,
+                    agent: config.agent,
+                    model: config.model,
+                    result: result.runData.result,
+                    ...(fingerprint ? { fingerprint } : {}),
+                    ...(contentFingerprint ? { contentFingerprint } : {}),
+                }, { secrets: reporterSecrets, onFailure: onReporterFailure });
+            }
         }
         return result;
     };
@@ -274,8 +318,42 @@ export async function runExperiment(options) {
     });
     emit({ type: 'experiment:saved', outputDir });
     emit({ type: 'experiment:summary', results: experimentResults });
+    const classifications = options.classify
+        ? await options.classify(experimentResults, outputDir)
+        : undefined;
+    if (reporters.length > 0) {
+        await notifyExperimentComplete(reporters, {
+            schemaVersion: 1,
+            experimentName,
+            outputDir,
+            results: experimentResults,
+            reused: [...(options.reusedEvals ?? [])],
+            ...(classifications ? { classifications: Object.fromEntries(classifications) } : {}),
+        }, { secrets: reporterSecrets, onFailure: onReporterFailure });
+    }
     return experimentResults;
 }
+/** The credentials a run of this experiment injects, for redacting reporter payloads. */
+function reporterCredentials(agent, config, apiKey) {
+    // Without a definition there is no judge to resolve; the codegen key and the
+    // agentEnv values are still known.
+    if (!agent.definition)
+        return [apiKey, ...Object.values(config.agentEnv ?? {})];
+    const modelPolicy = config.modelPolicy ?? 'agent-default';
+    const options = {
+        prompt: '',
+        model: modelPolicy === 'native-default' ? undefined : config.model,
+        modelPolicy,
+        timeout: config.timeout * 1000,
+        apiKey,
+        agentOptions: config.agentOptions,
+        webResearch: config.webResearch,
+        disableBundledSkills: config.disableBundledSkills,
+        judge: config.judge,
+        agentEnv: config.agentEnv,
+    };
+    return runCredentials(agent.definition, options);
+}
 /**
  * Run a single eval (for testing/debugging).
  */
@@ -286,6 +364,7 @@ export async function runSingleEval(fixture, options) {
         webResearch: options.webResearch,
     }, options.judge?.agent);
     const agent = getAgent(agentName);
+    const o11yAgentName = agent.definition?.o11yAgentName ?? agentName;
     const models = Array.isArray(options.model) ? options.model : [options.model];
     const prompt = options.editPrompt ? options.editPrompt(fixture.prompt) : fixture.prompt;
     const results = [];
@@ -306,8 +385,11 @@ export async function runSingleEval(fixture, options) {
             webResearch: options.webResearch,
             disableBundledSkills: options.disableBundledSkills,
             judge: options.judge,
+            agentEnv: options.agentEnv,
+            verifier: options.verifier,
+            protectedPaths: options.protectedPaths,
         });
-        results.push(agentResultToEvalRunData(agentResult));
+        results.push(agentResultToEvalRunData(agentResult, { o11yAgentName }));
     }
     // TODO: remove this on the next major and return an array directly...it's just here to prevent breaking changes
     if (!Array.isArray(options.model)) {