@inbrowser/agent 0.1.0 → 0.3.0

package/dist/eval/index.js

@@ -0,0 +1,7 @@
+export { SKILL_NAMES, applyWorkspaceOverrides, parseFixture, validateFixture } from './fixture.js';
+export { defaultSystemPromptBuilder, runFixture, runFixtures } from './runner.js';
+export { createSpecRegistry, evaluateSpec } from './spec-framework.js';
+export { aggregateTrials, collectMetrics, extractTrialMetrics } from './metric-collector.js';
+export { POLARITY, compareMetrics, renderJson, renderMarkdown } from './comparison-report.js';
+export { CUSTOM_SPEC_NAMES, SPEC_FINAL_RULES_EXCLUDES_LITERAL, SPEC_FINAL_RULES_INCLUDES_LITERAL, SPEC_FINAL_RUNTIME_RUN_SUMMARY_OK, SPEC_GAME_RULES_SIMULATOR_ACCEPTS_POSITIVE_AND_REJECTS_CHEAT, SPEC_PYRIC_AGENTS_LINT_CLEAN_AND_RULE_REJECTS_CHEAT, SPEC_REPORT_MENTIONS_ALL_OF, SPEC_REPORT_MENTIONS_AT_LEAST_ONE_OF, SPEC_TRACE_CONTAINS_TOOL_CALL_BY_NAME, STARTER_SPEC_NAMES, finalRulesExcludesLiteral, finalRulesIncludesLiteral, finalRuntimeRunSummaryOk, gameRulesSimulatorAcceptsPositiveAndRejectsCheat, pyricAgentsLintCleanAndRuleRejectsCheat, registerAllSpecs, registerCustomSpecs, registerStarterSpecs, reportMentionsAllOf, reportMentionsAtLeastOneOf, traceContainsToolCallByName, } from './spec-helpers.js';
+//# sourceMappingURL=index.js.map

package/dist/eval/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,WAAW,EAAE,uBAAuB,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAQnG,OAAO,EAAE,0BAA0B,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAGlF,OAAO,EAAE,kBAAkB,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AASvE,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAW7F,OAAO,EAAE,QAAQ,EAAE,cAAc,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAE9F,OAAO,EACL,iBAAiB,EACjB,iCAAiC,EACjC,iCAAiC,EACjC,iCAAiC,EACjC,4DAA4D,EAC5D,mDAAmD,EACnD,2BAA2B,EAC3B,oCAAoC,EACpC,qCAAqC,EACrC,kBAAkB,EAClB,yBAAyB,EACzB,yBAAyB,EACzB,wBAAwB,EACxB,gDAAgD,EAChD,uCAAuC,EACvC,gBAAgB,EAChB,mBAAmB,EACnB,oBAAoB,EACpB,mBAAmB,EACnB,0BAA0B,EAC1B,2BAA2B,GAC5B,MAAM,mBAAmB,CAAC"}

package/dist/eval/load-node.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Node-only fixture file/directory loader.
+ *
+ * Browser-safe parsing and validation live in `./fixture.ts`. This
+ * module wraps those with `node:fs` reads. Imported by consumers via
+ * `@inbrowser/agent/node`, not the universal entry.
+ */
+import { type TaskFixture, type ValidationError } from './fixture.js';
+export declare class FixtureLoadError extends Error {
+    readonly file: string;
+    readonly errors: ValidationError[];
+    constructor(file: string, errors: ValidationError[]);
+}
+export declare function loadFixture(filePath: string): TaskFixture;
+export declare function loadFixtures(dirPath: string): TaskFixture[];
+//# sourceMappingURL=load-node.d.ts.map

package/dist/eval/load-node.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"load-node.d.ts","sourceRoot":"","sources":["../../src/eval/load-node.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAKH,OAAO,EAAE,KAAK,WAAW,EAAE,KAAK,eAAe,EAAgB,MAAM,cAAc,CAAC;AAEpF,qBAAa,gBAAiB,SAAQ,KAAK;IACzC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,eAAe,EAAE,CAAC;gBAEvB,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,eAAe,EAAE;CAOpD;AAED,wBAAgB,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,WAAW,CAOzD;AAED,wBAAgB,YAAY,CAAC,OAAO,EAAE,MAAM,GAAG,WAAW,EAAE,CA8B3D"}

package/dist/eval/load-node.js

@@ -0,0 +1,58 @@
+/**
+ * Node-only fixture file/directory loader.
+ *
+ * Browser-safe parsing and validation live in `./fixture.ts`. This
+ * module wraps those with `node:fs` reads. Imported by consumers via
+ * `@inbrowser/agent/node`, not the universal entry.
+ */
+import { readFileSync, readdirSync, statSync } from 'node:fs';
+import { join } from 'node:path';
+import { parseFixture } from './fixture.js';
+export class FixtureLoadError extends Error {
+    file;
+    errors;
+    constructor(file, errors) {
+        const summary = errors.map((e) => `  - ${e.path ? `${e.path}: ` : ''}${e.message}`).join('\n');
+        super(`fixture "${file}" failed validation:\n${summary}`);
+        this.name = 'FixtureLoadError';
+        this.file = file;
+        this.errors = errors;
+    }
+}
+export function loadFixture(filePath) {
+    const json = readFileSync(filePath, 'utf8');
+    const result = parseFixture(json);
+    if (!result.ok) {
+        throw new FixtureLoadError(filePath, result.errors);
+    }
+    return result.fixture;
+}
+export function loadFixtures(dirPath) {
+    const fixtures = [];
+    const failures = [];
+    for (const entry of readdirSync(dirPath)) {
+        if (!entry.endsWith('.fixture.json'))
+            continue;
+        const full = join(dirPath, entry);
+        if (!statSync(full).isFile())
+            continue;
+        const json = readFileSync(full, 'utf8');
+        const result = parseFixture(json);
+        if (!result.ok) {
+            failures.push({ file: full, errors: result.errors });
+        }
+        else {
+            fixtures.push(result.fixture);
+        }
+    }
+    if (failures.length > 0) {
+        const summary = failures
+            .map((f) => `- ${f.file}:\n${f.errors
+            .map((e) => `    ${e.path ? `${e.path}: ` : ''}${e.message}`)
+            .join('\n')}`)
+            .join('\n');
+        throw new Error(`one or more fixtures failed validation:\n${summary}`);
+    }
+    return fixtures.sort((a, b) => a.id.localeCompare(b.id));
+}
+//# sourceMappingURL=load-node.js.map

package/dist/eval/load-node.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"load-node.js","sourceRoot":"","sources":["../../src/eval/load-node.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAC9D,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAA0C,YAAY,EAAE,MAAM,cAAc,CAAC;AAEpF,MAAM,OAAO,gBAAiB,SAAQ,KAAK;IAChC,IAAI,CAAS;IACb,MAAM,CAAoB;IAEnC,YAAY,IAAY,EAAE,MAAyB;QACjD,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/F,KAAK,CAAC,YAAY,IAAI,yBAAyB,OAAO,EAAE,CAAC,CAAC;QAC1D,IAAI,CAAC,IAAI,GAAG,kBAAkB,CAAC;QAC/B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;CACF;AAED,MAAM,UAAU,WAAW,CAAC,QAAgB;IAC1C,MAAM,IAAI,GAAG,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC5C,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IAClC,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;QACf,MAAM,IAAI,gBAAgB,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;IACtD,CAAC;IACD,OAAO,MAAM,CAAC,OAAO,CAAC;AACxB,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,OAAe;IAC1C,MAAM,QAAQ,GAAkB,EAAE,CAAC;IACnC,MAAM,QAAQ,GAAkD,EAAE,CAAC;IAEnE,KAAK,MAAM,KAAK,IAAI,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;QACzC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,eAAe,CAAC;YAAE,SAAS;QAC/C,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QAClC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE;YAAE,SAAS;QACvC,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACxC,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;QAClC,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;YACf,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QACvD,CAAC;aAAM,CAAC;YACN,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;IAED,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,OAAO,GAAG,QAAQ;aACrB,GAAG,CACF,CAAC,CAAC,EAAE,EAAE,CACJ,KAAK,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,MAAM;aACtB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,OAAO,EAAE,CAAC;aAC5D,IAAI,CAAC,IAAI,CAAC,EAAE,CAClB;aACA,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,MAAM,IAAI,KAAK,CAAC,4CAA4C,OAAO,EAAE,CAAC,CAAC;IACzE,CAAC;IAED,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC"}

package/dist/eval/metric-collector.d.ts

@@ -0,0 +1,209 @@
+/**
+ * `collectMetrics` — the eval harness's metric extractor + aggregator.
+ *
+ * Bridges raw `RunRecord`s (the per-trial captures the runner produces)
+ * and the comparison report. Given an array of records, an optional
+ * parallel array of spec evaluations, and a tool registry to classify
+ * tool calls, the collector returns one `MetricsTable` per fixture.
+ * Each table carries:
+ *
+ *   - a row per trial (`TrialMetrics`) with the eight phase-one metrics,
+ *   - one aggregated row (`AggregatedMetrics`) summarising mean and
+ *     N-1 standard deviation across trials.
+ *
+ * The eight metrics are extracted exactly as the implementation plan
+ * specifies:
+ *
+ *   1. `taskSuccess`        — pulled from the supplied `SpecResult.ok`.
+ *   2. `wallClockMs`        — `completedAt - startedAt`.
+ *   3. `promptTokens`       — sum of `usage.promptTokens` across all
+ *                             `llm_response` events that carry usage.
+ *   4. `completionTokens`   — sum of `usage.outputTokens` across the
+ *                             same events.
+ *   5. `toolCallCount`      — total tool calls across `llm_response`
+ *                             events, split into `reads` vs
+ *                             `mutations` by the `parallelSafe` tag.
+ *                             Names not registered in the supplied
+ *                             tool registry count as mutations.
+ *   6. `turnCount`          — count of distinct `requestId`s in the
+ *                             trace.
+ *   7. `peakContextWindowBytes`     — max of `JSON.stringify(messages).length`
+ *                                     across `llm_request` events.
+ *   8. `truthfulnessViolationRate`  — `analyzeTruthfulness(trace).violationRate`.
+ *   9. `dispatchVsLlmRatio` — sum of `dispatchMs` / sum of `llmMs`
+ *                             across rows from `turnTimingTable(trace)`.
+ *                             `undefined` when either total is zero.
+ *
+ * Numeric metrics with no data resolve to `undefined`, not `0`, so a
+ * downstream consumer can distinguish "no data" from "really zero".
+ *
+ * Aggregation: mean is the arithmetic average across trials that have
+ * a defined value for the metric; `undefined` when no trial has data.
+ * Spread is the sample standard deviation (N-1 denominator). A single
+ * defined value yields `stdDev: 0`. `taskSuccess` aggregates as a
+ * success rate — booleans are cast to `0`/`1` before averaging.
+ *
+ * No comparison logic, no persistence. The comparison report is a
+ * separate branch (`eval/comparison-report`).
+ *
+ * Browser-safe — no Node imports, no provider-specific code.
+ *
+ * Note on naming: there are two `RunRecord` types in this package.
+ * This collector consumes the eval-harness one defined at
+ * `./run-record.js`. The package root re-exports it as
+ * `EvalRunRecord` so it does not collide with the unrelated
+ * per-MCP-tool-call `RunRecord` at `../metrics/runs.js`.
+ */
+import type { ToolRegistry } from '../types/tools.js';
+import type { RunRecord } from './run-record.js';
+import type { SpecResult } from './spec-framework.js';
+/**
+ * Per-trial metric row. One per `RunRecord` consumed. Numeric metrics
+ * are `undefined` when the trial produced no data for them (e.g. no
+ * `llm_response.usage` events → `promptTokens: undefined`). `taskSuccess`
+ * is `undefined` when the caller passed no spec evaluation for the
+ * trial.
+ */
+export interface TrialMetrics {
+    /** Echoed from `record.fixture.id`. */
+    fixtureId: string;
+    /** Echoed from `record.trial`. */
+    trial: number;
+    /** Spec verdict for this trial. `undefined` when no evaluation was
+     *  supplied or the supplied evaluation was `undefined`. */
+    taskSuccess: boolean | undefined;
+    /** `completedAt - startedAt` from the record. Always defined. */
+    wallClockMs: number;
+    /** Sum of `usage.promptTokens` across `llm_response` events that
+     *  carry usage. `undefined` when no such event was emitted. */
+    promptTokens: number | undefined;
+    /** Sum of `usage.outputTokens` across `llm_response` events that
+     *  carry usage. `undefined` when no such event was emitted. */
+    completionTokens: number | undefined;
+    /** Total tool calls + read/mutation split. `total` is `undefined`
+     *  when no `llm_response` event carried any tool calls (a no-tool
+     *  run is not the same as a run that emitted zero tool calls
+     *  unintentionally — but at the extraction layer both look the
+     *  same; downstream can decide). */
+    toolCallCount: {
+        total: number | undefined;
+        reads: number | undefined;
+        mutations: number | undefined;
+    };
+    /** Count of distinct `requestId` values across the trace.
+     *  `undefined` when the trace contains no LLM events. */
+    turnCount: number | undefined;
+    /** Max of `JSON.stringify(messages).length` across the trace's
+     *  `llm_request` events. `undefined` when no such event exists. */
+    peakContextWindowBytes: number | undefined;
+    /** `analyzeTruthfulness(trace).violationRate`. `undefined` when
+     *  the trace contains no assistant turns (i.e. nothing to score). */
+    truthfulnessViolationRate: number | undefined;
+    /** Sum-of-dispatchMs divided by sum-of-llmMs across turn-timing
+     *  rows. `undefined` when either sum is zero. */
+    dispatchVsLlmRatio: number | undefined;
+}
+/**
+ * Aggregate of one numeric column across the trials of a fixture.
+ *
+ * `mean` is the arithmetic average across trials that had a defined
+ * value for the column. `stdDev` is the sample (N-1) standard
+ * deviation across the same trials. Both fields are `undefined` when
+ * no trial had data for the column. A single defined value yields
+ * `mean` equal to that value and `stdDev: 0`.
+ *
+ * `count` reports how many trials contributed a defined value, which
+ * a downstream report needs to weight or warn about thin samples.
+ */
+export interface AggregateStat {
+    mean: number | undefined;
+    stdDev: number | undefined;
+    count: number;
+}
+/**
+ * One row per fixture summarising mean + spread across its trials.
+ * `taskSuccessRate` is the mean of booleans cast to `0`/`1`. The
+ * remaining columns are sample-stat aggregates of the numeric trial
+ * metrics. Read/mutation totals follow the same shape as their
+ * per-trial counterpart.
+ */
+export interface AggregatedMetrics {
+    /** Echoed from the fixture id. */
+    fixtureId: string;
+    /** Number of trials contributing to this row. */
+    trials: number;
+    taskSuccessRate: AggregateStat;
+    wallClockMs: AggregateStat;
+    promptTokens: AggregateStat;
+    completionTokens: AggregateStat;
+    toolCallCount: {
+        total: AggregateStat;
+        reads: AggregateStat;
+        mutations: AggregateStat;
+    };
+    turnCount: AggregateStat;
+    peakContextWindowBytes: AggregateStat;
+    truthfulnessViolationRate: AggregateStat;
+    dispatchVsLlmRatio: AggregateStat;
+}
+/**
+ * One fixture's per-trial rows plus its aggregated row. The
+ * comparison report consumes a pair of these (baseline vs variant)
+ * and decides whether the variant moved the needle.
+ */
+export interface MetricsTable {
+    fixtureId: string;
+    trials: TrialMetrics[];
+    aggregate: AggregatedMetrics;
+}
+/**
+ * Input to `collectMetrics`. `evaluations` is positionally parallel
+ * to `records` — index `i` of `evaluations` is the spec result for
+ * `records[i]`. A missing entry (either the array is shorter or the
+ * slot is `undefined`) leaves `taskSuccess` undefined for that trial.
+ *
+ * `toolRegistry` is consulted to classify each emitted tool call as a
+ * read (parallel-safe) or a mutation (not parallel-safe). Tools whose
+ * name is not registered count as mutations.
+ */
+export interface CollectMetricsInput {
+    /** The per-trial captures from `runFixture` / `runFixtures`. Order
+     *  is preserved in the returned tables. */
+    records: readonly RunRecord[];
+    /** Parallel to `records`. Optional. `undefined` slots and a shorter
+     *  array both translate to `taskSuccess: undefined` on the row. */
+    evaluations?: readonly (SpecResult | undefined)[];
+    /** Source of truth for `parallelSafe` tags. The collector reads it
+     *  via `registry.list()` once; the returned handlers are scanned by
+     *  `name`. */
+    toolRegistry: ToolRegistry;
+}
+/**
+ * Compute one `MetricsTable` per fixture from a flat batch of
+ * `RunRecord`s. Records are grouped by `fixture.id` in first-seen
+ * order; within a group trials are kept in input order. The returned
+ * array preserves fixture order from the input.
+ *
+ * `evaluations` (when supplied) is consumed positionally — index `i`
+ * pairs with `records[i]`. A missing slot leaves the trial's
+ * `taskSuccess` undefined.
+ *
+ * Never throws on missing data: every metric extractor degrades to
+ * `undefined` rather than throwing. A malformed trace (e.g. an
+ * `llm_response` with no `usage`) just contributes nothing to the
+ * affected column.
+ */
+export declare function collectMetrics(input: CollectMetricsInput): MetricsTable[];
+/**
+ * Extract a single `TrialMetrics` row from a record + optional
+ * evaluation. Exported for tests that want to exercise the eight
+ * extractors against a hand-built record without going through the
+ * fixture-grouping layer.
+ */
+export declare function extractTrialMetrics(record: RunRecord, evaluation: SpecResult | undefined, toolRegistry: ToolRegistry): TrialMetrics;
+/**
+ * Aggregate a list of `TrialMetrics` for a single fixture. Exported
+ * for tests that want to exercise aggregation in isolation.
+ */
+export declare function aggregateTrials(fixtureId: string, trials: readonly TrialMetrics[]): AggregatedMetrics;
+//# sourceMappingURL=metric-collector.d.ts.map

package/dist/eval/metric-collector.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"metric-collector.d.ts","sourceRoot":"","sources":["../../src/eval/metric-collector.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuDG;AAKH,OAAO,KAAK,EAAe,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAEnE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEtD;;;;;;GAMG;AACH,MAAM,WAAW,YAAY;IAC3B,uCAAuC;IACvC,SAAS,EAAE,MAAM,CAAC;IAClB,kCAAkC;IAClC,KAAK,EAAE,MAAM,CAAC;IACd;+DAC2D;IAC3D,WAAW,EAAE,OAAO,GAAG,SAAS,CAAC;IACjC,iEAAiE;IACjE,WAAW,EAAE,MAAM,CAAC;IACpB;mEAC+D;IAC/D,YAAY,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC;mEAC+D;IAC/D,gBAAgB,EAAE,MAAM,GAAG,SAAS,CAAC;IACrC;;;;wCAIoC;IACpC,aAAa,EAAE;QACb,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC;QAC1B,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC;QAC1B,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;KAC/B,CAAC;IACF;6DACyD;IACzD,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B;uEACmE;IACnE,sBAAsB,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3C;yEACqE;IACrE,yBAAyB,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9C;qDACiD;IACjD,kBAAkB,EAAE,MAAM,GAAG,SAAS,CAAC;CACxC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;IACzB,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;;GAMG;AACH,MAAM,WAAW,iBAAiB;IAChC,kCAAkC;IAClC,SAAS,EAAE,MAAM,CAAC;IAClB,iDAAiD;IACjD,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,EAAE,aAAa,CAAC;IAC/B,WAAW,EAAE,aAAa,CAAC;IAC3B,YAAY,EAAE,aAAa,CAAC;IAC5B,gBAAgB,EAAE,aAAa,CAAC;IAChC,aAAa,EAAE;QACb,KAAK,EAAE,aAAa,CAAC;QACrB,KAAK,EAAE,aAAa,CAAC;QACrB,SAAS,EAAE,aAAa,CAAC;KAC1B,CAAC;IACF,SAAS,EAAE,aAAa,CAAC;IACzB,sBAAsB,EAAE,aAAa,CAAC;IACtC,yBAAyB,EAAE,aAAa,CAAC;IACzC,kBAAkB,EAAE,aAAa,CAAC;CACnC;AAED;;;;GAIG;AACH,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,YAAY,EAAE,CAAC;IACvB,SAAS,EAAE,iBAAiB,CAAC;CAC9B;AAED;;;;;;;;;GASG;AACH,MAAM,WAAW,mBAAmB;IAClC;+CAC2C;IAC3C,OAAO,EAAE,SAAS,SAAS,EAAE,CAAC;IAC9B;uEACmE;IACnE,WAAW,CAAC,EAAE,SAAS,CAAC,UAAU,GAAG,SAAS,CAAC,EAAE,CAAC;IAClD;;kBAEc;IACd,YAAY,EAAE,YAAY,CAAC;CAC5B;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,mBAAmB,GAAG,YAAY,EAAE,CAqCzE;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CACjC,MAAM,EAAE,SAAS,EACjB,UAAU,EAAE,UAAU,GAAG,SAAS,EAClC,YAAY,EAAE,YAAY,GACzB,YAAY,CAEd;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAC7B,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,SAAS,YAAY,EAAE,GAC9B,iBAAiB,CAoBnB"}

package/dist/eval/metric-collector.js

@@ -0,0 +1,293 @@
+/**
+ * `collectMetrics` — the eval harness's metric extractor + aggregator.
+ *
+ * Bridges raw `RunRecord`s (the per-trial captures the runner produces)
+ * and the comparison report. Given an array of records, an optional
+ * parallel array of spec evaluations, and a tool registry to classify
+ * tool calls, the collector returns one `MetricsTable` per fixture.
+ * Each table carries:
+ *
+ *   - a row per trial (`TrialMetrics`) with the eight phase-one metrics,
+ *   - one aggregated row (`AggregatedMetrics`) summarising mean and
+ *     N-1 standard deviation across trials.
+ *
+ * The eight metrics are extracted exactly as the implementation plan
+ * specifies:
+ *
+ *   1. `taskSuccess`        — pulled from the supplied `SpecResult.ok`.
+ *   2. `wallClockMs`        — `completedAt - startedAt`.
+ *   3. `promptTokens`       — sum of `usage.promptTokens` across all
+ *                             `llm_response` events that carry usage.
+ *   4. `completionTokens`   — sum of `usage.outputTokens` across the
+ *                             same events.
+ *   5. `toolCallCount`      — total tool calls across `llm_response`
+ *                             events, split into `reads` vs
+ *                             `mutations` by the `parallelSafe` tag.
+ *                             Names not registered in the supplied
+ *                             tool registry count as mutations.
+ *   6. `turnCount`          — count of distinct `requestId`s in the
+ *                             trace.
+ *   7. `peakContextWindowBytes`     — max of `JSON.stringify(messages).length`
+ *                                     across `llm_request` events.
+ *   8. `truthfulnessViolationRate`  — `analyzeTruthfulness(trace).violationRate`.
+ *   9. `dispatchVsLlmRatio` — sum of `dispatchMs` / sum of `llmMs`
+ *                             across rows from `turnTimingTable(trace)`.
+ *                             `undefined` when either total is zero.
+ *
+ * Numeric metrics with no data resolve to `undefined`, not `0`, so a
+ * downstream consumer can distinguish "no data" from "really zero".
+ *
+ * Aggregation: mean is the arithmetic average across trials that have
+ * a defined value for the metric; `undefined` when no trial has data.
+ * Spread is the sample standard deviation (N-1 denominator). A single
+ * defined value yields `stdDev: 0`. `taskSuccess` aggregates as a
+ * success rate — booleans are cast to `0`/`1` before averaging.
+ *
+ * No comparison logic, no persistence. The comparison report is a
+ * separate branch (`eval/comparison-report`).
+ *
+ * Browser-safe — no Node imports, no provider-specific code.
+ *
+ * Note on naming: there are two `RunRecord` types in this package.
+ * This collector consumes the eval-harness one defined at
+ * `./run-record.js`. The package root re-exports it as
+ * `EvalRunRecord` so it does not collide with the unrelated
+ * per-MCP-tool-call `RunRecord` at `../metrics/runs.js`.
+ */
+import { turnTimingTable } from '../diagnostics/timing.js';
+import { analyzeTruthfulness } from '../diagnostics/truthfulness.js';
+import { isParallelSafe } from '../tools.js';
+/**
+ * Compute one `MetricsTable` per fixture from a flat batch of
+ * `RunRecord`s. Records are grouped by `fixture.id` in first-seen
+ * order; within a group trials are kept in input order. The returned
+ * array preserves fixture order from the input.
+ *
+ * `evaluations` (when supplied) is consumed positionally — index `i`
+ * pairs with `records[i]`. A missing slot leaves the trial's
+ * `taskSuccess` undefined.
+ *
+ * Never throws on missing data: every metric extractor degrades to
+ * `undefined` rather than throwing. A malformed trace (e.g. an
+ * `llm_response` with no `usage`) just contributes nothing to the
+ * affected column.
+ */
+export function collectMetrics(input) {
+    const { records, evaluations, toolRegistry } = input;
+    const readNameSet = buildReadNameSet(toolRegistry);
+    // First-seen fixture order. We keep an ordered list of ids alongside
+    // a per-id bucket so the returned `MetricsTable[]` preserves the
+    // caller's fixture ordering rather than relying on `Map` iteration
+    // (which is insertion-ordered in v8 but explicit is clearer).
+    const order = [];
+    const buckets = new Map();
+    for (let i = 0; i < records.length; i++) {
+        const record = records[i];
+        if (!record)
+            continue;
+        const evaluation = evaluations?.[i];
+        const trial = extractTrial(record, evaluation, readNameSet);
+        const fixtureId = trial.fixtureId;
+        let bucket = buckets.get(fixtureId);
+        if (!bucket) {
+            bucket = [];
+            buckets.set(fixtureId, bucket);
+            order.push(fixtureId);
+        }
+        bucket.push(trial);
+    }
+    const tables = [];
+    for (const fixtureId of order) {
+        const trials = buckets.get(fixtureId);
+        if (!trials || trials.length === 0)
+            continue;
+        tables.push({
+            fixtureId,
+            trials,
+            aggregate: aggregateTrials(fixtureId, trials),
+        });
+    }
+    return tables;
+}
+/**
+ * Extract a single `TrialMetrics` row from a record + optional
+ * evaluation. Exported for tests that want to exercise the eight
+ * extractors against a hand-built record without going through the
+ * fixture-grouping layer.
+ */
+export function extractTrialMetrics(record, evaluation, toolRegistry) {
+    return extractTrial(record, evaluation, buildReadNameSet(toolRegistry));
+}
+/**
+ * Aggregate a list of `TrialMetrics` for a single fixture. Exported
+ * for tests that want to exercise aggregation in isolation.
+ */
+export function aggregateTrials(fixtureId, trials) {
+    return {
+        fixtureId,
+        trials: trials.length,
+        taskSuccessRate: aggregateSamples(trials.map((t) => (t.taskSuccess === undefined ? undefined : t.taskSuccess ? 1 : 0))),
+        wallClockMs: aggregateSamples(trials.map((t) => t.wallClockMs)),
+        promptTokens: aggregateSamples(trials.map((t) => t.promptTokens)),
+        completionTokens: aggregateSamples(trials.map((t) => t.completionTokens)),
+        toolCallCount: {
+            total: aggregateSamples(trials.map((t) => t.toolCallCount.total)),
+            reads: aggregateSamples(trials.map((t) => t.toolCallCount.reads)),
+            mutations: aggregateSamples(trials.map((t) => t.toolCallCount.mutations)),
+        },
+        turnCount: aggregateSamples(trials.map((t) => t.turnCount)),
+        peakContextWindowBytes: aggregateSamples(trials.map((t) => t.peakContextWindowBytes)),
+        truthfulnessViolationRate: aggregateSamples(trials.map((t) => t.truthfulnessViolationRate)),
+        dispatchVsLlmRatio: aggregateSamples(trials.map((t) => t.dispatchVsLlmRatio)),
+    };
+}
+// ---------- internals ----------
+function extractTrial(record, evaluation, readNameSet) {
+    const trace = record.trace;
+    const tokens = sumTokens(trace);
+    const toolCalls = countToolCalls(trace, readNameSet);
+    const turnCount = countDistinctRequestIds(trace);
+    const peakContextWindowBytes = peakContextBytes(trace);
+    const truthfulnessViolationRate = computeTruthfulness(trace);
+    const dispatchVsLlmRatio = computeDispatchVsLlmRatio(trace);
+    return {
+        fixtureId: record.fixture.id,
+        trial: record.trial,
+        taskSuccess: evaluation === undefined ? undefined : evaluation.ok,
+        wallClockMs: record.completedAt - record.startedAt,
+        promptTokens: tokens.promptTokens,
+        completionTokens: tokens.completionTokens,
+        toolCallCount: toolCalls,
+        turnCount,
+        peakContextWindowBytes,
+        truthfulnessViolationRate,
+        dispatchVsLlmRatio,
+    };
+}
+/** Build the set of registered tool names whose handler is
+ *  `parallelSafe`. A name absent from this set is treated as a
+ *  mutation (or unknown → mutation, per the brief). */
+function buildReadNameSet(registry) {
+    const reads = new Set();
+    const handlers = registry.list();
+    for (const h of handlers) {
+        if (isParallelSafe(h))
+            reads.add(h.name);
+    }
+    return reads;
+}
+function sumTokens(trace) {
+    let prompt;
+    let completion;
+    for (const ev of trace) {
+        if (ev.kind !== 'llm_response')
+            continue;
+        const usage = ev.data.usage;
+        if (!usage)
+            continue;
+        prompt = (prompt ?? 0) + (usage.promptTokens ?? 0);
+        completion = (completion ?? 0) + (usage.outputTokens ?? 0);
+    }
+    return { promptTokens: prompt, completionTokens: completion };
+}
+function countToolCalls(trace, readNameSet) {
+    let total = 0;
+    let reads = 0;
+    let mutations = 0;
+    let sawAny = false;
+    for (const ev of trace) {
+        if (ev.kind !== 'llm_response')
+            continue;
+        sawAny = true;
+        for (const call of ev.data.toolCalls) {
+            total += 1;
+            if (readNameSet.has(call.name))
+                reads += 1;
+            else
+                mutations += 1;
+        }
+    }
+    if (!sawAny) {
+        // No `llm_response` events at all — undefined is the right signal
+        // for "no data". Distinguishes a fully aborted trial from one
+        // that simply emitted zero tool calls.
+        return { total: undefined, reads: undefined, mutations: undefined };
+    }
+    return { total, reads, mutations };
+}
+function countDistinctRequestIds(trace) {
+    const ids = new Set();
+    for (const ev of trace) {
+        if (ev.kind === 'llm_request' ||
+            ev.kind === 'llm_response' ||
+            ev.kind === 'turn_dispatch_complete') {
+            ids.add(ev.data.requestId);
+        }
+    }
+    return ids.size === 0 ? undefined : ids.size;
+}
+function peakContextBytes(trace) {
+    let peak;
+    for (const ev of trace) {
+        if (ev.kind !== 'llm_request')
+            continue;
+        const bytes = JSON.stringify(ev.data.messages).length;
+        if (peak === undefined || bytes > peak)
+            peak = bytes;
+    }
+    return peak;
+}
+function computeTruthfulness(trace) {
+    // `analyzeTruthfulness` returns `violationRate: 0` when there are
+    // zero assistant turns to score. The brief says "no data → undefined,
+    // not zero", so we surface undefined in that degenerate case.
+    const report = analyzeTruthfulness(trace);
+    if (report.totalAssistantTurns === 0)
+        return undefined;
+    return report.violationRate;
+}
+function computeDispatchVsLlmRatio(trace) {
+    const rows = turnTimingTable(trace);
+    let llmSum = 0;
+    let dispatchSum = 0;
+    for (const row of rows) {
+        if (typeof row.llmMs === 'number')
+            llmSum += row.llmMs;
+        if (typeof row.dispatchMs === 'number')
+            dispatchSum += row.dispatchMs;
+    }
+    if (llmSum === 0 || dispatchSum === 0)
+        return undefined;
+    return dispatchSum / llmSum;
+}
+/**
+ * Aggregate an array of optional samples. Returns `mean` and `stdDev`
+ * across the defined entries. Sample standard deviation uses an N-1
+ * denominator; a single defined sample yields `stdDev: 0`. When zero
+ * samples are defined, both `mean` and `stdDev` are `undefined`.
+ */
+function aggregateSamples(samples) {
+    const defined = [];
+    for (const s of samples) {
+        if (typeof s === 'number' && Number.isFinite(s))
+            defined.push(s);
+    }
+    const count = defined.length;
+    if (count === 0)
+        return { mean: undefined, stdDev: undefined, count: 0 };
+    let sum = 0;
+    for (const v of defined)
+        sum += v;
+    const mean = sum / count;
+    if (count === 1)
+        return { mean, stdDev: 0, count };
+    let sqSum = 0;
+    for (const v of defined) {
+        const d = v - mean;
+        sqSum += d * d;
+    }
+    // N-1 denominator. Guaranteed `count >= 2` here.
+    const stdDev = Math.sqrt(sqSum / (count - 1));
+    return { mean, stdDev, count };
+}
+//# sourceMappingURL=metric-collector.js.map

package/dist/eval/metric-collector.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"metric-collector.js","sourceRoot":"","sources":["../../src/eval/metric-collector.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuDG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAC3D,OAAO,EAAE,mBAAmB,EAAE,MAAM,gCAAgC,CAAC;AACrE,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAoI7C;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,cAAc,CAAC,KAA0B;IACvD,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,GAAG,KAAK,CAAC;IACrD,MAAM,WAAW,GAAG,gBAAgB,CAAC,YAAY,CAAC,CAAC;IAEnD,qEAAqE;IACrE,iEAAiE;IACjE,mEAAmE;IACnE,8DAA8D;IAC9D,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,OAAO,GAAG,IAAI,GAAG,EAA0B,CAAC;IAElD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAC1B,IAAI,CAAC,MAAM;YAAE,SAAS;QACtB,MAAM,UAAU,GAAG,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC;QACpC,MAAM,KAAK,GAAG,YAAY,CAAC,MAAM,EAAE,UAAU,EAAE,WAAW,CAAC,CAAC;QAC5D,MAAM,SAAS,GAAG,KAAK,CAAC,SAAS,CAAC;QAClC,IAAI,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACpC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,GAAG,EAAE,CAAC;YACZ,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;YAC/B,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACxB,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACrB,CAAC;IAED,MAAM,MAAM,GAAmB,EAAE,CAAC;IAClC,KAAK,MAAM,SAAS,IAAI,KAAK,EAAE,CAAC;QAC9B,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACtC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAC7C,MAAM,CAAC,IAAI,CAAC;YACV,SAAS;YACT,MAAM;YACN,SAAS,EAAE,eAAe,CAAC,SAAS,EAAE,MAAM,CAAC;SAC9C,CAAC,CAAC;IACL,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,mBAAmB,CACjC,MAAiB,EACjB,UAAkC,EAClC,YAA0B;IAE1B,OAAO,YAAY,CAAC,MAAM,EAAE,UAAU,EAAE,gBAAgB,CAAC,YAAY,CAAC,CAAC,CAAC;AAC1E,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAC7B,SAAiB,EACjB,MAA+B;IAE/B,OAAO;QACL,SAAS;QACT,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,eAAe,EAAE,gBAAgB,CAC/B,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CACrF;QACD,WAAW,EAAE,gBAAgB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;QAC/D,YAAY,EAAE,gBAAgB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;QACjE,gBAAgB,EAAE,gBAAgB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC;QACzE,aAAa,EAAE;YACb,KAAK,EAAE,gBAAgB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;YACjE,KAAK,EAAE,gBAAgB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;YACjE,SAAS,EAAE,gBAAgB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;SAC1E;QACD,SAAS,EAAE,gBAAgB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAC3D,sBAAsB,EAAE,gBAAgB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC;QACrF,yBAAyB,EAAE,gBAAgB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,yBAAyB,CAAC,CAAC;QAC3F,kBAAkB,EAAE,gBAAgB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC;KAC9E,CAAC;AACJ,CAAC;AAED,kCAAkC;AAElC,SAAS,YAAY,CACnB,MAAiB,EACjB,UAAkC,EAClC,WAAgC;IAEhC,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;IAC3B,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;IAChC,MAAM,SAAS,GAAG,cAAc,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;IACrD,MAAM,SAAS,GAAG,uBAAuB,CAAC,KAAK,CAAC,CAAC;IACjD,MAAM,sBAAsB,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;IACvD,MAAM,yBAAyB,GAAG,mBAAmB,CAAC,KAAK,CAAC,CAAC;IAC7D,MAAM,kBAAkB,GAAG,yBAAyB,CAAC,KAAK,CAAC,CAAC;IAE5D,OAAO;QACL,SAAS,EAAE,MAAM,CAAC,OAAO,CAAC,EAAE;QAC5B,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,WAAW,EAAE,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,EAAE;QACjE,WAAW,EAAE,MAAM,CAAC,WAAW,GAAG,MAAM,CAAC,SAAS;QAClD,YAAY,EAAE,MAAM,CAAC,YAAY;QACjC,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;QACzC,aAAa,EAAE,SAAS;QACxB,SAAS;QACT,sBAAsB;QACtB,yBAAyB;QACzB,kBAAkB;KACnB,CAAC;AACJ,CAAC;AAED;;uDAEuD;AACvD,SAAS,gBAAgB,CAAC,QAAsB;IAC9C,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAChC,MAAM,QAAQ,GAAkB,QAAQ,CAAC,IAAI,EAAE,CAAC;IAChD,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,IAAI,cAAc,CAAC,CAAC,CAAC;YAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC3C,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,SAAS,CAAC,KAA4B;IAI7C,IAAI,MAA0B,CAAC;IAC/B,IAAI,UAA8B,CAAC;IACnC,KAAK,MAAM,EAAE,IAAI,KAAK,EAAE,CAAC;QACvB,IAAI,EAAE,CAAC,IAAI,KAAK,cAAc;YAAE,SAAS;QACzC,MAAM,KAAK,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC;QAC5B,IAAI,CAAC,KAAK;YAAE,SAAS;QACrB,MAAM,GAAG,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,YAAY,IAAI,CAAC,CAAC,CAAC;QACnD,UAAU,GAAG,CAAC,UAAU,IAAI,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,YAAY,IAAI,CAAC,CAAC,CAAC;IAC7D,CAAC;IACD,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,gBAAgB,EAAE,UAAU,EAAE,CAAC;AAChE,CAAC;AAED,SAAS,cAAc,CACrB,KAA4B,EAC5B,WAAgC;IAEhC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,MAAM,GAAG,KAAK,CAAC;IACnB,KAAK,MAAM,EAAE,IAAI,KAAK,EAAE,CAAC;QACvB,IAAI,EAAE,CAAC,IAAI,KAAK,cAAc;YAAE,SAAS;QACzC,MAAM,GAAG,IAAI,CAAC;QACd,KAAK,MAAM,IAAI,IAAI,EAAE,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACrC,KAAK,IAAI,CAAC,CAAC;YACX,IAAI,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC;gBAAE,KAAK,IAAI,CAAC,CAAC;;gBACtC,SAAS,IAAI,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IACD,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,kEAAkE;QAClE,8DAA8D;QAC9D,uCAAuC;QACvC,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC;IACtE,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;AACrC,CAAC;AAED,SAAS,uBAAuB,CAAC,KAA4B;IAC3D,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAC;IAC9B,KAAK,MAAM,EAAE,IAAI,KAAK,EAAE,CAAC;QACvB,IACE,EAAE,CAAC,IAAI,KAAK,aAAa;YACzB,EAAE,CAAC,IAAI,KAAK,cAAc;YAC1B,EAAE,CAAC,IAAI,KAAK,wBAAwB,EACpC,CAAC;YACD,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC7B,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC;AAC/C,CAAC;AAED,SAAS,gBAAgB,CAAC,KAA4B;IACpD,IAAI,IAAwB,CAAC;IAC7B,KAAK,MAAM,EAAE,IAAI,KAAK,EAAE,CAAC;QACvB,IAAI,EAAE,CAAC,IAAI,KAAK,aAAa;YAAE,SAAS;QACxC,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;QACtD,IAAI,IAAI,KAAK,SAAS,IAAI,KAAK,GAAG,IAAI;YAAE,IAAI,GAAG,KAAK,CAAC;IACvD,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,mBAAmB,CAAC,KAA4B;IACvD,kEAAkE;IAClE,sEAAsE;IACtE,8DAA8D;IAC9D,MAAM,MAAM,GAAG,mBAAmB,CAAC,KAAK,CAAC,CAAC;IAC1C,IAAI,MAAM,CAAC,mBAAmB,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IACvD,OAAO,MAAM,CAAC,aAAa,CAAC;AAC9B,CAAC;AAED,SAAS,yBAAyB,CAAC,KAA4B;IAC7D,MAAM,IAAI,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;IACpC,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ;YAAE,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC;QACvD,IAAI,OAAO,GAAG,CAAC,UAAU,KAAK,QAAQ;YAAE,WAAW,IAAI,GAAG,CAAC,UAAU,CAAC;IACxE,CAAC;IACD,IAAI,MAAM,KAAK,CAAC,IAAI,WAAW,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IACxD,OAAO,WAAW,GAAG,MAAM,CAAC;AAC9B,CAAC;AAED;;;;;GAKG;AACH,SAAS,gBAAgB,CAAC,OAAwC;IAChE,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;YAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACnE,CAAC;IACD,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC;IAC7B,IAAI,KAAK,KAAK,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;IACzE,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,MAAM,CAAC,IAAI,OAAO;QAAE,GAAG,IAAI,CAAC,CAAC;IAClC,MAAM,IAAI,GAAG,GAAG,GAAG,KAAK,CAAC;IACzB,IAAI,KAAK,KAAK,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC;IACnD,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC;QACnB,KAAK,IAAI,CAAC,GAAG,CAAC,CAAC;IACjB,CAAC;IACD,iDAAiD;IACjD,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;IAC9C,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;AACjC,CAAC"}