@leostera/evalkit 0.0.2 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1 +0,0 @@
1
- {"version":3,"file":"index-BwNYw4-_.d.mts","names":[],"sources":["../../core/src/identity.ts","../../core/src/matrix.ts","../../core/src/config.ts","../../core/src/schema.ts","../../core/src/trajectory.ts","../../core/src/index.ts"],"mappings":";;cAEa,oBAAkB,OAAA;KACnB,sBAAsB,mBAAmB;;iBAGrC,SAAS;;cAOZ,YAAU,OAAA,cAAA,OAAA;KAKX,cAAc,WAAW;KAEzB,YAAY,cAAc,eAAe,2BACxC;cAEA,mBAAiB,OAAA,cAAA,OAAA;iBAMd,kBAAkB,cAAc,cAAc,MAAM,QAAK,OAAA,cAAA,OAAA;iBAWzD,YAAY,cAAc,cACxC,MAAM,OACN,MAAM,OACL,YAAY;iBAIC,iBAAiB,cAAc,cAC7C,eACA,eAAe;EACZ,MAAM;EAAO,MAAM;EAAM,KAAK,YAAY;;iBAY/B,aAAa,eAAe,eAAe,eAAe;;;;iBCtD1D,oBAAoB,OAAO;KAqB/B;;EAEV;;EAEA,aAAa,SAAS,wBAAwB;;EAE9C,YAAY;;KAEF;EACV;;EAEA;EACA,MAAM;EACN,YAAY;;KAEF;EACV;EACA;EACA,gBAAgB;EAChB,YAAY,SAAS,wBAAwB;EAC7C,WAAW;;EAEX,iBAAiB;;EAEjB,mBAAmB;;KAET,aAAa;EACvB;EACA,MAAM,YAAY;EAClB,MAAM,YAAY,kBAAkB,iBAAiB;;iBAWvC,uBAAuB,UAAU,sBAC/C,YAAY,IACX,aAAa;;;KCpEJ;;EAEV;EACA;EACA;;EAEA,iBAAiB;EACjB,WAAW;;EAEX,SAAS,KAAK;IAA0C;;EACxD;IAAc;IAAsB;IAAiB;;EACrD;EACA;;;iBAIc,mBAAmB,UAAU,eAC3C,eAAe,IACd;;;;cChBU,iBAAe,OAAA;cAOf,qBAAmB,OAAA;;;;;cAMnB,mBAAiB,OAAA;;;;;;cAQjB,iBAAiB,OAAO,OAAO;cAoB/B,mBAAiB,OAAA;;;;;;;;;;;;;;;;;;cAWjB,qBAAmB,OAAA;;;;;;;;;;;;;;;;;;;;cAanB,mBAAiB,OAAA;;;;;;;;;;;;;;;;;;;cAIjB,qBAAmB,OAAA;;;;;;;;;;;;;;;;;;;;cAKnB,aAAW,OAAA;;;;;cAMX,oBAAkB,OAAA;;;;;;;;;;;;;;cAalB,mBAAiB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;cAWjB,wBAAsB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;cAgBtB,oBAAkB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;cASlB,qBAAmB,OAAA;;;;;cAKnB,oBAAkB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;cASlB,kBAAgB,OAAA;;;;;;;;;;;;;cAgJhB,uBAAuB,OAAO,OAAO;cAGrC,gBAAc,OAAA;;;;;;;iBClSX,qBAAqB,mBAAmB;;;KCF5C;KACA,YACV,gBAAgB;GAAiB,cAAc;;KACrC;GAAgB,cAAc;;KAE9B;;EAEV;EACA;EACA;EACA;;KAGU;;;;;KAMA;EACV;EACA,gBAAgB;;KAGN,gBAAgB,QAAQ,OAAO,kBAAkB;KAEjD;EACV;EACA;EACA;;KAGU;EACV;EACA;EACA;;KAGU;EACN;EAAiB;;EAEjB;EACA;EACA,SAAS;EACT;;EAGA;EACA;EACA;EACA,WAAW;EACX;;EAGA;EACA;EACA;EACA,QAAQ;EACR;;EAEA;EAAsB;EAAc;;EAEpC;EACA;EACA;EACA;EACA,QAAQ;;EAER;EAAmB,SAAS;EAAW;;EACvC;EAAe,OAAO;EAAe;;KAE/B;EACN;EAAuB;;EACvB;EAAiC;EAAc;;EAC/C;EAAmC;EAAc;;EACjD;EAAiC;EAAc;;EAE/C;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;;EAGA;EACA;EACA,OAAO;EACP;;EAEA;EAAwB;EAAgB;;EAExC;EACA;EACA;EACA;;EAGA;EACA;EACA,OAAO;EACP;;EAEA;EAAyB;;EACzB;EAAe,OAAO;EAAe;;KAE/B,mBACT;EAAa;MAAoB;EAAgB;;KAExC;;EAEV;;KAGU,qBAAqB;KACrB,qBAAqB;KAErB;EACV;EACA;EACA;EACA;EACA,UAAU;;EAEV,aAAa;EACb,UAAU;;EAEV,WAAW;;EAEX,oBAAoB;;KAGV,iBAAiB,KAC3B;KAIU,WAAW;WACZ,WAAW;;WAEX,WAAW;EACpB,MAAM;IACJ,SAAS;IACT,UAAU;IACV,UAAU,OAAO,oBAAoB;MACnC,QAAQ,WAAW;;KAGb,WAAW;;EAErB,KAAK,SAAS,WAAW;EACzB,SAAS;;;;;;iBAOK,kBAAkB,UAAU,YAAY,OAAO,IAAI;KAKvD;KAEA;EACV;EACA;EACA;EACA,YAAY;;KAGF;EACV;EACA;EACA;EACA,YAAY;;KAGF;EACV;EACA;EACA;EACA,YAAY;;KAGF;EACV;EACA,OACE,SAAS,iBACR,UAAU,YAAY,QAAQ,UAAU;;KAGjC,UACV,mBAAmB,cAAc,gBAAgB;;;;;iBAenC,UACd,aACA,UAAS,QAAQ,KAAK,qCACrB;iBASa,KACd,aACA,SAAS,KAAK,+BACb;iBAIa,WACd,cACA,cACA,YAAY,oBACX;iBAIa,QAAQ,QAAQ,2BAA2B;KAI/C;EACV;EACA;;KAGU;EACV;EACA,aAAa;;KAGH;EACV;EACA,kBAAkB;IAAa;;EAC/B,6BAA6B,QAAQ;IAAY;;IAC/C;IACA;;EAEF;IACE;IACA;IACA;IACA,WAAW;IACX;IACA,SAAS;;EAEX;;KAGU;EACV;EACA;EACA;IAAY;IAAc,YAAY;;;KAE5B,iBACV,WAAW,YAAY,cAAc;;iBAGvB,eACd,UAAU,iCACT;iBAMa,KAAK,kBAAkB;iBAIvB,MAAM,aAAa,aAAa;KAIpC;EACV,QAAQ;EACR,QAAQ;;EAER,SAAS;;KAGC;EAGN;EACA;EACA;EACA,WAAW;;EAEX,QAAQ;;KAGF;EACV;EACA;EACA;EACA;EACA;EACA,WAAW;EACX,QAAQ;EACR;EACA,QAAQ;;KAGE;EACV,WAAW;EACX,WAAW;;KAGD;EACV,SAAS;EACT;IAAc,iBAAiB;;EAC/B,WAAW;;EAEX,OAAO;;KAGG;EACV;EACA;EACA;EACA,IACE,SAAS,2BACE,aAAa,kBAAkB;;KAGlC;EACV;EACA;EACA;;EAEA;;KAGU,cAAc,kBAAkB;iBAE5B,UACd,cACA,KAAK,wBACL,UAAS,KAAK,sCACb;iBAIa,MACd,cACA,SAAS,KAAK,2CACb;KAMS;EACV;;EAEA;;EAEA;;KAGU,eAAe,eAAe,aAAa;EACrD;EACA;EACA,OAAO;;EAEP,QAAQ;EACR,WAAW;EACX,YAAY;EACZ,SAAS;EACT,SAAS;EACT,WAAW;;iBAGG,iBAAiB,UAAU,gBAAgB,YAAY,IAAI;KAuB/D,UACV,wBAAwB,4BAA4B;EAEpD;EACA;EACA,OAAO;;iBAGO,kBAAkB,eAAe,WAC/C,OAAO,SACN;KAKS,mBAAmB,iBAAiB,YAAY;iBAE5C,YAAY;EAAS;;KAkBzB;;EAEV,gBAAgB;EAChB,iBAAiB;EACjB,mBAAmB;EACnB,IAAI,cAAc;EAClB,SAAS,cAAc;EACvB,YAAY;EACZ,iBAAiB;EACjB,WAAW;;KAGD;EACV;EACA;EACA;;KAGU;EACV;EACA;EACA;;KAGU;EACV,MAAM;EACN;EACA;EACA,aAAa;;KAGH;EACV;EACA;EACA;EACA;EACA;IACE;IACA;IACA;IACA;IACA,UAAU;MAAQ,MAAM;MAAkB;;;EAE5C,UAAU;EACV;EACA,SAAS;IAAQ;IAAc,MAAM;;;;;;;iBAOvB,oBACR,gCAAgC,oBACtC,eAAe,iBAAiB;KAyGtB;KAEA;EACV;EACA;EACA,QAAQ;EACR;EACA;EACA,aAAa;EACb;IAAW;IAAY;;EACvB,MAAM;EACN;;KAGU;EACV;EACA;EACA,QAAQ;EACR;EACA,UAAU;EACV;EACA;EACA,aAAa;EACb;IAAW;IAAY;;EACvB,MAAM;EACN;;KAGU;EACV;EACA;EACA;EACA;EACA;EACA;EACA;EACA,WAAW;EACX,QAAQ;EACR;EACA,QAAQ;EACR;IAAoB;IAAoB;;;KAG9B;EACV,SAAS;EACT,cAAc;EACd;EACA;EACA;;KAGU;;EAEV;EACA;EACA;;KAGU;EACV,QAAQ;EACR;EACA;EACA,UAAU;EACV,YAAY;EACZ,QAAQ;;KAGE;EACV,QAAQ;EACR;EACA;EACA;EACA;EACA;EACA,QAAQ;;KAGE;EACV,YAAY,OAAO,kBAAkB;EACrC,YAAY,SAAS,eAAe;;EAEpC,cAAc,cAAc,MAAM,aAAa;EAC/C,SAAS,SAAS,eAAe;;KAGvB;WACD;EACT,WAAW,UAAU,gBAAgB,QAAQ;EAC7C,SAAS,SAAS,aAAa;;KAGrB;EACV,SAAS,UAAU,cAAc,QAAQ;;KAG/B;EACV;EACA,QAAQ;EACR;EACA,UAAU;EACV;EACA;EACA,QAAQ;EACR;EACA;EACA,UAAU;EACV,QAAQ;;KAGE;EACV;EACA;EACA;EACA;;KAGU,YAAY;EACtB;EACA;EACA;EACA;EACA,mBAAmB;;EAEnB,SAAS;;iBAOK,YAAY,iBAAiB"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"src-CopGdrwX.mjs","names":[],"sources":["../../core/src/identity.ts","../../core/src/matrix.ts","../../core/src/schema.ts","../../core/src/trajectory.ts","../../core/src/index.ts"],"sourcesContent":["import * as Schema from 'effect/Schema';\n\nexport const ResourceKindSchema = Schema.Literal('run', 'trial', 'artifact');\nexport type ResourceKind = typeof ResourceKindSchema.Type;\n\n/** Human-authored project-local identifier; never a generated resource URI. */\nexport function authorId(value: string): string {\n if (!/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(value))\n throw new Error(`ID must be lowercase kebab-case: ${value}`);\n return value;\n}\n\n/** RFC 4122 textual UUID. EvalKit accepts v4/v7 and other valid UUID versions. */\nexport const UuidSchema = Schema.String.pipe(\n Schema.pattern(\n /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i,\n ),\n);\nexport type Uuid = typeof UuidSchema.Type;\n\nexport type ResourceUri<TKind extends ResourceKind = ResourceKind> =\n `evalkit:${TKind}:${string}`;\n\nexport const ResourceUriSchema = Schema.String.pipe(\n Schema.pattern(\n /^evalkit:(run|trial|artifact):[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i,\n ),\n);\n\nexport function resourceUriSchema<TKind extends ResourceKind>(kind: TKind) {\n return Schema.String.pipe(\n Schema.pattern(\n new RegExp(\n `^evalkit:${kind}:[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`,\n 'i',\n ),\n ),\n );\n}\n\nexport function resourceUri<TKind extends ResourceKind>(\n kind: TKind,\n uuid: Uuid,\n): ResourceUri<TKind> {\n return `evalkit:${kind}:${uuid}`;\n}\n\nexport function parseResourceUri<TKind extends ResourceKind>(\n value: string,\n expectedKind?: TKind,\n): { kind: TKind; uuid: Uuid; uri: ResourceUri<TKind> } {\n const match = /^evalkit:(run|trial|artifact):(.+)$/.exec(value);\n if (!match) throw new Error(`Invalid EvalKit resource URI: ${value}`);\n const kind = match[1] as TKind;\n if (expectedKind && kind !== expectedKind)\n throw new Error(\n `Expected an EvalKit ${expectedKind} URI; received ${value}`,\n );\n const uuid = Schema.decodeUnknownSync(UuidSchema)(match[2]) as Uuid;\n return { kind, uuid, uri: resourceUri(kind, uuid) };\n}\n\nexport function resourceUuid(value: string, expectedKind?: ResourceKind): Uuid {\n return parseResourceUri(value, expectedKind).uuid;\n}\n","import {\n authoringId,\n type EvalDefinition,\n type JsonObject,\n type JsonValue,\n} from './index.js';\n\n/** Canonical JSON: keys sorted recursively, array order preserved. Rejects non-JSON inputs. */\nexport function canonicalParameters(value: JsonValue): string {\n if (value === null || typeof value === 'string' || typeof value === 'boolean')\n return JSON.stringify(value);\n if (typeof value === 'number' && Number.isFinite(value))\n return JSON.stringify(value);\n if (Array.isArray(value))\n return `[${value.map(canonicalParameters).join(',')}]`;\n if (\n typeof value === 'object' &&\n Object.getPrototypeOf(value) === Object.prototype\n ) {\n return `{${Object.keys(value)\n .sort()\n .map(\n (key) => `${JSON.stringify(key)}:${canonicalParameters(value[key]!)}`,\n )\n .join(',')}}`;\n }\n throw new Error('Matrix parameters must be finite JSON values');\n}\n\nexport type MatrixSelection = {\n /** Base eval IDs. Unknown values fail before dispatch. */\n evals?: readonly string[];\n /** Select existing dimension values, rather than overwriting every cell with the same value. */\n parameters?: Readonly<Record<string, readonly JsonValue[]>>;\n /** Overrides for non-dimension execution settings (e.g. maxTokens). */\n overrides?: JsonObject;\n};\nexport type EvalMatrixCell = {\n matrixId: string;\n /** Stable canonical key including matrix, base eval and effective parameters. */\n key: string;\n eval: EvalDefinition;\n parameters: JsonObject;\n};\nexport type EvalMatrixDefinition = {\n id: string;\n name?: string;\n evals: readonly EvalDefinition[];\n parameters: Readonly<Record<string, readonly JsonValue[]>>;\n defaults?: JsonObject;\n /** Complete correlated assignments, crossed with independent parameters. */\n cases?: readonly JsonObject[];\n /** Partial patterns over declared dimensions; matching cells are never run. */\n exclude?: readonly JsonObject[];\n};\nexport type EvalMatrix = EvalMatrixDefinition & {\n kind: 'matrix';\n count(selection?: MatrixSelection): number;\n cells(selection?: MatrixSelection): IterableIterator<EvalMatrixCell>;\n};\n\nconst COUNT_BUDGET = 100_000;\nconst equal = (a: JsonValue, b: JsonValue) =>\n canonicalParameters(a) === canonicalParameters(b);\nconst matches = (pattern: JsonObject, values: JsonObject) =>\n Object.entries(pattern).every(\n ([key, value]) => Object.hasOwn(values, key) && equal(values[key]!, value),\n );\n\nexport function defineEvalMatrix<const T extends EvalMatrixDefinition>(\n definition: T,\n): EvalMatrix & T {\n authoringId(definition);\n const identities = new Set<string>();\n for (const evaluation of definition.evals) {\n const id = authoringId(evaluation);\n if (identities.has(id)) throw new Error(`Duplicate matrix eval: ${id}`);\n identities.add(id);\n }\n const axes = Object.entries(definition.parameters).sort(([a], [b]) =>\n a.localeCompare(b),\n );\n for (const [axis, choices] of axes) {\n if (!axis || !choices.length)\n throw new Error(`Matrix axis ${axis} must have at least one value`);\n const keys = choices.map(canonicalParameters);\n if (new Set(keys).size !== keys.length)\n throw new Error(`Duplicate values for matrix axis: ${axis}`);\n }\n canonicalParameters(definition.defaults ?? {});\n if (definition.cases && !definition.cases.length)\n throw new Error('Matrix cases must have at least one assignment');\n const cases = definition.cases ?? [{}];\n const caseKeys = Object.keys(cases[0]!).sort();\n const caseSignatures = new Set<string>();\n for (const assignment of cases) {\n canonicalParameters(assignment);\n if (Object.keys(assignment).sort().join('\\0') !== caseKeys.join('\\0'))\n throw new Error('Matrix cases must have the same dimensions');\n for (const key of caseKeys) {\n if (\n Object.hasOwn(definition.parameters, key) ||\n Object.hasOwn(definition.defaults ?? {}, key)\n )\n throw new Error(\n `Matrix case dimension conflicts with axis or default: ${key}`,\n );\n }\n const signature = canonicalParameters(assignment);\n if (caseSignatures.has(signature)) throw new Error('Duplicate matrix case');\n caseSignatures.add(signature);\n }\n const declared = new Set([...caseKeys, ...axes.map(([key]) => key)]);\n const exclusions = definition.exclude ?? [];\n for (const [index, pattern] of exclusions.entries()) {\n canonicalParameters(pattern);\n if (!Object.keys(pattern).length)\n throw new Error(`Matrix exclude[${index}] must specify a dimension`);\n for (const key of Object.keys(pattern))\n if (!declared.has(key))\n throw new Error(`Unknown matrix exclude dimension: ${key}`);\n if (\n !cases.some((assignment) =>\n Object.entries(pattern).every(\n ([key, value]) =>\n !Object.hasOwn(assignment, key) || equal(assignment[key]!, value),\n ),\n ) ||\n !axes.every(\n ([key, choices]) =>\n !Object.hasOwn(pattern, key) ||\n choices.some((choice) => equal(choice, pattern[key]!)),\n )\n )\n throw new Error(`Matrix exclude[${index}] matches no declared cell`);\n }\n const caseChoices = new Map(\n caseKeys.map((key) => [key, cases.map((assignment) => assignment[key]!)]),\n );\n\n function select(selection: MatrixSelection = {}) {\n for (const key of Object.keys(selection.parameters ?? {})) {\n if (!declared.has(key)) throw new Error(`Unknown matrix axis: ${key}`);\n }\n for (const key of Object.keys(selection.overrides ?? {})) {\n if (declared.has(key))\n throw new Error(`Select axis ${key}; do not override it`);\n }\n canonicalParameters(selection.overrides ?? {});\n for (const name of selection.evals ?? []) {\n if (!definition.evals.some((e) => e.id === name))\n throw new Error(`Unknown matrix eval: ${name}`);\n }\n const evals = definition.evals.filter(\n (e) => !selection.evals || selection.evals.includes(e.id),\n );\n const selected = new Map<string, Set<string>>();\n for (const [key, values] of Object.entries(selection.parameters ?? {})) {\n const available = definition.parameters[key] ?? caseChoices.get(key)!;\n const keys = new Set(values.map(canonicalParameters));\n for (const value of keys)\n if (!available.some((choice) => canonicalParameters(choice) === value))\n throw new Error(`Unknown ${key} value: ${value}`);\n selected.set(key, keys);\n }\n const filteredCases = cases.filter((assignment) =>\n caseKeys.every(\n (key) =>\n !selected.has(key) ||\n selected.get(key)!.has(canonicalParameters(assignment[key]!)),\n ),\n );\n const selectedAxes = axes.map(\n ([axis, choices]) =>\n [\n axis,\n choices.filter(\n (choice) =>\n !selected.has(axis) ||\n selected.get(axis)!.has(canonicalParameters(choice)),\n ),\n ] as const,\n );\n return {\n evals,\n cases: filteredCases,\n axes: selectedAxes,\n defaults: { ...definition.defaults, ...selection.overrides },\n };\n }\n\n /** Inclusion/exclusion counts sparse forbidden patterns without expanding the product. */\n function countCase(\n assignment: JsonObject,\n selectedAxes: readonly (readonly [string, readonly JsonValue[]])[],\n budget: { remaining: number },\n ): number {\n const patterns = exclusions\n .filter((pattern) =>\n Object.entries(pattern).every(\n ([key, value]) =>\n !Object.hasOwn(assignment, key) || equal(assignment[key]!, value),\n ),\n )\n .map(\n (pattern) =>\n Object.fromEntries(\n Object.entries(pattern).filter(\n ([key]) => !Object.hasOwn(assignment, key),\n ),\n ) as JsonObject,\n );\n const product = selectedAxes.reduce(\n (count, [, values]) => count * values.length,\n 1,\n );\n if (!patterns.length) return product;\n if (patterns.some((pattern) => !Object.keys(pattern).length)) return 0;\n function intersection(pattern: JsonObject): number {\n return selectedAxes.reduce(\n (count, [axis, choices]) =>\n count *\n (Object.hasOwn(pattern, axis)\n ? Number(choices.some((choice) => equal(choice, pattern[axis]!)))\n : choices.length),\n 1,\n );\n }\n let eligible = product;\n function visit(start: number, combined: JsonObject, sign: number): void {\n for (let index = start; index < patterns.length; index++) {\n if (--budget.remaining < 0)\n throw new Error(\n 'Exact matrix count exceeds planning budget; narrow the selection or exclusions',\n );\n const pattern = patterns[index]!;\n if (\n Object.entries(pattern).some(\n ([key, value]) =>\n Object.hasOwn(combined, key) && !equal(combined[key]!, value),\n )\n )\n continue;\n const merged = { ...combined, ...pattern };\n const matches = intersection(merged);\n if (!matches) continue;\n eligible += sign * matches;\n visit(index + 1, merged, -sign);\n }\n }\n visit(0, {}, -1);\n return eligible;\n }\n\n return {\n ...definition,\n kind: 'matrix',\n count(selection) {\n const selected = select(selection);\n const budget = { remaining: COUNT_BUDGET };\n const perEval = selected.cases.reduce(\n (count, assignment) =>\n count + countCase(assignment, selected.axes, budget),\n 0,\n );\n const count = perEval * selected.evals.length;\n if (!Number.isSafeInteger(count))\n throw new Error('Matrix exceeds safe cell count');\n return count;\n },\n *cells(selection) {\n const selected = select(selection);\n function* expand(\n index: number,\n values: JsonObject,\n ): Generator<JsonObject> {\n if (index === selected.axes.length) {\n yield values;\n return;\n }\n const [axis, choices] = selected.axes[index]!;\n for (const choice of choices)\n yield* expand(index + 1, { ...values, [axis]: choice });\n }\n for (const evaluation of selected.evals) {\n for (const assignment of selected.cases) {\n for (const dimensions of expand(0, assignment)) {\n if (exclusions.some((pattern) => matches(pattern, dimensions)))\n continue;\n const parameters = { ...selected.defaults, ...dimensions };\n yield {\n matrixId: definition.id,\n eval: evaluation,\n parameters,\n key: canonicalParameters({\n matrix: authoringId(definition),\n eval: authoringId(evaluation),\n parameters,\n }),\n };\n }\n }\n }\n },\n };\n}\n","import * as Schema from 'effect/Schema';\nimport { resourceUriSchema } from './identity.js';\nimport type { JsonValue, TrajectoryEvent } from './index.js';\n\n/** Runtime schemas for serialized EvalKit report and API boundaries. */\nexport const RunStatusSchema = Schema.Literal(\n 'running',\n 'completed',\n 'failed',\n 'cancelled',\n);\n\nexport const RecordedErrorSchema = Schema.Struct({\n name: Schema.String,\n message: Schema.String,\n stack: Schema.optional(Schema.String),\n});\n\nexport const AutIdentitySchema = Schema.Struct({\n name: Schema.optional(Schema.String),\n kind: Schema.String,\n id: Schema.String,\n version: Schema.optional(Schema.String),\n});\n\n// Payloads and scorer evidence are provider-neutral JSON, not benchmark-specific objects.\nexport const JsonValueSchema: Schema.Schema<JsonValue> = Schema.suspend(() =>\n Schema.Union(\n Schema.Null,\n Schema.Boolean,\n Schema.Number,\n Schema.String,\n Schema.mutable(Schema.Array(JsonValueSchema)),\n Schema.Record({ key: Schema.String, value: JsonValueSchema }),\n ),\n);\n\nconst ParametersSchema = Schema.Record({\n key: Schema.String,\n value: JsonValueSchema,\n});\nconst MatrixCellSchema = Schema.Struct({\n id: Schema.String,\n cellKey: Schema.String,\n});\n\nexport const RunMetadataSchema = Schema.Struct({\n schemaVersion: Schema.Literal(2, 3),\n runUri: resourceUriSchema('run'),\n evalId: Schema.String,\n suiteId: Schema.optional(Schema.String),\n parameters: Schema.optional(ParametersSchema),\n matrix: Schema.optional(MatrixCellSchema),\n aut: Schema.optional(AutIdentitySchema),\n startedAt: Schema.String,\n});\n\nexport const TrialMetadataSchema = Schema.Struct({\n schemaVersion: Schema.Literal(2, 3),\n runUri: resourceUriSchema('run'),\n trialUri: resourceUriSchema('trial'),\n trialIndex: Schema.Number,\n evalId: Schema.String,\n parameters: Schema.optional(ParametersSchema),\n matrix: Schema.optional(MatrixCellSchema),\n aut: Schema.optional(AutIdentitySchema),\n startedAt: Schema.String,\n});\n\n/** On-disk manifests include a lifecycle status; older finalized manifests may still say running. */\nexport const RunManifestSchema = Schema.Struct({\n ...RunMetadataSchema.fields,\n status: Schema.optional(RunStatusSchema),\n});\nexport const TrialManifestSchema = Schema.Struct({\n ...TrialMetadataSchema.fields,\n status: Schema.optional(RunStatusSchema),\n});\n\nexport const UsageSchema = Schema.Struct({\n inputTokens: Schema.optional(Schema.Number),\n outputTokens: Schema.optional(Schema.Number),\n totalTokens: Schema.optional(Schema.Number),\n});\n\nexport const JudgeRunInfoSchema = Schema.Struct({\n agent: Schema.optional(\n Schema.Struct({\n id: Schema.String,\n kind: Schema.String,\n name: Schema.optional(Schema.String),\n version: Schema.optional(Schema.String),\n }),\n ),\n usage: Schema.optional(UsageSchema),\n events: Schema.optional(Schema.mutable(Schema.Array(JsonValueSchema))),\n});\n\nexport const ScoreResultSchema = Schema.Struct({\n name: Schema.String,\n kind: Schema.Literal('predicate', 'judge'),\n value: Schema.optional(Schema.Number),\n passed: Schema.optional(Schema.Boolean),\n explanation: Schema.optional(Schema.String),\n evidence: Schema.optional(JsonValueSchema),\n judge: Schema.optional(JudgeRunInfoSchema),\n durationMs: Schema.Number,\n error: Schema.optional(RecordedErrorSchema),\n});\nexport const CheckpointResultSchema = Schema.Struct({\n step: Schema.Number,\n kind: Schema.Literal('predicate', 'judge', 'expect-tool-call'),\n name: Schema.String,\n status: Schema.Literal('passed', 'failed', 'error', 'skipped'),\n value: Schema.optional(Schema.Number),\n passed: Schema.optional(Schema.Boolean),\n explanation: Schema.optional(Schema.String),\n evidence: Schema.optional(JsonValueSchema),\n judge: Schema.optional(JudgeRunInfoSchema),\n durationMs: Schema.optional(Schema.Number),\n error: Schema.optional(RecordedErrorSchema),\n matchedToolCall: Schema.optional(\n Schema.Struct({ eventIndex: Schema.Number, id: Schema.String }),\n ),\n});\nexport const TrialScoringSchema = Schema.Struct({\n results: Schema.mutable(Schema.Array(ScoreResultSchema)),\n checkpoints: Schema.optional(\n Schema.mutable(Schema.Array(CheckpointResultSchema)),\n ),\n skippedScorers: Schema.optional(Schema.mutable(Schema.Array(Schema.String))),\n overall: Schema.optional(Schema.Number),\n passed: Schema.Boolean,\n});\nexport const ArtifactEntrySchema = Schema.Struct({\n path: Schema.String,\n kind: Schema.Literal('file', 'directory'),\n size: Schema.optional(Schema.Number),\n});\nexport const TrialSummarySchema = Schema.Struct({\n status: RunStatusSchema,\n endedAt: Schema.String,\n durationMs: Schema.optional(Schema.Number),\n scoring: Schema.optional(TrialScoringSchema),\n artifacts: Schema.optional(Schema.mutable(Schema.Array(ArtifactEntrySchema))),\n error: Schema.optional(RecordedErrorSchema),\n});\n\nexport const RunSummarySchema = Schema.Struct({\n status: RunStatusSchema,\n endedAt: Schema.String,\n durationMs: Schema.optional(Schema.Number),\n trialCount: Schema.Number,\n passed: Schema.Number,\n failed: Schema.Number,\n error: Schema.optional(RecordedErrorSchema),\n});\n\nconst autEvent = {\n started: Schema.Struct({\n source: Schema.Literal('aut'),\n kind: Schema.Literal('started'),\n timestamp: Schema.String,\n }),\n message: Schema.Struct({\n source: Schema.Literal('aut'),\n kind: Schema.Literal('message'),\n role: Schema.Literal('system', 'user', 'assistant', 'tool'),\n content: JsonValueSchema,\n timestamp: Schema.String,\n }),\n toolCall: Schema.Struct({\n source: Schema.Literal('aut'),\n kind: Schema.Literal('tool-call'),\n id: Schema.String,\n name: Schema.String,\n arguments: JsonValueSchema,\n timestamp: Schema.String,\n }),\n toolResult: Schema.Struct({\n source: Schema.Literal('aut'),\n kind: Schema.Literal('tool-result'),\n id: Schema.String,\n name: Schema.optional(Schema.String),\n result: JsonValueSchema,\n timestamp: Schema.String,\n }),\n turnStarted: Schema.Struct({\n source: Schema.Literal('aut'),\n kind: Schema.Literal('turn-started'),\n turn: Schema.Number,\n timestamp: Schema.String,\n }),\n turnCompleted: Schema.Struct({\n source: Schema.Literal('aut'),\n kind: Schema.Literal('turn-completed'),\n turn: Schema.Number,\n timestamp: Schema.String,\n latencyMs: Schema.optional(Schema.Number),\n usage: Schema.optional(UsageSchema),\n }),\n completed: Schema.Struct({\n source: Schema.Literal('aut'),\n kind: Schema.Literal('completed'),\n output: Schema.optional(JsonValueSchema),\n timestamp: Schema.String,\n }),\n error: Schema.Struct({\n source: Schema.Literal('aut'),\n kind: Schema.Literal('error'),\n error: RecordedErrorSchema,\n timestamp: Schema.String,\n }),\n};\nconst runnerEvent = {\n trialStarted: Schema.Struct({\n source: Schema.Literal('runner'),\n kind: Schema.Literal('trial-started'),\n timestamp: Schema.String,\n }),\n stepStarted: Schema.Struct({\n source: Schema.Literal('runner'),\n kind: Schema.Literal('transcript-step-started'),\n step: Schema.Number,\n timestamp: Schema.String,\n }),\n stepCompleted: Schema.Struct({\n source: Schema.Literal('runner'),\n kind: Schema.Literal('transcript-step-completed'),\n step: Schema.Number,\n timestamp: Schema.String,\n }),\n stepSkipped: Schema.Struct({\n source: Schema.Literal('runner'),\n kind: Schema.Literal('transcript-step-skipped'),\n step: Schema.Number,\n timestamp: Schema.String,\n }),\n checkpointStarted: Schema.Struct({\n source: Schema.Literal('runner'),\n kind: Schema.Literal('checkpoint-started'),\n step: Schema.Number,\n name: Schema.String,\n timestamp: Schema.String,\n }),\n checkpointCompleted: Schema.Struct({\n source: Schema.Literal('runner'),\n kind: Schema.Literal('checkpoint-completed'),\n step: Schema.Number,\n status: Schema.Literal('passed', 'failed'),\n timestamp: Schema.String,\n }),\n checkpointError: Schema.Struct({\n source: Schema.Literal('runner'),\n kind: Schema.Literal('checkpoint-error'),\n step: Schema.Number,\n error: RecordedErrorSchema,\n timestamp: Schema.String,\n }),\n scorerStarted: Schema.Struct({\n source: Schema.Literal('runner'),\n kind: Schema.Literal('scorer-started'),\n scorer: Schema.String,\n timestamp: Schema.String,\n }),\n scorerCompleted: Schema.Struct({\n source: Schema.Literal('runner'),\n kind: Schema.Literal('scorer-completed'),\n scorer: Schema.String,\n value: Schema.Number,\n timestamp: Schema.String,\n }),\n scorerFailed: Schema.Struct({\n source: Schema.Literal('runner'),\n kind: Schema.Literal('scorer-failed'),\n scorer: Schema.String,\n error: RecordedErrorSchema,\n timestamp: Schema.String,\n }),\n trialCompleted: Schema.Struct({\n source: Schema.Literal('runner'),\n kind: Schema.Literal('trial-completed'),\n timestamp: Schema.String,\n }),\n error: Schema.Struct({\n source: Schema.Literal('runner'),\n kind: Schema.Literal('error'),\n error: RecordedErrorSchema,\n timestamp: Schema.String,\n }),\n};\n\nexport const TrajectoryEventSchema: Schema.Schema<TrajectoryEvent> =\n Schema.Union(...Object.values(autEvent), ...Object.values(runnerEvent));\n\nexport const ApiErrorSchema = Schema.Struct({\n error: Schema.String,\n message: Schema.String,\n});\n","import * as Schema from 'effect/Schema';\nimport type { TrajectoryEvent } from './index.js';\nimport { TrajectoryEventSchema } from './schema.js';\n\n/** Decode one private report's JSONL trajectory; reject malformed/non-event lines with their line number. */\nexport function parseTrajectoryJsonl(contents: string): TrajectoryEvent[] {\n const events: TrajectoryEvent[] = [];\n for (const [index, line] of contents.split(/\\r?\\n/).entries()) {\n if (!line.trim()) continue;\n try {\n events.push(\n Schema.decodeUnknownSync(TrajectoryEventSchema)(JSON.parse(line)),\n );\n } catch (cause) {\n throw new Error(`Invalid trajectory JSONL at line ${index + 1}`, {\n cause,\n });\n }\n }\n return events;\n}\n","import { authorId, type ResourceUri } from './identity.js';\nexport { defineConfig, type EvalkitConfig } from './config.js';\n\nexport type JsonPrimitive = boolean | number | string | null;\nexport type JsonValue =\n JsonPrimitive | JsonValue[] | { [key: string]: JsonValue };\nexport type JsonObject = { [key: string]: JsonValue };\n\nexport type AutIdentity = {\n /** Human-readable logical agent name shown in catalogs and dashboards. */\n name?: string;\n kind: string;\n id: string;\n version?: string;\n};\n\nexport type AgentRuntimeName = 'local' | 'sandbox' | 'remote';\n\n/**\n * A serializable runtime requirement for an AUT. Concrete runtime packages own\n * the interpretation of `kind` and `configuration`.\n */\nexport type AgentRuntime = {\n kind: string;\n configuration?: JsonObject;\n};\n\nexport type AgentRuntimes = Partial<Record<AgentRuntimeName, AgentRuntime>>;\n\nexport type Usage = {\n inputTokens?: number;\n outputTokens?: number;\n totalTokens?: number;\n};\n\nexport type RecordedError = {\n name: string;\n message: string;\n stack?: string;\n};\n\nexport type AutEvent =\n | { kind: 'started'; timestamp: string }\n | {\n kind: 'message';\n role: 'system' | 'user' | 'assistant' | 'tool';\n content: JsonValue;\n timestamp: string;\n }\n | {\n kind: 'tool-call';\n id: string;\n name: string;\n arguments: JsonValue;\n timestamp: string;\n }\n | {\n kind: 'tool-result';\n id: string;\n name?: string;\n result: JsonValue;\n timestamp: string;\n }\n | { kind: 'turn-started'; turn: number; timestamp: string }\n | {\n kind: 'turn-completed';\n turn: number;\n timestamp: string;\n latencyMs?: number;\n usage?: Usage;\n }\n | { kind: 'completed'; output?: JsonValue; timestamp: string }\n | { kind: 'error'; error: RecordedError; timestamp: string };\n\nexport type RunnerEvent =\n | { kind: 'trial-started'; timestamp: string }\n | { kind: 'transcript-step-started'; step: number; timestamp: string }\n | { kind: 'transcript-step-completed'; step: number; timestamp: string }\n | { kind: 'transcript-step-skipped'; step: number; timestamp: string }\n | {\n kind: 'checkpoint-started';\n step: number;\n name: string;\n timestamp: string;\n }\n | {\n kind: 'checkpoint-completed';\n step: number;\n status: 'passed' | 'failed';\n timestamp: string;\n }\n | {\n kind: 'checkpoint-error';\n step: number;\n error: RecordedError;\n timestamp: string;\n }\n | { kind: 'scorer-started'; scorer: string; timestamp: string }\n | {\n kind: 'scorer-completed';\n scorer: string;\n value: number;\n timestamp: string;\n }\n | {\n kind: 'scorer-failed';\n scorer: string;\n error: RecordedError;\n timestamp: string;\n }\n | { kind: 'trial-completed'; timestamp: string }\n | { kind: 'error'; error: RecordedError; timestamp: string };\n\nexport type TrajectoryEvent =\n (AutEvent & { source: 'aut' }) | (RunnerEvent & { source: 'runner' });\n\nexport type WorkspaceView = {\n /** Absolute path to this trial-scoped workspace. Do not persist it in reports. */\n root: string;\n};\n\nexport type CandidateWorkspace = WorkspaceView;\nexport type EvaluatorWorkspace = WorkspaceView;\n\nexport type AutContext = {\n runId: string;\n evalId: string;\n trialId: string;\n trialIndex: number;\n metadata: JsonObject;\n /** Run-time parameters supplied by the eval definition or CLI invocation. */\n parameters?: JsonObject;\n runtime?: AgentRuntimeName;\n /** Contains only resources visible to the AUT/model. */\n workspace: CandidateWorkspace;\n /** Private evaluator capability for trusted AUT adapters; never expose this to the model. */\n evaluatorWorkspace: EvaluatorWorkspace;\n};\n\nexport type FixtureContext = Omit<\n AutContext,\n 'workspace' | 'evaluatorWorkspace'\n>;\n\nexport type AutAdapter<TMessage = string> = {\n readonly identity?: AutIdentity;\n /** Available execution environments for this AUT. */\n readonly runtimes?: AgentRuntimes;\n start(options: {\n context: AutContext;\n runtime?: AgentRuntime;\n onEvent: (event: AutEvent) => void | Promise<void>;\n }): Promise<AutSession<TMessage>>;\n};\n\nexport type AutSession<TMessage = string> = {\n /** Resolves once the AUT has completed the turn initiated by this message. */\n send(message: TMessage): Promise<void>;\n close(): Promise<void>;\n};\n\n/**\n * Defines an Agent Under Test without coupling it to a runtime or transport.\n * It preserves the adapter's concrete type for application-specific helpers.\n */\nexport function defineAgent<const T extends AutAdapter>(agent: T): T {\n if (agent.identity) authoringId(agent.identity);\n return agent;\n}\n\nexport type FixtureVisibility = 'candidate' | 'evaluator';\n\nexport type DirectoryFixture = {\n kind: 'directory';\n src: string;\n dst: string;\n visibility: FixtureVisibility;\n};\n\nexport type FileFixture = {\n kind: 'file';\n src: string;\n dst: string;\n visibility: FixtureVisibility;\n};\n\nexport type InlineFixture = {\n kind: 'inline';\n file: string;\n data: string;\n visibility: FixtureVisibility;\n};\n\nexport type DynamicFixture = {\n kind: 'dynamic';\n create(\n context: FixtureContext,\n ): Fixture | Fixture[] | Promise<Fixture | Fixture[]>;\n};\n\nexport type Fixture =\n DirectoryFixture | FileFixture | InlineFixture | DynamicFixture;\n\nfunction defaultFixtureDestination(src: string): string {\n const normalized = src.replace(/\\\\/g, '/').replace(/\\/+$/, '');\n const destination = normalized.slice(normalized.lastIndexOf('/') + 1);\n if (!destination || destination === '.' || destination === '..') {\n throw new Error(`Fixture source needs a basename: ${src}`);\n }\n return destination;\n}\n\n/**\n * Declares a directory relative to the invoking project's working directory.\n * The short form is candidate-visible and copies beneath the source basename.\n */\nexport function directory(\n src: string,\n options: Partial<Omit<DirectoryFixture, 'kind' | 'src'>> = {},\n): DirectoryFixture {\n return {\n kind: 'directory',\n src,\n dst: options.dst ?? defaultFixtureDestination(src),\n visibility: options.visibility ?? 'candidate',\n };\n}\n\nexport function file(\n src: string,\n options: Omit<FileFixture, 'kind' | 'src'>,\n): FileFixture {\n return { kind: 'file', src, ...options };\n}\n\nexport function inlineFile(\n file: string,\n data: string,\n visibility: FixtureVisibility,\n): InlineFixture {\n return { kind: 'inline', file, data, visibility };\n}\n\nexport function dynamic(create: DynamicFixture['create']): DynamicFixture {\n return { kind: 'dynamic', create };\n}\n\nexport type UserStep = {\n kind: 'user';\n message: string;\n};\n\nexport type AgentStep = {\n kind: 'agent';\n expectation: JsonObject;\n};\n\nexport type TurnView = {\n userStepIndex: number;\n events: readonly (AutEvent & { source: 'aut' })[];\n assistantMessages: readonly (Extract<AutEvent, { kind: 'message' }> & {\n role: 'assistant';\n source: 'aut';\n })[];\n toolCalls: readonly {\n eventIndex: number;\n id: string;\n name: string;\n arguments: JsonValue;\n resultObservation: 'observed' | 'absent' | 'ambiguous';\n result?: JsonValue;\n }[];\n lastAssistantText?: string;\n};\n\nexport type ExpectToolCallStep = {\n kind: 'expect-tool-call';\n name: string;\n expected: { name: string; arguments?: JsonValue };\n};\nexport type TranscriptStep =\n UserStep | AgentStep | ScoringRule | ExpectToolCallStep;\n\n/** Observes a tool call in the preceding turn; never invokes a tool. */\nexport function expectToolCall(\n expected: ExpectToolCallStep['expected'],\n): ExpectToolCallStep {\n if (!expected.name.trim())\n throw new Error('Expected tool name must not be empty');\n return { kind: 'expect-tool-call', name: expected.name, expected };\n}\n\nexport function user(message: string): UserStep {\n return { kind: 'user', message };\n}\n\nexport function agent(expectation: JsonObject): AgentStep {\n return { kind: 'agent', expectation };\n}\n\nexport type JudgeRunInfo = {\n agent?: AutIdentity;\n usage?: Usage;\n /** Judge-agent observations; never attributed to the AUT trajectory. */\n events?: JsonValue[];\n};\n\nexport type ScoreValue =\n | number\n | {\n value: number;\n passed?: boolean;\n explanation?: string;\n evidence?: JsonValue;\n /** Judge-agent evidence, never inferred from AUT usage. */\n judge?: JudgeRunInfo;\n };\n\nexport type ScoreResult = {\n name: string;\n kind: 'predicate' | 'judge';\n value?: number;\n passed?: boolean;\n explanation?: string;\n evidence?: JsonValue;\n judge?: JudgeRunInfo;\n durationMs: number;\n error?: RecordedError;\n};\n\nexport type ArtifactView = {\n candidate: CandidateWorkspace;\n evaluator: EvaluatorWorkspace;\n};\n\nexport type ScoringContext = {\n context: AutContext;\n trajectory: { events: readonly TrajectoryEvent[] };\n artifacts: ArtifactView;\n /** Most recent completed turn; absent if no user message has been sent. */\n turn?: TurnView;\n};\n\nexport type PredicateScorer = {\n kind: 'predicate';\n name: string;\n supportsPartial?: boolean;\n run(\n context: ScoringContext,\n ): boolean | ScoreValue | Promise<boolean | ScoreValue>;\n};\n\nexport type JudgeRule = {\n kind: 'judge';\n name: string;\n rubric: string;\n /** Run even when the scenario stopped early or execution failed. */\n supportsPartial?: boolean;\n};\n\nexport type ScoringRule = PredicateScorer | JudgeRule;\n\nexport function predicate(\n name: string,\n run: PredicateScorer['run'],\n options: Pick<PredicateScorer, 'supportsPartial'> = {},\n): PredicateScorer {\n return { kind: 'predicate', name, run, ...options };\n}\n\nexport function judge(\n name: string,\n options: Pick<JudgeRule, 'rubric' | 'supportsPartial'>,\n): JudgeRule {\n if (!name.trim() || !options.rubric.trim())\n throw new Error('Judge name and rubric must not be empty');\n return { kind: 'judge', name, ...options };\n}\n\nexport type EvalPolicy = {\n timeoutMs?: number;\n /** Stop later transcript steps after a failed checkpoint assertion. Default false. */\n failfast?: boolean;\n /** Number of independent trials requested by local and hosted runners. */\n trials?: number;\n};\n\nexport type EvalDefinition<TAgent extends AutAdapter = AutAdapter> = {\n id: string;\n name?: string;\n agent: TAgent;\n /** Separate judge agent; its model and tools are never inherited from the AUT. */\n judge?: AutAdapter;\n fixtures?: Fixture[];\n transcript: TranscriptStep[];\n scoring: ScoringRule[];\n policy?: EvalPolicy;\n metadata?: JsonObject;\n};\n\nexport function defineEval<const T extends EvalDefinition>(definition: T): T {\n authoringId(definition);\n let hasUser = false;\n for (const step of definition.transcript) {\n if (step.kind === 'user') hasUser = true;\n else if (step.kind !== 'agent' && !hasUser)\n throw new Error('Checkpoint requires a preceding user step');\n }\n if (\n (definition.transcript.some((step) => step.kind === 'judge') ||\n definition.scoring.some((rule) => rule.kind === 'judge')) &&\n !definition.judge\n )\n throw new Error('Judge rules require an eval judge agent');\n if (definition.judge && definition.judge === definition.agent)\n throw new Error('Judge agent must be distinct from the agent under test');\n return definition;\n}\n\nexport { defineEvalMatrix, canonicalParameters } from './matrix.js';\nexport type { EvalMatrix, EvalMatrixCell, MatrixSelection } from './matrix.js';\nimport type { EvalMatrix } from './matrix.js';\n\nexport type EvalSuite<\n TEvals extends readonly EvalDefinition[] = readonly EvalDefinition[],\n> = {\n id: string;\n name?: string;\n evals: TEvals;\n};\n\nexport function defineSuite<const TSuite extends EvalSuite>(\n suite: TSuite,\n): TSuite {\n authoringId(suite);\n return suite;\n}\n\nexport type EvalRegistration = EvalDefinition | EvalSuite | EvalMatrix;\n\nexport function authoringId(value: { id: string }): string {\n if ('uri' in value || 'uuid' in value || 'slug' in value)\n throw new Error('Authored resources use id, not uri, uuid, or slug');\n return authorId(value.id);\n}\n\nfunction isEvalMatrix(\n registration: EvalRegistration,\n): registration is EvalMatrix {\n return 'kind' in registration && registration.kind === 'matrix';\n}\n\nfunction isEvalSuite(\n registration: EvalRegistration,\n): registration is EvalSuite {\n return !isEvalMatrix(registration) && 'evals' in registration;\n}\n\nexport type EvalRegistry = {\n /** All registered evals, flattened from standalone entries and suites. */\n evals: readonly EvalDefinition[];\n suites: readonly EvalSuite[];\n matrices: readonly EvalMatrix[];\n get(uri: string): EvalDefinition | undefined;\n getSuite(uri: string): EvalSuite | undefined;\n metadata(): EvalRegistryMetadata[];\n suiteMetadata(): EvalSuiteMetadata[];\n catalog(): EvalCatalogEntry[];\n};\n\nexport type EvalRegistryMetadata = {\n id: string;\n name?: string;\n suiteId?: string;\n};\n\nexport type EvalSuiteMetadata = {\n id: string;\n name?: string;\n evalIds: string[];\n};\n\nexport type EvalCatalogFixture = {\n kind: Fixture['kind'];\n source: string;\n destination?: string;\n visibility?: FixtureVisibility;\n};\n\nexport type EvalCatalogEntry = {\n id: string;\n path: string;\n name?: string;\n suiteId?: string;\n agent: {\n name?: string;\n kind: string;\n id?: string;\n version?: string;\n runtimes: Array<{ name: AgentRuntimeName; kind: string }>;\n };\n fixtures: EvalCatalogFixture[];\n trialCount: number;\n scorers: Array<{ name: string; kind: ScoringRule['kind'] }>;\n};\n\n/**\n * Creates an explicit, statically imported eval registry. A suite is a\n * path-like grouping of evals; nesting is presentation derived from its ID.\n */\nexport function registerEvals<\n const TRegistrations extends readonly EvalRegistration[],\n>(registrations: TRegistrations): EvalRegistry {\n const byId = new Map<string, EvalDefinition>();\n const suiteById = new Map<string, EvalSuite>();\n const suiteIdByEvalId = new Map<string, EvalSuite>();\n const standalone: EvalDefinition[] = [];\n const matrices: EvalMatrix[] = [];\n\n for (const registration of registrations) {\n if (isEvalMatrix(registration)) {\n const id = authoringId(registration);\n if (matrices.some((matrix) => authoringId(matrix) === id))\n throw new Error(`Eval registry contains duplicate matrix ID: ${id}`);\n matrices.push(registration);\n continue;\n }\n const suite = isEvalSuite(registration) ? registration : undefined;\n if (suite) {\n const id = authoringId(suite);\n if (suiteById.has(id))\n throw new Error(`Eval registry contains duplicate suite ID: ${id}`);\n suiteById.set(id, suite);\n }\n const evaluations: readonly EvalDefinition[] = suite\n ? suite.evals\n : [registration as EvalDefinition];\n for (const evaluation of evaluations) {\n const id = authoringId(evaluation);\n if (byId.has(id))\n throw new Error(`Eval registry contains duplicate eval ID: ${id}`);\n byId.set(id, evaluation);\n if (suite) suiteIdByEvalId.set(id, suite);\n else standalone.push(evaluation);\n }\n }\n\n const evals = [\n ...standalone,\n ...Array.from(suiteById.values()).flatMap((suite) => suite.evals),\n ];\n return {\n evals,\n suites: [...suiteById.values()],\n matrices,\n get: (id) => byId.get(id),\n getSuite: (id) => suiteById.get(id),\n metadata: () =>\n evals.map((evaluation) => {\n const suite = suiteIdByEvalId.get(authoringId(evaluation));\n return {\n id: evaluation.id,\n ...(evaluation.name ? { name: evaluation.name } : {}),\n ...(suite ? { suiteId: suite.id } : {}),\n };\n }),\n suiteMetadata: () =>\n [...suiteById.values()].map((suite) => ({\n id: suite.id,\n evalIds: suite.evals.map(authoringId),\n ...(suite.name ? { name: suite.name } : {}),\n })),\n catalog: () =>\n evals.map((evaluation) => {\n const suite = suiteIdByEvalId.get(authoringId(evaluation));\n const identity = evaluation.agent.identity;\n return {\n id: evaluation.id,\n path: suite ? `${suite.id}#${evaluation.id}` : evaluation.id,\n ...(evaluation.name ? { name: evaluation.name } : {}),\n ...(suite ? { suiteId: suite.id } : {}),\n agent: {\n ...(identity?.name ? { name: identity.name } : {}),\n kind: identity?.kind ?? 'adapter',\n ...(identity?.id ? { id: identity.id } : {}),\n ...(identity?.version ? { version: identity.version } : {}),\n runtimes: Object.entries(evaluation.agent.runtimes ?? {}).map(\n ([name, runtime]) => ({\n name: name as AgentRuntimeName,\n kind: runtime.kind,\n }),\n ),\n },\n fixtures: (evaluation.fixtures ?? []).map((fixture) =>\n fixture.kind === 'dynamic'\n ? { kind: 'dynamic', source: 'dynamic' }\n : fixture.kind === 'inline'\n ? {\n kind: 'inline',\n source: fixture.file,\n destination: fixture.file,\n visibility: fixture.visibility,\n }\n : {\n kind: fixture.kind,\n source: fixture.src,\n destination: fixture.dst,\n visibility: fixture.visibility,\n },\n ),\n trialCount: evaluation.policy?.trials ?? 1,\n scorers: evaluation.scoring.map(({ name, kind }) => ({ name, kind })),\n };\n }),\n };\n}\n\nexport type RunStatus = 'running' | 'completed' | 'failed' | 'cancelled';\n\nexport type RunMetadata = {\n schemaVersion: 2 | 3;\n runId: string;\n runUri: ResourceUri<'run'>;\n evalId: string;\n suiteId?: string;\n parameters?: JsonObject;\n matrix?: { id: string; cellKey: string };\n aut?: AutIdentity;\n startedAt: string;\n};\n\nexport type TrialMetadata = {\n schemaVersion: 2 | 3;\n runId: string;\n runUri: ResourceUri<'run'>;\n trialId: string;\n trialUri: ResourceUri<'trial'>;\n trialIndex: number;\n evalId: string;\n parameters?: JsonObject;\n matrix?: { id: string; cellKey: string };\n aut?: AutIdentity;\n startedAt: string;\n};\n\nexport type CheckpointResult = {\n step: number;\n kind: 'predicate' | 'judge' | 'expect-tool-call';\n name: string;\n status: 'passed' | 'failed' | 'error' | 'skipped';\n value?: number;\n passed?: boolean;\n explanation?: string;\n evidence?: JsonValue;\n judge?: JudgeRunInfo;\n durationMs?: number;\n error?: RecordedError;\n matchedToolCall?: { eventIndex: number; id: string };\n};\n\nexport type TrialScoring = {\n results: ScoreResult[];\n checkpoints?: CheckpointResult[];\n skippedScorers?: string[];\n overall?: number;\n passed: boolean;\n};\n\nexport type ArtifactEntry = {\n /** Path relative to this trial's artifact root. */\n path: string;\n kind: 'file' | 'directory';\n size?: number;\n};\n\nexport type TrialSummary = {\n status: RunStatus;\n endedAt: string;\n durationMs?: number;\n scoring?: TrialScoring;\n artifacts?: ArtifactEntry[];\n error?: RecordedError;\n};\n\nexport type RunSummary = {\n status: RunStatus;\n endedAt: string;\n durationMs?: number;\n trialCount: number;\n passed: number;\n failed: number;\n error?: RecordedError;\n};\n\nexport type TrialWriter = {\n appendEvent(event: TrajectoryEvent): Promise<void>;\n writeScores(scoring: TrialScoring): Promise<void>;\n /** Writes a file below this trial's artifact root. */\n writeArtifact(path: string, data: Uint8Array): Promise<void>;\n finalize(summary: TrialSummary): Promise<void>;\n};\n\nexport type RunWriter = {\n readonly location: string;\n startTrial(metadata: TrialMetadata): Promise<TrialWriter>;\n finalize(summary: RunSummary): Promise<void>;\n};\n\nexport type ReportStore = {\n startRun(metadata: RunMetadata): Promise<RunWriter>;\n};\n\nexport type TrialResult = {\n runId: string;\n runUri: ResourceUri<'run'>;\n trialId: string;\n trialUri: ResourceUri<'trial'>;\n trialIndex: number;\n evalId: string;\n status: RunStatus;\n reportLocation: string;\n durationMs?: number;\n scoring?: TrialScoring;\n error?: RecordedError;\n};\n\nexport type AggregateScoring = {\n passed: number;\n failed: number;\n passRate: number;\n overall?: number;\n};\n\nexport type RunResult = TrialResult & {\n runId: string;\n trialCount?: number;\n passed?: number;\n failed?: number;\n aggregateScoring?: AggregateScoring;\n /** Present for an aggregate multi-trial run. */\n trials?: TrialResult[];\n};\n\nexport * from './identity.js';\nexport * from './schema.js';\nexport * from './trajectory.js';\n\nexport function recordError(error: unknown): RecordedError {\n if (error instanceof Error) {\n return {\n name: error.name,\n message: error.message,\n ...(error.stack ? { stack: error.stack } : {}),\n };\n }\n\n return { name: 'Error', message: String(error) };\n}\n"],"mappings":";;AAEA,MAAa,qBAAqB,OAAO,QAAQ,OAAO,SAAS,UAAU;;AAI3E,SAAgB,SAAS,OAAuB;CAC9C,IAAI,CAAC,6BAA6B,KAAK,KAAK,GAC1C,MAAM,IAAI,MAAM,oCAAoC,OAAO;CAC7D,OAAO;AACT;;AAGA,MAAa,aAAa,OAAO,OAAO,KACtC,OAAO,QACL,4EACF,CACF;AAMA,MAAa,oBAAoB,OAAO,OAAO,KAC7C,OAAO,QACL,yGACF,CACF;AAEA,SAAgB,kBAA8C,MAAa;CACzE,OAAO,OAAO,OAAO,KACnB,OAAO,QACL,IAAI,OACF,YAAY,KAAK,4EACjB,GACF,CACF,CACF;AACF;AAEA,SAAgB,YACd,MACA,MACoB;CACpB,OAAO,WAAW,KAAK,GAAG;AAC5B;AAEA,SAAgB,iBACd,OACA,cACsD;CACtD,MAAM,QAAQ,sCAAsC,KAAK,KAAK;CAC9D,IAAI,CAAC,OAAO,MAAM,IAAI,MAAM,iCAAiC,OAAO;CACpE,MAAM,OAAO,MAAM;CACnB,IAAI,gBAAgB,SAAS,cAC3B,MAAM,IAAI,MACR,uBAAuB,aAAa,iBAAiB,OACvD;CACF,MAAM,OAAO,OAAO,kBAAkB,UAAU,CAAC,CAAC,MAAM,EAAE;CAC1D,OAAO;EAAE;EAAM;EAAM,KAAK,YAAY,MAAM,IAAI;CAAE;AACpD;AAEA,SAAgB,aAAa,OAAe,cAAmC;CAC7E,OAAO,iBAAiB,OAAO,YAAY,CAAC,CAAC;AAC/C;;;;ACxDA,SAAgB,oBAAoB,OAA0B;CAC5D,IAAI,UAAU,QAAQ,OAAO,UAAU,YAAY,OAAO,UAAU,WAClE,OAAO,KAAK,UAAU,KAAK;CAC7B,IAAI,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK,GACpD,OAAO,KAAK,UAAU,KAAK;CAC7B,IAAI,MAAM,QAAQ,KAAK,GACrB,OAAO,IAAI,MAAM,IAAI,mBAAmB,CAAC,CAAC,KAAK,GAAG,EAAE;CACtD,IACE,OAAO,UAAU,YACjB,OAAO,eAAe,KAAK,MAAM,OAAO,WAExC,OAAO,IAAI,OAAO,KAAK,KAAK,CAAC,CAC1B,KAAK,CAAC,CACN,KACE,QAAQ,GAAG,KAAK,UAAU,GAAG,EAAE,GAAG,oBAAoB,MAAM,IAAK,GACpE,CAAC,CACA,KAAK,GAAG,EAAE;CAEf,MAAM,IAAI,MAAM,8CAA8C;AAChE;AAkCA,MAAM,eAAe;AACrB,MAAM,SAAS,GAAc,MAC3B,oBAAoB,CAAC,MAAM,oBAAoB,CAAC;AAClD,MAAM,WAAW,SAAqB,WACpC,OAAO,QAAQ,OAAO,CAAC,CAAC,OACrB,CAAC,KAAK,WAAW,OAAO,OAAO,QAAQ,GAAG,KAAK,MAAM,OAAO,MAAO,KAAK,CAC3E;AAEF,SAAgB,iBACd,YACgB;CAChB,YAAY,UAAU;CACtB,MAAM,6BAAa,IAAI,IAAY;CACnC,KAAK,MAAM,cAAc,WAAW,OAAO;EACzC,MAAM,KAAK,YAAY,UAAU;EACjC,IAAI,WAAW,IAAI,EAAE,GAAG,MAAM,IAAI,MAAM,0BAA0B,IAAI;EACtE,WAAW,IAAI,EAAE;CACnB;CACA,MAAM,OAAO,OAAO,QAAQ,WAAW,UAAU,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,OAC7D,EAAE,cAAc,CAAC,CACnB;CACA,KAAK,MAAM,CAAC,MAAM,YAAY,MAAM;EAClC,IAAI,CAAC,QAAQ,CAAC,QAAQ,QACpB,MAAM,IAAI,MAAM,eAAe,KAAK,8BAA8B;EACpE,MAAM,OAAO,QAAQ,IAAI,mBAAmB;EAC5C,IAAI,IAAI,IAAI,IAAI,CAAC,CAAC,SAAS,KAAK,QAC9B,MAAM,IAAI,MAAM,qCAAqC,MAAM;CAC/D;CACA,oBAAoB,WAAW,YAAY,CAAC,CAAC;CAC7C,IAAI,WAAW,SAAS,CAAC,WAAW,MAAM,QACxC,MAAM,IAAI,MAAM,gDAAgD;CAClE,MAAM,QAAQ,WAAW,SAAS,CAAC,CAAC,CAAC;CACrC,MAAM,WAAW,OAAO,KAAK,MAAM,EAAG,CAAC,CAAC,KAAK;CAC7C,MAAM,iCAAiB,IAAI,IAAY;CACvC,KAAK,MAAM,cAAc,OAAO;EAC9B,oBAAoB,UAAU;EAC9B,IAAI,OAAO,KAAK,UAAU,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,IAAI,MAAM,SAAS,KAAK,IAAI,GAClE,MAAM,IAAI,MAAM,4CAA4C;EAC9D,KAAK,MAAM,OAAO,UAChB,IACE,OAAO,OAAO,WAAW,YAAY,GAAG,KACxC,OAAO,OAAO,WAAW,YAAY,CAAC,GAAG,GAAG,GAE5C,MAAM,IAAI,MACR,yDAAyD,KAC3D;EAEJ,MAAM,YAAY,oBAAoB,UAAU;EAChD,IAAI,eAAe,IAAI,SAAS,GAAG,MAAM,IAAI,MAAM,uBAAuB;EAC1E,eAAe,IAAI,SAAS;CAC9B;CACA,MAAM,2BAAW,IAAI,IAAI,CAAC,GAAG,UAAU,GAAG,KAAK,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC;CACnE,MAAM,aAAa,WAAW,WAAW,CAAC;CAC1C,KAAK,MAAM,CAAC,OAAO,YAAY,WAAW,QAAQ,GAAG;EACnD,oBAAoB,OAAO;EAC3B,IAAI,CAAC,OAAO,KAAK,OAAO,CAAC,CAAC,QACxB,MAAM,IAAI,MAAM,kBAAkB,MAAM,2BAA2B;EACrE,KAAK,MAAM,OAAO,OAAO,KAAK,OAAO,GACnC,IAAI,CAAC,SAAS,IAAI,GAAG,GACnB,MAAM,IAAI,MAAM,qCAAqC,KAAK;EAC9D,IACE,CAAC,MAAM,MAAM,eACX,OAAO,QAAQ,OAAO,CAAC,CAAC,OACrB,CAAC,KAAK,WACL,CAAC,OAAO,OAAO,YAAY,GAAG,KAAK,MAAM,WAAW,MAAO,KAAK,CACpE,CACF,KACA,CAAC,KAAK,OACH,CAAC,KAAK,aACL,CAAC,OAAO,OAAO,SAAS,GAAG,KAC3B,QAAQ,MAAM,WAAW,MAAM,QAAQ,QAAQ,IAAK,CAAC,CACzD,GAEA,MAAM,IAAI,MAAM,kBAAkB,MAAM,2BAA2B;CACvE;CACA,MAAM,cAAc,IAAI,IACtB,SAAS,KAAK,QAAQ,CAAC,KAAK,MAAM,KAAK,eAAe,WAAW,IAAK,CAAC,CAAC,CAC1E;CAEA,SAAS,OAAO,YAA6B,CAAC,GAAG;EAC/C,KAAK,MAAM,OAAO,OAAO,KAAK,UAAU,cAAc,CAAC,CAAC,GACtD,IAAI,CAAC,SAAS,IAAI,GAAG,GAAG,MAAM,IAAI,MAAM,wBAAwB,KAAK;EAEvE,KAAK,MAAM,OAAO,OAAO,KAAK,UAAU,aAAa,CAAC,CAAC,GACrD,IAAI,SAAS,IAAI,GAAG,GAClB,MAAM,IAAI,MAAM,eAAe,IAAI,qBAAqB;EAE5D,oBAAoB,UAAU,aAAa,CAAC,CAAC;EAC7C,KAAK,MAAM,QAAQ,UAAU,SAAS,CAAC,GACrC,IAAI,CAAC,WAAW,MAAM,MAAM,MAAM,EAAE,OAAO,IAAI,GAC7C,MAAM,IAAI,MAAM,wBAAwB,MAAM;EAElD,MAAM,QAAQ,WAAW,MAAM,QAC5B,MAAM,CAAC,UAAU,SAAS,UAAU,MAAM,SAAS,EAAE,EAAE,CAC1D;EACA,MAAM,2BAAW,IAAI,IAAyB;EAC9C,KAAK,MAAM,CAAC,KAAK,WAAW,OAAO,QAAQ,UAAU,cAAc,CAAC,CAAC,GAAG;GACtE,MAAM,YAAY,WAAW,WAAW,QAAQ,YAAY,IAAI,GAAG;GACnE,MAAM,OAAO,IAAI,IAAI,OAAO,IAAI,mBAAmB,CAAC;GACpD,KAAK,MAAM,SAAS,MAClB,IAAI,CAAC,UAAU,MAAM,WAAW,oBAAoB,MAAM,MAAM,KAAK,GACnE,MAAM,IAAI,MAAM,WAAW,IAAI,UAAU,OAAO;GACpD,SAAS,IAAI,KAAK,IAAI;EACxB;EAmBA,OAAO;GACL;GACA,OApBoB,MAAM,QAAQ,eAClC,SAAS,OACN,QACC,CAAC,SAAS,IAAI,GAAG,KACjB,SAAS,IAAI,GAAG,CAAC,CAAE,IAAI,oBAAoB,WAAW,IAAK,CAAC,CAChE,CAemB;GACnB,MAdmB,KAAK,KACvB,CAAC,MAAM,aACN,CACE,MACA,QAAQ,QACL,WACC,CAAC,SAAS,IAAI,IAAI,KAClB,SAAS,IAAI,IAAI,CAAC,CAAE,IAAI,oBAAoB,MAAM,CAAC,CACvD,CACF,CAKe;GACjB,UAAU;IAAE,GAAG,WAAW;IAAU,GAAG,UAAU;GAAU;EAC7D;CACF;;CAGA,SAAS,UACP,YACA,cACA,QACQ;EACR,MAAM,WAAW,WACd,QAAQ,YACP,OAAO,QAAQ,OAAO,CAAC,CAAC,OACrB,CAAC,KAAK,WACL,CAAC,OAAO,OAAO,YAAY,GAAG,KAAK,MAAM,WAAW,MAAO,KAAK,CACpE,CACF,CAAC,CACA,KACE,YACC,OAAO,YACL,OAAO,QAAQ,OAAO,CAAC,CAAC,QACrB,CAAC,SAAS,CAAC,OAAO,OAAO,YAAY,GAAG,CAC3C,CACF,CACJ;EACF,MAAM,UAAU,aAAa,QAC1B,OAAO,GAAG,YAAY,QAAQ,OAAO,QACtC,CACF;EACA,IAAI,CAAC,SAAS,QAAQ,OAAO;EAC7B,IAAI,SAAS,MAAM,YAAY,CAAC,OAAO,KAAK,OAAO,CAAC,CAAC,MAAM,GAAG,OAAO;EACrE,SAAS,aAAa,SAA6B;GACjD,OAAO,aAAa,QACjB,OAAO,CAAC,MAAM,aACb,SACC,OAAO,OAAO,SAAS,IAAI,IACxB,OAAO,QAAQ,MAAM,WAAW,MAAM,QAAQ,QAAQ,KAAM,CAAC,CAAC,IAC9D,QAAQ,SACd,CACF;EACF;EACA,IAAI,WAAW;EACf,SAAS,MAAM,OAAe,UAAsB,MAAoB;GACtE,KAAK,IAAI,QAAQ,OAAO,QAAQ,SAAS,QAAQ,SAAS;IACxD,IAAI,EAAE,OAAO,YAAY,GACvB,MAAM,IAAI,MACR,gFACF;IACF,MAAM,UAAU,SAAS;IACzB,IACE,OAAO,QAAQ,OAAO,CAAC,CAAC,MACrB,CAAC,KAAK,WACL,OAAO,OAAO,UAAU,GAAG,KAAK,CAAC,MAAM,SAAS,MAAO,KAAK,CAChE,GAEA;IACF,MAAM,SAAS;KAAE,GAAG;KAAU,GAAG;IAAQ;IACzC,MAAM,UAAU,aAAa,MAAM;IACnC,IAAI,CAAC,SAAS;IACd,YAAY,OAAO;IACnB,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI;GAChC;EACF;EACA,MAAM,GAAG,CAAC,GAAG,EAAE;EACf,OAAO;CACT;CAEA,OAAO;EACL,GAAG;EACH,MAAM;EACN,MAAM,WAAW;GACf,MAAM,WAAW,OAAO,SAAS;GACjC,MAAM,SAAS,EAAE,WAAW,aAAa;GAMzC,MAAM,QALU,SAAS,MAAM,QAC5B,OAAO,eACN,QAAQ,UAAU,YAAY,SAAS,MAAM,MAAM,GACrD,CAEkB,IAAI,SAAS,MAAM;GACvC,IAAI,CAAC,OAAO,cAAc,KAAK,GAC7B,MAAM,IAAI,MAAM,gCAAgC;GAClD,OAAO;EACT;EACA,CAAC,MAAM,WAAW;GAChB,MAAM,WAAW,OAAO,SAAS;GACjC,UAAU,OACR,OACA,QACuB;IACvB,IAAI,UAAU,SAAS,KAAK,QAAQ;KAClC,MAAM;KACN;IACF;IACA,MAAM,CAAC,MAAM,WAAW,SAAS,KAAK;IACtC,KAAK,MAAM,UAAU,SACnB,OAAO,OAAO,QAAQ,GAAG;KAAE,GAAG;MAAS,OAAO;IAAO,CAAC;GAC1D;GACA,KAAK,MAAM,cAAc,SAAS,OAChC,KAAK,MAAM,cAAc,SAAS,OAChC,KAAK,MAAM,cAAc,OAAO,GAAG,UAAU,GAAG;IAC9C,IAAI,WAAW,MAAM,YAAY,QAAQ,SAAS,UAAU,CAAC,GAC3D;IACF,MAAM,aAAa;KAAE,GAAG,SAAS;KAAU,GAAG;IAAW;IACzD,MAAM;KACJ,UAAU,WAAW;KACrB,MAAM;KACN;KACA,KAAK,oBAAoB;MACvB,QAAQ,YAAY,UAAU;MAC9B,MAAM,YAAY,UAAU;MAC5B;KACF,CAAC;IACH;GACF;EAGN;CACF;AACF;;;;AC5SA,MAAa,kBAAkB,OAAO,QACpC,WACA,aACA,UACA,WACF;AAEA,MAAa,sBAAsB,OAAO,OAAO;CAC/C,MAAM,OAAO;CACb,SAAS,OAAO;CAChB,OAAO,OAAO,SAAS,OAAO,MAAM;AACtC,CAAC;AAED,MAAa,oBAAoB,OAAO,OAAO;CAC7C,MAAM,OAAO,SAAS,OAAO,MAAM;CACnC,MAAM,OAAO;CACb,IAAI,OAAO;CACX,SAAS,OAAO,SAAS,OAAO,MAAM;AACxC,CAAC;AAGD,MAAa,kBAA4C,OAAO,cAC9D,OAAO,MACL,OAAO,MACP,OAAO,SACP,OAAO,QACP,OAAO,QACP,OAAO,QAAQ,OAAO,MAAM,eAAe,CAAC,GAC5C,OAAO,OAAO;CAAE,KAAK,OAAO;CAAQ,OAAO;AAAgB,CAAC,CAC9D,CACF;AAEA,MAAM,mBAAmB,OAAO,OAAO;CACrC,KAAK,OAAO;CACZ,OAAO;AACT,CAAC;AACD,MAAM,mBAAmB,OAAO,OAAO;CACrC,IAAI,OAAO;CACX,SAAS,OAAO;AAClB,CAAC;AAED,MAAa,oBAAoB,OAAO,OAAO;CAC7C,eAAe,OAAO,QAAQ,GAAG,CAAC;CAClC,QAAQ,kBAAkB,KAAK;CAC/B,QAAQ,OAAO;CACf,SAAS,OAAO,SAAS,OAAO,MAAM;CACtC,YAAY,OAAO,SAAS,gBAAgB;CAC5C,QAAQ,OAAO,SAAS,gBAAgB;CACxC,KAAK,OAAO,SAAS,iBAAiB;CACtC,WAAW,OAAO;AACpB,CAAC;AAED,MAAa,sBAAsB,OAAO,OAAO;CAC/C,eAAe,OAAO,QAAQ,GAAG,CAAC;CAClC,QAAQ,kBAAkB,KAAK;CAC/B,UAAU,kBAAkB,OAAO;CACnC,YAAY,OAAO;CACnB,QAAQ,OAAO;CACf,YAAY,OAAO,SAAS,gBAAgB;CAC5C,QAAQ,OAAO,SAAS,gBAAgB;CACxC,KAAK,OAAO,SAAS,iBAAiB;CACtC,WAAW,OAAO;AACpB,CAAC;;AAGD,MAAa,oBAAoB,OAAO,OAAO;CAC7C,GAAG,kBAAkB;CACrB,QAAQ,OAAO,SAAS,eAAe;AACzC,CAAC;AACD,MAAa,sBAAsB,OAAO,OAAO;CAC/C,GAAG,oBAAoB;CACvB,QAAQ,OAAO,SAAS,eAAe;AACzC,CAAC;AAED,MAAa,cAAc,OAAO,OAAO;CACvC,aAAa,OAAO,SAAS,OAAO,MAAM;CAC1C,cAAc,OAAO,SAAS,OAAO,MAAM;CAC3C,aAAa,OAAO,SAAS,OAAO,MAAM;AAC5C,CAAC;AAED,MAAa,qBAAqB,OAAO,OAAO;CAC9C,OAAO,OAAO,SACZ,OAAO,OAAO;EACZ,IAAI,OAAO;EACX,MAAM,OAAO;EACb,MAAM,OAAO,SAAS,OAAO,MAAM;EACnC,SAAS,OAAO,SAAS,OAAO,MAAM;CACxC,CAAC,CACH;CACA,OAAO,OAAO,SAAS,WAAW;CAClC,QAAQ,OAAO,SAAS,OAAO,QAAQ,OAAO,MAAM,eAAe,CAAC,CAAC;AACvE,CAAC;AAED,MAAa,oBAAoB,OAAO,OAAO;CAC7C,MAAM,OAAO;CACb,MAAM,OAAO,QAAQ,aAAa,OAAO;CACzC,OAAO,OAAO,SAAS,OAAO,MAAM;CACpC,QAAQ,OAAO,SAAS,OAAO,OAAO;CACtC,aAAa,OAAO,SAAS,OAAO,MAAM;CAC1C,UAAU,OAAO,SAAS,eAAe;CACzC,OAAO,OAAO,SAAS,kBAAkB;CACzC,YAAY,OAAO;CACnB,OAAO,OAAO,SAAS,mBAAmB;AAC5C,CAAC;AACD,MAAa,yBAAyB,OAAO,OAAO;CAClD,MAAM,OAAO;CACb,MAAM,OAAO,QAAQ,aAAa,SAAS,kBAAkB;CAC7D,MAAM,OAAO;CACb,QAAQ,OAAO,QAAQ,UAAU,UAAU,SAAS,SAAS;CAC7D,OAAO,OAAO,SAAS,OAAO,MAAM;CACpC,QAAQ,OAAO,SAAS,OAAO,OAAO;CACtC,aAAa,OAAO,SAAS,OAAO,MAAM;CAC1C,UAAU,OAAO,SAAS,eAAe;CACzC,OAAO,OAAO,SAAS,kBAAkB;CACzC,YAAY,OAAO,SAAS,OAAO,MAAM;CACzC,OAAO,OAAO,SAAS,mBAAmB;CAC1C,iBAAiB,OAAO,SACtB,OAAO,OAAO;EAAE,YAAY,OAAO;EAAQ,IAAI,OAAO;CAAO,CAAC,CAChE;AACF,CAAC;AACD,MAAa,qBAAqB,OAAO,OAAO;CAC9C,SAAS,OAAO,QAAQ,OAAO,MAAM,iBAAiB,CAAC;CACvD,aAAa,OAAO,SAClB,OAAO,QAAQ,OAAO,MAAM,sBAAsB,CAAC,CACrD;CACA,gBAAgB,OAAO,SAAS,OAAO,QAAQ,OAAO,MAAM,OAAO,MAAM,CAAC,CAAC;CAC3E,SAAS,OAAO,SAAS,OAAO,MAAM;CACtC,QAAQ,OAAO;AACjB,CAAC;AACD,MAAa,sBAAsB,OAAO,OAAO;CAC/C,MAAM,OAAO;CACb,MAAM,OAAO,QAAQ,QAAQ,WAAW;CACxC,MAAM,OAAO,SAAS,OAAO,MAAM;AACrC,CAAC;AACD,MAAa,qBAAqB,OAAO,OAAO;CAC9C,QAAQ;CACR,SAAS,OAAO;CAChB,YAAY,OAAO,SAAS,OAAO,MAAM;CACzC,SAAS,OAAO,SAAS,kBAAkB;CAC3C,WAAW,OAAO,SAAS,OAAO,QAAQ,OAAO,MAAM,mBAAmB,CAAC,CAAC;CAC5E,OAAO,OAAO,SAAS,mBAAmB;AAC5C,CAAC;AAED,MAAa,mBAAmB,OAAO,OAAO;CAC5C,QAAQ;CACR,SAAS,OAAO;CAChB,YAAY,OAAO,SAAS,OAAO,MAAM;CACzC,YAAY,OAAO;CACnB,QAAQ,OAAO;CACf,QAAQ,OAAO;CACf,OAAO,OAAO,SAAS,mBAAmB;AAC5C,CAAC;AAED,MAAM,WAAW;CACf,SAAS,OAAO,OAAO;EACrB,QAAQ,OAAO,QAAQ,KAAK;EAC5B,MAAM,OAAO,QAAQ,SAAS;EAC9B,WAAW,OAAO;CACpB,CAAC;CACD,SAAS,OAAO,OAAO;EACrB,QAAQ,OAAO,QAAQ,KAAK;EAC5B,MAAM,OAAO,QAAQ,SAAS;EAC9B,MAAM,OAAO,QAAQ,UAAU,QAAQ,aAAa,MAAM;EAC1D,SAAS;EACT,WAAW,OAAO;CACpB,CAAC;CACD,UAAU,OAAO,OAAO;EACtB,QAAQ,OAAO,QAAQ,KAAK;EAC5B,MAAM,OAAO,QAAQ,WAAW;EAChC,IAAI,OAAO;EACX,MAAM,OAAO;EACb,WAAW;EACX,WAAW,OAAO;CACpB,CAAC;CACD,YAAY,OAAO,OAAO;EACxB,QAAQ,OAAO,QAAQ,KAAK;EAC5B,MAAM,OAAO,QAAQ,aAAa;EAClC,IAAI,OAAO;EACX,MAAM,OAAO,SAAS,OAAO,MAAM;EACnC,QAAQ;EACR,WAAW,OAAO;CACpB,CAAC;CACD,aAAa,OAAO,OAAO;EACzB,QAAQ,OAAO,QAAQ,KAAK;EAC5B,MAAM,OAAO,QAAQ,cAAc;EACnC,MAAM,OAAO;EACb,WAAW,OAAO;CACpB,CAAC;CACD,eAAe,OAAO,OAAO;EAC3B,QAAQ,OAAO,QAAQ,KAAK;EAC5B,MAAM,OAAO,QAAQ,gBAAgB;EACrC,MAAM,OAAO;EACb,WAAW,OAAO;EAClB,WAAW,OAAO,SAAS,OAAO,MAAM;EACxC,OAAO,OAAO,SAAS,WAAW;CACpC,CAAC;CACD,WAAW,OAAO,OAAO;EACvB,QAAQ,OAAO,QAAQ,KAAK;EAC5B,MAAM,OAAO,QAAQ,WAAW;EAChC,QAAQ,OAAO,SAAS,eAAe;EACvC,WAAW,OAAO;CACpB,CAAC;CACD,OAAO,OAAO,OAAO;EACnB,QAAQ,OAAO,QAAQ,KAAK;EAC5B,MAAM,OAAO,QAAQ,OAAO;EAC5B,OAAO;EACP,WAAW,OAAO;CACpB,CAAC;AACH;AACA,MAAM,cAAc;CAClB,cAAc,OAAO,OAAO;EAC1B,QAAQ,OAAO,QAAQ,QAAQ;EAC/B,MAAM,OAAO,QAAQ,eAAe;EACpC,WAAW,OAAO;CACpB,CAAC;CACD,aAAa,OAAO,OAAO;EACzB,QAAQ,OAAO,QAAQ,QAAQ;EAC/B,MAAM,OAAO,QAAQ,yBAAyB;EAC9C,MAAM,OAAO;EACb,WAAW,OAAO;CACpB,CAAC;CACD,eAAe,OAAO,OAAO;EAC3B,QAAQ,OAAO,QAAQ,QAAQ;EAC/B,MAAM,OAAO,QAAQ,2BAA2B;EAChD,MAAM,OAAO;EACb,WAAW,OAAO;CACpB,CAAC;CACD,aAAa,OAAO,OAAO;EACzB,QAAQ,OAAO,QAAQ,QAAQ;EAC/B,MAAM,OAAO,QAAQ,yBAAyB;EAC9C,MAAM,OAAO;EACb,WAAW,OAAO;CACpB,CAAC;CACD,mBAAmB,OAAO,OAAO;EAC/B,QAAQ,OAAO,QAAQ,QAAQ;EAC/B,MAAM,OAAO,QAAQ,oBAAoB;EACzC,MAAM,OAAO;EACb,MAAM,OAAO;EACb,WAAW,OAAO;CACpB,CAAC;CACD,qBAAqB,OAAO,OAAO;EACjC,QAAQ,OAAO,QAAQ,QAAQ;EAC/B,MAAM,OAAO,QAAQ,sBAAsB;EAC3C,MAAM,OAAO;EACb,QAAQ,OAAO,QAAQ,UAAU,QAAQ;EACzC,WAAW,OAAO;CACpB,CAAC;CACD,iBAAiB,OAAO,OAAO;EAC7B,QAAQ,OAAO,QAAQ,QAAQ;EAC/B,MAAM,OAAO,QAAQ,kBAAkB;EACvC,MAAM,OAAO;EACb,OAAO;EACP,WAAW,OAAO;CACpB,CAAC;CACD,eAAe,OAAO,OAAO;EAC3B,QAAQ,OAAO,QAAQ,QAAQ;EAC/B,MAAM,OAAO,QAAQ,gBAAgB;EACrC,QAAQ,OAAO;EACf,WAAW,OAAO;CACpB,CAAC;CACD,iBAAiB,OAAO,OAAO;EAC7B,QAAQ,OAAO,QAAQ,QAAQ;EAC/B,MAAM,OAAO,QAAQ,kBAAkB;EACvC,QAAQ,OAAO;EACf,OAAO,OAAO;EACd,WAAW,OAAO;CACpB,CAAC;CACD,cAAc,OAAO,OAAO;EAC1B,QAAQ,OAAO,QAAQ,QAAQ;EAC/B,MAAM,OAAO,QAAQ,eAAe;EACpC,QAAQ,OAAO;EACf,OAAO;EACP,WAAW,OAAO;CACpB,CAAC;CACD,gBAAgB,OAAO,OAAO;EAC5B,QAAQ,OAAO,QAAQ,QAAQ;EAC/B,MAAM,OAAO,QAAQ,iBAAiB;EACtC,WAAW,OAAO;CACpB,CAAC;CACD,OAAO,OAAO,OAAO;EACnB,QAAQ,OAAO,QAAQ,QAAQ;EAC/B,MAAM,OAAO,QAAQ,OAAO;EAC5B,OAAO;EACP,WAAW,OAAO;CACpB,CAAC;AACH;AAEA,MAAa,wBACX,OAAO,MAAM,GAAG,OAAO,OAAO,QAAQ,GAAG,GAAG,OAAO,OAAO,WAAW,CAAC;AAExE,MAAa,iBAAiB,OAAO,OAAO;CAC1C,OAAO,OAAO;CACd,SAAS,OAAO;AAClB,CAAC;;;;ACrSD,SAAgB,qBAAqB,UAAqC;CACxE,MAAM,SAA4B,CAAC;CACnC,KAAK,MAAM,CAAC,OAAO,SAAS,SAAS,MAAM,OAAO,CAAC,CAAC,QAAQ,GAAG;EAC7D,IAAI,CAAC,KAAK,KAAK,GAAG;EAClB,IAAI;GACF,OAAO,KACL,OAAO,kBAAkB,qBAAqB,CAAC,CAAC,KAAK,MAAM,IAAI,CAAC,CAClE;EACF,SAAS,OAAO;GACd,MAAM,IAAI,MAAM,oCAAoC,QAAQ,KAAK,EAC/D,MACF,CAAC;EACH;CACF;CACA,OAAO;AACT;;;;;;;ACiJA,SAAgB,YAAwC,OAAa;CACnE,IAAI,MAAM,UAAU,YAAY,MAAM,QAAQ;CAC9C,OAAO;AACT;AAmCA,SAAS,0BAA0B,KAAqB;CACtD,MAAM,aAAa,IAAI,QAAQ,OAAO,GAAG,CAAC,CAAC,QAAQ,QAAQ,EAAE;CAC7D,MAAM,cAAc,WAAW,MAAM,WAAW,YAAY,GAAG,IAAI,CAAC;CACpE,IAAI,CAAC,eAAe,gBAAgB,OAAO,gBAAgB,MACzD,MAAM,IAAI,MAAM,oCAAoC,KAAK;CAE3D,OAAO;AACT;;;;;AAMA,SAAgB,UACd,KACA,UAA2D,CAAC,GAC1C;CAClB,OAAO;EACL,MAAM;EACN;EACA,KAAK,QAAQ,OAAO,0BAA0B,GAAG;EACjD,YAAY,QAAQ,cAAc;CACpC;AACF;AAEA,SAAgB,KACd,KACA,SACa;CACb,OAAO;EAAE,MAAM;EAAQ;EAAK,GAAG;CAAQ;AACzC;AAEA,SAAgB,WACd,MACA,MACA,YACe;CACf,OAAO;EAAE,MAAM;EAAU;EAAM;EAAM;CAAW;AAClD;AAEA,SAAgB,QAAQ,QAAkD;CACxE,OAAO;EAAE,MAAM;EAAW;CAAO;AACnC;;AAuCA,SAAgB,eACd,UACoB;CACpB,IAAI,CAAC,SAAS,KAAK,KAAK,GACtB,MAAM,IAAI,MAAM,sCAAsC;CACxD,OAAO;EAAE,MAAM;EAAoB,MAAM,SAAS;EAAM;CAAS;AACnE;AAEA,SAAgB,KAAK,SAA2B;CAC9C,OAAO;EAAE,MAAM;EAAQ;CAAQ;AACjC;AAEA,SAAgB,MAAM,aAAoC;CACxD,OAAO;EAAE,MAAM;EAAS;CAAY;AACtC;AAgEA,SAAgB,UACd,MACA,KACA,UAAoD,CAAC,GACpC;CACjB,OAAO;EAAE,MAAM;EAAa;EAAM;EAAK,GAAG;CAAQ;AACpD;AAEA,SAAgB,MACd,MACA,SACW;CACX,IAAI,CAAC,KAAK,KAAK,KAAK,CAAC,QAAQ,OAAO,KAAK,GACvC,MAAM,IAAI,MAAM,yCAAyC;CAC3D,OAAO;EAAE,MAAM;EAAS;EAAM,GAAG;CAAQ;AAC3C;AAuBA,SAAgB,WAA2C,YAAkB;CAC3E,YAAY,UAAU;CACtB,IAAI,UAAU;CACd,KAAK,MAAM,QAAQ,WAAW,YAC5B,IAAI,KAAK,SAAS,QAAQ,UAAU;MAC/B,IAAI,KAAK,SAAS,WAAW,CAAC,SACjC,MAAM,IAAI,MAAM,2CAA2C;CAE/D,KACG,WAAW,WAAW,MAAM,SAAS,KAAK,SAAS,OAAO,KACzD,WAAW,QAAQ,MAAM,SAAS,KAAK,SAAS,OAAO,MACzD,CAAC,WAAW,OAEZ,MAAM,IAAI,MAAM,yCAAyC;CAC3D,IAAI,WAAW,SAAS,WAAW,UAAU,WAAW,OACtD,MAAM,IAAI,MAAM,wDAAwD;CAC1E,OAAO;AACT;AAcA,SAAgB,YACd,OACQ;CACR,YAAY,KAAK;CACjB,OAAO;AACT;AAIA,SAAgB,YAAY,OAA+B;CACzD,IAAI,SAAS,SAAS,UAAU,SAAS,UAAU,OACjD,MAAM,IAAI,MAAM,mDAAmD;CACrE,OAAO,SAAS,MAAM,EAAE;AAC1B;AAEA,SAAS,aACP,cAC4B;CAC5B,OAAO,UAAU,gBAAgB,aAAa,SAAS;AACzD;AAEA,SAAS,YACP,cAC2B;CAC3B,OAAO,CAAC,aAAa,YAAY,KAAK,WAAW;AACnD;;;;;AAsDA,SAAgB,cAEd,eAA6C;CAC7C,MAAM,uBAAO,IAAI,IAA4B;CAC7C,MAAM,4BAAY,IAAI,IAAuB;CAC7C,MAAM,kCAAkB,IAAI,IAAuB;CACnD,MAAM,aAA+B,CAAC;CACtC,MAAM,WAAyB,CAAC;CAEhC,KAAK,MAAM,gBAAgB,eAAe;EACxC,IAAI,aAAa,YAAY,GAAG;GAC9B,MAAM,KAAK,YAAY,YAAY;GACnC,IAAI,SAAS,MAAM,WAAW,YAAY,MAAM,MAAM,EAAE,GACtD,MAAM,IAAI,MAAM,+CAA+C,IAAI;GACrE,SAAS,KAAK,YAAY;GAC1B;EACF;EACA,MAAM,QAAQ,YAAY,YAAY,IAAI,eAAe,KAAA;EACzD,IAAI,OAAO;GACT,MAAM,KAAK,YAAY,KAAK;GAC5B,IAAI,UAAU,IAAI,EAAE,GAClB,MAAM,IAAI,MAAM,8CAA8C,IAAI;GACpE,UAAU,IAAI,IAAI,KAAK;EACzB;EACA,MAAM,cAAyC,QAC3C,MAAM,QACN,CAAC,YAA8B;EACnC,KAAK,MAAM,cAAc,aAAa;GACpC,MAAM,KAAK,YAAY,UAAU;GACjC,IAAI,KAAK,IAAI,EAAE,GACb,MAAM,IAAI,MAAM,6CAA6C,IAAI;GACnE,KAAK,IAAI,IAAI,UAAU;GACvB,IAAI,OAAO,gBAAgB,IAAI,IAAI,KAAK;QACnC,WAAW,KAAK,UAAU;EACjC;CACF;CAEA,MAAM,QAAQ,CACZ,GAAG,YACH,GAAG,MAAM,KAAK,UAAU,OAAO,CAAC,CAAC,CAAC,SAAS,UAAU,MAAM,KAAK,CAClE;CACA,OAAO;EACL;EACA,QAAQ,CAAC,GAAG,UAAU,OAAO,CAAC;EAC9B;EACA,MAAM,OAAO,KAAK,IAAI,EAAE;EACxB,WAAW,OAAO,UAAU,IAAI,EAAE;EAClC,gBACE,MAAM,KAAK,eAAe;GACxB,MAAM,QAAQ,gBAAgB,IAAI,YAAY,UAAU,CAAC;GACzD,OAAO;IACL,IAAI,WAAW;IACf,GAAI,WAAW,OAAO,EAAE,MAAM,WAAW,KAAK,IAAI,CAAC;IACnD,GAAI,QAAQ,EAAE,SAAS,MAAM,GAAG,IAAI,CAAC;GACvC;EACF,CAAC;EACH,qBACE,CAAC,GAAG,UAAU,OAAO,CAAC,CAAC,CAAC,KAAK,WAAW;GACtC,IAAI,MAAM;GACV,SAAS,MAAM,MAAM,IAAI,WAAW;GACpC,GAAI,MAAM,OAAO,EAAE,MAAM,MAAM,KAAK,IAAI,CAAC;EAC3C,EAAE;EACJ,eACE,MAAM,KAAK,eAAe;GACxB,MAAM,QAAQ,gBAAgB,IAAI,YAAY,UAAU,CAAC;GACzD,MAAM,WAAW,WAAW,MAAM;GAClC,OAAO;IACL,IAAI,WAAW;IACf,MAAM,QAAQ,GAAG,MAAM,GAAG,GAAG,WAAW,OAAO,WAAW;IAC1D,GAAI,WAAW,OAAO,EAAE,MAAM,WAAW,KAAK,IAAI,CAAC;IACnD,GAAI,QAAQ,EAAE,SAAS,MAAM,GAAG,IAAI,CAAC;IACrC,OAAO;KACL,GAAI,UAAU,OAAO,EAAE,MAAM,SAAS,KAAK,IAAI,CAAC;KAChD,MAAM,UAAU,QAAQ;KACxB,GAAI,UAAU,KAAK,EAAE,IAAI,SAAS,GAAG,IAAI,CAAC;KAC1C,GAAI,UAAU,UAAU,EAAE,SAAS,SAAS,QAAQ,IAAI,CAAC;KACzD,UAAU,OAAO,QAAQ,WAAW,MAAM,YAAY,CAAC,CAAC,CAAC,CAAC,KACvD,CAAC,MAAM,cAAc;MACd;MACN,MAAM,QAAQ;KAChB,EACF;IACF;IACA,WAAW,WAAW,YAAY,CAAC,EAAA,CAAG,KAAK,YACzC,QAAQ,SAAS,YACb;KAAE,MAAM;KAAW,QAAQ;IAAU,IACrC,QAAQ,SAAS,WACf;KACE,MAAM;KACN,QAAQ,QAAQ;KAChB,aAAa,QAAQ;KACrB,YAAY,QAAQ;IACtB,IACA;KACE,MAAM,QAAQ;KACd,QAAQ,QAAQ;KAChB,aAAa,QAAQ;KACrB,YAAY,QAAQ;IACtB,CACR;IACA,YAAY,WAAW,QAAQ,UAAU;IACzC,SAAS,WAAW,QAAQ,KAAK,EAAE,MAAM,YAAY;KAAE;KAAM;IAAK,EAAE;GACtE;EACF,CAAC;CACL;AACF;AAoIA,SAAgB,YAAY,OAA+B;CACzD,IAAI,iBAAiB,OACnB,OAAO;EACL,MAAM,MAAM;EACZ,SAAS,MAAM;EACf,GAAI,MAAM,QAAQ,EAAE,OAAO,MAAM,MAAM,IAAI,CAAC;CAC9C;CAGF,OAAO;EAAE,MAAM;EAAS,SAAS,OAAO,KAAK;CAAE;AACjD"}