@tangle-network/agent-eval 0.102.1 → 0.103.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. package/dist/adapters/http.d.ts +2 -2
  2. package/dist/adapters/langchain.d.ts +2 -2
  3. package/dist/adapters/otel.d.ts +5 -5
  4. package/dist/analyst/index.d.ts +9 -9
  5. package/dist/analyst/index.js +3 -3
  6. package/dist/{analyze-runs-Cd-A_K4l.d.ts → analyze-runs-rF2eGxZ_.d.ts} +3 -3
  7. package/dist/belief-state/index.d.ts +3 -3
  8. package/dist/belief-state/index.js +1 -1
  9. package/dist/benchmarks/index.d.ts +2 -2
  10. package/dist/builder-eval/index.js +2 -2
  11. package/dist/campaign/index.d.ts +43 -16
  12. package/dist/campaign/index.js +9 -9
  13. package/dist/campaign/index.js.map +1 -1
  14. package/dist/{chunk-Z3FLN24V.js → chunk-2B4HPFXN.js} +104 -43
  15. package/dist/chunk-2B4HPFXN.js.map +1 -0
  16. package/dist/{chunk-7RBJANJD.js → chunk-3722PKPB.js} +2 -2
  17. package/dist/{chunk-YLKDN7JV.js → chunk-66T25VWE.js} +5 -5
  18. package/dist/chunk-66T25VWE.js.map +1 -0
  19. package/dist/{chunk-6FIAJHCU.js → chunk-6YVAN6R3.js} +4 -4
  20. package/dist/{chunk-ABOIVNXL.js → chunk-AGYDMORK.js} +1 -1
  21. package/dist/chunk-AGYDMORK.js.map +1 -0
  22. package/dist/{chunk-B2TMQM62.js → chunk-BHCFJGL4.js} +2 -2
  23. package/dist/{chunk-IH7LYRHL.js → chunk-BXZVBX3D.js} +3 -3
  24. package/dist/{chunk-6GT4NI4V.js → chunk-CLS3374R.js} +2 -2
  25. package/dist/{chunk-HRGUJTER.js → chunk-HYGRFL7C.js} +1 -1
  26. package/dist/{chunk-HRGUJTER.js.map → chunk-HYGRFL7C.js.map} +1 -1
  27. package/dist/{chunk-2NSLDY4B.js → chunk-J5MUFXEY.js} +2 -2
  28. package/dist/{chunk-UFSG7ACU.js → chunk-JZXGWLK5.js} +1 -1
  29. package/dist/chunk-JZXGWLK5.js.map +1 -0
  30. package/dist/{chunk-IN3SHQML.js → chunk-LMSQ6EFA.js} +2 -2
  31. package/dist/{chunk-AIGWQEME.js → chunk-LOZOZYHU.js} +2 -2
  32. package/dist/{chunk-NF7OZ4J7.js → chunk-QCBB6ZIU.js} +2 -2
  33. package/dist/chunk-RQNOLV3I.js +855 -0
  34. package/dist/chunk-RQNOLV3I.js.map +1 -0
  35. package/dist/{chunk-JU6ZX3CX.js → chunk-SMCACT4Z.js} +2 -2
  36. package/dist/{chunk-6Q2DYRWV.js → chunk-TCPP4Z5Q.js} +5 -5
  37. package/dist/{chunk-VDGPPGE3.js → chunk-UMEAR2FI.js} +2 -2
  38. package/dist/{chunk-VDGPPGE3.js.map → chunk-UMEAR2FI.js.map} +1 -1
  39. package/dist/{chunk-QIT2XZ4E.js → chunk-VNM52AGA.js} +3 -3
  40. package/dist/chunk-VNM52AGA.js.map +1 -0
  41. package/dist/{chunk-CMJSTXUR.js → chunk-YFYIOSNV.js} +107 -20
  42. package/dist/chunk-YFYIOSNV.js.map +1 -0
  43. package/dist/{code-agent-session-Ce-9u7YM.d.ts → code-agent-session-Cen-qD2y.d.ts} +1 -1
  44. package/dist/contract/index.d.ts +18 -18
  45. package/dist/contract/index.js +10 -10
  46. package/dist/{control-C8RmK9H4.d.ts → control-KofK3gfG.d.ts} +1 -1
  47. package/dist/control.d.ts +2 -2
  48. package/dist/control.js +3 -3
  49. package/dist/{corpus-CiSzzLa5.d.ts → corpus-CysLCiK4.d.ts} +1 -1
  50. package/dist/{default-registry-ZhqsTr4K.d.ts → default-registry-Ez4cxuZ4.d.ts} +2 -2
  51. package/dist/diagnose.d.ts +3 -3
  52. package/dist/diagnose.js +3 -3
  53. package/dist/{gepa-DeyPTlvx.d.ts → gepa-COlCAkHN.d.ts} +22 -1
  54. package/dist/governance/index.d.ts +1 -1
  55. package/dist/hosted/index.d.ts +5 -5
  56. package/dist/{index-B-bFgiAF.d.ts → index-unCSYRJJ.d.ts} +1 -1
  57. package/dist/index.d.ts +36 -28
  58. package/dist/index.js +32 -18
  59. package/dist/index.js.map +1 -1
  60. package/dist/{insight-report-k0sRTzKg.d.ts → insight-report-DumCfEur.d.ts} +2 -2
  61. package/dist/{judge-calibration-0p2QcWNE.d.ts → judge-calibration-7C-IDmKr.d.ts} +3 -0
  62. package/dist/{kind-factory-D0nk7AKV.d.ts → kind-factory-BLHxwX71.d.ts} +1 -1
  63. package/dist/meta-eval/index.d.ts +4 -4
  64. package/dist/meta-eval/index.js +3 -3
  65. package/dist/{multi-layer-verifier-DUZXrPDA.d.ts → multi-layer-verifier-CI4jdX-q.d.ts} +3 -0
  66. package/dist/multishot/index.d.ts +2 -2
  67. package/dist/openapi.json +1 -1
  68. package/dist/pipelines/index.d.ts +1 -1
  69. package/dist/pipelines/index.js +3 -3
  70. package/dist/{policy-edit-BDQzzsBU.d.ts → policy-edit-nUhuFtLF.d.ts} +2 -2
  71. package/dist/{pre-registration-Dzg61IQA.d.ts → pre-registration-CUOSGAZK.d.ts} +45 -12
  72. package/dist/product-benchmark/index.d.ts +104 -1
  73. package/dist/product-benchmark/index.js +15 -1
  74. package/dist/{provenance-BhJm32vN.d.ts → provenance-B2VsP0jP.d.ts} +20 -4
  75. package/dist/{query-B7GGjRox.d.ts → query-0aTmbmQe.d.ts} +1 -0
  76. package/dist/{release-report-BQ1Ziyu-.d.ts → release-report-DkaCZ9k4.d.ts} +5 -2
  77. package/dist/reporting.d.ts +6 -6
  78. package/dist/reporting.js +4 -4
  79. package/dist/{researcher-B_ODTAJs.d.ts → researcher-SDAezfML.d.ts} +2 -2
  80. package/dist/rl.d.ts +9 -9
  81. package/dist/rl.js +7 -7
  82. package/dist/{rubric-predictive-validity-0MdjTt8R.d.ts → rubric-predictive-validity-ZGIlJNce.d.ts} +1 -1
  83. package/dist/{run-campaign-3NWW5PLF.js → run-campaign-DAHKO5CT.js} +3 -3
  84. package/dist/{run-record-MRdJ-Kq2.d.ts → run-record-CPfd1ARZ.d.ts} +3 -0
  85. package/dist/{runtime-trajectory-8w0_jmtR.d.ts → runtime-trajectory-DZ8ei-Jo.d.ts} +1 -1
  86. package/dist/{semantic-concept-judge-D-IlH5v1.d.ts → semantic-concept-judge-C6mDEBIo.d.ts} +3 -3
  87. package/dist/{statistics-xP-cWc5k.d.ts → statistics-D88peojY.d.ts} +1 -1
  88. package/dist/{summary-report-C0nnxOD8.d.ts → summary-report-Fc_YFJat.d.ts} +1 -1
  89. package/dist/traces.d.ts +2 -2
  90. package/dist/traces.js +4 -4
  91. package/dist/{types-DFI_Z-ZL.d.ts → types-Bihq6-a3.d.ts} +1 -1
  92. package/dist/{types-Dz9cKF0g.d.ts → types-DA9yj-Jd.d.ts} +1 -1
  93. package/dist/workflow/index.d.ts +7 -7
  94. package/dist/workflow/index.js +3 -3
  95. package/package.json +1 -1
  96. package/dist/chunk-63MBSQTX.js +0 -350
  97. package/dist/chunk-63MBSQTX.js.map +0 -1
  98. package/dist/chunk-ABOIVNXL.js.map +0 -1
  99. package/dist/chunk-CMJSTXUR.js.map +0 -1
  100. package/dist/chunk-QIT2XZ4E.js.map +0 -1
  101. package/dist/chunk-UFSG7ACU.js.map +0 -1
  102. package/dist/chunk-YLKDN7JV.js.map +0 -1
  103. package/dist/chunk-Z3FLN24V.js.map +0 -1
  104. /package/dist/{chunk-7RBJANJD.js.map → chunk-3722PKPB.js.map} +0 -0
  105. /package/dist/{chunk-6FIAJHCU.js.map → chunk-6YVAN6R3.js.map} +0 -0
  106. /package/dist/{chunk-B2TMQM62.js.map → chunk-BHCFJGL4.js.map} +0 -0
  107. /package/dist/{chunk-IH7LYRHL.js.map → chunk-BXZVBX3D.js.map} +0 -0
  108. /package/dist/{chunk-6GT4NI4V.js.map → chunk-CLS3374R.js.map} +0 -0
  109. /package/dist/{chunk-2NSLDY4B.js.map → chunk-J5MUFXEY.js.map} +0 -0
  110. /package/dist/{chunk-IN3SHQML.js.map → chunk-LMSQ6EFA.js.map} +0 -0
  111. /package/dist/{chunk-AIGWQEME.js.map → chunk-LOZOZYHU.js.map} +0 -0
  112. /package/dist/{chunk-NF7OZ4J7.js.map → chunk-QCBB6ZIU.js.map} +0 -0
  113. /package/dist/{chunk-JU6ZX3CX.js.map → chunk-SMCACT4Z.js.map} +0 -0
  114. /package/dist/{chunk-6Q2DYRWV.js.map → chunk-TCPP4Z5Q.js.map} +0 -0
  115. /package/dist/{run-campaign-3NWW5PLF.js.map → run-campaign-DAHKO5CT.js.map} +0 -0
@@ -1 +0,0 @@
1
- {"version":3,"sources":["../src/product-benchmark/index.ts"],"sourcesContent":["import { existsSync, readFileSync, statSync } from 'node:fs'\nimport { dirname, join } from 'node:path'\nimport { ValidationError } from '../errors'\n\nexport const productBenchmarkSplits = ['practice', 'dev', 'holdout', 'safety', 'sentinel'] as const\n\nexport type ProductBenchmarkSplit = (typeof productBenchmarkSplits)[number]\n\nexport interface ProductBenchmarkRepoRef {\n readonly url: string\n readonly commit: string\n readonly branch: string\n}\n\nexport interface ProductBenchmarkSubstrateVersions {\n readonly agentEval: string\n readonly agentRuntime: string\n readonly agentInterface: string\n readonly sandbox: string\n readonly agentBench?: string\n}\n\nexport interface ProductBenchmarkProfileRef {\n readonly id: string\n readonly profileHash: string\n readonly agentProfilePath: string\n}\n\nexport interface ProductBenchmarkArm {\n readonly id: string\n readonly profileId: string\n readonly mutableSurfaces: readonly string[]\n readonly policyAxes: Record<string, unknown>\n}\n\nexport interface ProductBenchmarkScenario {\n readonly id: string\n readonly split: ProductBenchmarkSplit\n readonly tags: readonly string[]\n readonly sourceAllowedForSynthesis: boolean\n}\n\nexport interface ProductBenchmarkBudgets {\n readonly maxUsd: number\n readonly maxCells: number\n readonly maxWallMs: number\n}\n\nexport interface ProductBenchmarkManifest {\n readonly schemaVersion: 1\n readonly projectId: string\n readonly benchmarkId: string\n readonly repo: ProductBenchmarkRepoRef\n readonly substrate: ProductBenchmarkSubstrateVersions\n readonly profiles: readonly ProductBenchmarkProfileRef[]\n readonly arms: readonly ProductBenchmarkArm[]\n readonly scenarios: readonly ProductBenchmarkScenario[]\n readonly budgets: ProductBenchmarkBudgets\n readonly expectedArtifactDir: string\n}\n\nexport interface AgentProfileRuntimeReceipt {\n readonly model: string\n readonly harness: string\n readonly backend: string\n readonly reasoningEffort?: string\n}\n\nexport type RuntimeResolution = AgentProfileRuntimeReceipt\n\nexport interface ProductBenchmarkRecord {\n readonly schemaVersion: 1\n readonly projectId: string\n readonly benchmarkId: string\n readonly runId: string\n readonly scenarioId: string\n readonly split: ProductBenchmarkSplit\n readonly armId: string\n readonly rep: number\n readonly agentProfile: {\n readonly id: string\n readonly hash: string\n readonly path: string\n readonly declared: RuntimeResolution\n readonly resolved: RuntimeResolution\n }\n readonly model: {\n readonly provider: string\n readonly id: string\n }\n readonly backend: {\n readonly kind: string\n readonly version: string\n }\n readonly outcome: {\n readonly pass: boolean\n readonly score: number\n readonly dimensions: Record<string, number>\n readonly failureMode: string | null\n }\n readonly usage: {\n readonly inputTokens: number\n readonly outputTokens: number\n readonly costUsd: number\n readonly wallMs: number\n readonly toolCalls: number\n }\n readonly integrity: {\n readonly realBackend: boolean\n readonly rawCapture: boolean\n readonly traceCapture: boolean\n readonly noStubRows: boolean\n readonly priced: boolean\n readonly profileMaterialized: boolean\n }\n readonly artifacts: {\n readonly records: string\n readonly traces: string\n readonly raws: string\n readonly scores: string\n readonly workspace: string\n }\n}\n\nexport interface ProductBenchmarkRunInput {\n readonly manifestPath: string\n readonly recordsPath: string\n readonly artifactRoot?: string\n readonly checkArtifacts?: boolean\n}\n\nexport interface ProductBenchmarkValidationReport {\n readonly manifestPath: string\n readonly recordsPath: string\n readonly records: number\n readonly projects: readonly string[]\n readonly benchmarks: readonly string[]\n readonly arms: readonly string[]\n readonly scenarios: readonly string[]\n readonly passed: number\n readonly failed: number\n readonly inputTokens: number\n readonly outputTokens: number\n readonly costUsd: number\n readonly wallMs: number\n readonly integrityFailures: readonly string[]\n readonly missingArtifacts: readonly string[]\n}\n\nexport interface ProductBenchmarkArtifactPaths {\n readonly manifestPath: string\n readonly recordsPath: string\n}\n\nfunction isObject(value: unknown): value is Record<string, unknown> {\n return Boolean(value) && typeof value === 'object' && !Array.isArray(value)\n}\n\nfunction fail(path: string, message: string): never {\n throw new ValidationError(`${path}: ${message}`)\n}\n\nfunction wrapValidationError(path: string, err: unknown): never {\n if (err instanceof ValidationError) {\n throw new ValidationError(`${path}: ${err.message}`, { cause: err })\n }\n throw new ValidationError(`${path}: ${err instanceof Error ? err.message : String(err)}`, {\n cause: err,\n })\n}\n\nfunction expectObject(value: unknown, path: string): Record<string, unknown> {\n if (!isObject(value)) fail(path, 'must be an object')\n return value\n}\n\nfunction expectString(value: unknown, path: string): string {\n if (typeof value !== 'string' || value.trim().length === 0)\n fail(path, 'must be a non-empty string')\n return value\n}\n\nfunction expectBoolean(value: unknown, path: string): boolean {\n if (typeof value !== 'boolean') fail(path, 'must be a boolean')\n return value\n}\n\nfunction expectNumber(\n value: unknown,\n path: string,\n opts: { readonly min?: number; readonly max?: number; readonly integer?: boolean } = {},\n): number {\n if (typeof value !== 'number' || !Number.isFinite(value)) fail(path, 'must be a finite number')\n if (opts.integer && !Number.isInteger(value)) fail(path, 'must be an integer')\n if (opts.min !== undefined && value < opts.min) fail(path, `must be >= ${opts.min}`)\n if (opts.max !== undefined && value > opts.max) fail(path, `must be <= ${opts.max}`)\n return value\n}\n\nfunction expectStringArray(value: unknown, path: string): string[] {\n if (!Array.isArray(value)) fail(path, 'must be an array')\n return value.map((entry, index) => expectString(entry, `${path}[${index}]`))\n}\n\nfunction expectObjectArray(value: unknown, path: string): Record<string, unknown>[] {\n if (!Array.isArray(value)) fail(path, 'must be an array')\n return value.map((entry, index) => expectObject(entry, `${path}[${index}]`))\n}\n\nfunction expectSplit(value: unknown, path: string): ProductBenchmarkSplit {\n const split = expectString(value, path)\n if (!productBenchmarkSplits.includes(split as ProductBenchmarkSplit)) {\n fail(path, `must be one of ${productBenchmarkSplits.join(', ')}`)\n }\n return split as ProductBenchmarkSplit\n}\n\nfunction optionalString(value: unknown, path: string): string | undefined {\n if (value === undefined) return undefined\n return expectString(value, path)\n}\n\nfunction expectDimensions(value: unknown, path: string): Record<string, number> {\n const obj = expectObject(value, path)\n const out: Record<string, number> = {}\n for (const [key, raw] of Object.entries(obj)) out[key] = expectNumber(raw, `${path}.${key}`)\n return out\n}\n\nfunction expectRuntimeResolution(value: unknown, path: string): RuntimeResolution {\n const obj = expectObject(value, path)\n return {\n model: expectString(obj.model, `${path}.model`),\n harness: expectString(obj.harness, `${path}.harness`),\n backend: expectString(obj.backend, `${path}.backend`),\n ...(obj.reasoningEffort !== undefined\n ? { reasoningEffort: optionalString(obj.reasoningEffort, `${path}.reasoningEffort`) }\n : {}),\n }\n}\n\nexport function validateProductBenchmarkManifest(value: unknown): ProductBenchmarkManifest {\n const obj = expectObject(value, 'manifest')\n if (obj.schemaVersion !== 1) fail('manifest.schemaVersion', 'must be 1')\n const repo = expectObject(obj.repo, 'manifest.repo')\n const substrate = expectObject(obj.substrate, 'manifest.substrate')\n const profiles = expectObjectArray(obj.profiles, 'manifest.profiles').map((profile, index) => ({\n id: expectString(profile.id, `manifest.profiles[${index}].id`),\n profileHash: expectString(profile.profileHash, `manifest.profiles[${index}].profileHash`),\n agentProfilePath: expectString(\n profile.agentProfilePath,\n `manifest.profiles[${index}].agentProfilePath`,\n ),\n }))\n const arms = expectObjectArray(obj.arms, 'manifest.arms').map((arm, index) => ({\n id: expectString(arm.id, `manifest.arms[${index}].id`),\n profileId: expectString(arm.profileId, `manifest.arms[${index}].profileId`),\n mutableSurfaces: expectStringArray(\n arm.mutableSurfaces,\n `manifest.arms[${index}].mutableSurfaces`,\n ),\n policyAxes: expectObject(arm.policyAxes, `manifest.arms[${index}].policyAxes`),\n }))\n const scenarios = expectObjectArray(obj.scenarios, 'manifest.scenarios').map(\n (scenario, index) => ({\n id: expectString(scenario.id, `manifest.scenarios[${index}].id`),\n split: expectSplit(scenario.split, `manifest.scenarios[${index}].split`),\n tags: expectStringArray(scenario.tags, `manifest.scenarios[${index}].tags`),\n sourceAllowedForSynthesis: expectBoolean(\n scenario.sourceAllowedForSynthesis,\n `manifest.scenarios[${index}].sourceAllowedForSynthesis`,\n ),\n }),\n )\n const budgets = expectObject(obj.budgets, 'manifest.budgets')\n if (profiles.length === 0) fail('manifest.profiles', 'must contain at least one profile')\n if (arms.length === 0) fail('manifest.arms', 'must contain at least one arm')\n if (scenarios.length === 0) fail('manifest.scenarios', 'must contain at least one scenario')\n assertUnique(\n profiles.map((profile) => profile.id),\n 'manifest.profiles.id',\n )\n assertUnique(\n arms.map((arm) => arm.id),\n 'manifest.arms.id',\n )\n assertUnique(\n scenarios.map((scenario) => scenario.id),\n 'manifest.scenarios.id',\n )\n const profileIds = new Set(profiles.map((profile) => profile.id))\n for (const arm of arms) {\n if (!profileIds.has(arm.profileId))\n fail(`manifest.arms.${arm.id}.profileId`, `unknown profile ${arm.profileId}`)\n }\n return {\n schemaVersion: 1,\n projectId: expectString(obj.projectId, 'manifest.projectId'),\n benchmarkId: expectString(obj.benchmarkId, 'manifest.benchmarkId'),\n repo: {\n url: expectString(repo.url, 'manifest.repo.url'),\n commit: expectString(repo.commit, 'manifest.repo.commit'),\n branch: expectString(repo.branch, 'manifest.repo.branch'),\n },\n substrate: {\n agentEval: expectString(substrate.agentEval, 'manifest.substrate.agentEval'),\n agentRuntime: expectString(substrate.agentRuntime, 'manifest.substrate.agentRuntime'),\n agentInterface: expectString(substrate.agentInterface, 'manifest.substrate.agentInterface'),\n sandbox: expectString(substrate.sandbox, 'manifest.substrate.sandbox'),\n ...(substrate.agentBench !== undefined\n ? { agentBench: expectString(substrate.agentBench, 'manifest.substrate.agentBench') }\n : {}),\n },\n profiles,\n arms,\n scenarios,\n budgets: {\n maxUsd: expectNumber(budgets.maxUsd, 'manifest.budgets.maxUsd', { min: 0 }),\n maxCells: expectNumber(budgets.maxCells, 'manifest.budgets.maxCells', {\n min: 0,\n integer: true,\n }),\n maxWallMs: expectNumber(budgets.maxWallMs, 'manifest.budgets.maxWallMs', {\n min: 0,\n integer: true,\n }),\n },\n expectedArtifactDir: expectString(obj.expectedArtifactDir, 'manifest.expectedArtifactDir'),\n }\n}\n\nexport function validateProductBenchmarkRecord(value: unknown): ProductBenchmarkRecord {\n const obj = expectObject(value, 'record')\n if (obj.schemaVersion !== 1) fail('record.schemaVersion', 'must be 1')\n const agentProfile = expectObject(obj.agentProfile, 'record.agentProfile')\n const model = expectObject(obj.model, 'record.model')\n const backend = expectObject(obj.backend, 'record.backend')\n const outcome = expectObject(obj.outcome, 'record.outcome')\n const usage = expectObject(obj.usage, 'record.usage')\n const integrity = expectObject(obj.integrity, 'record.integrity')\n const artifacts = expectObject(obj.artifacts, 'record.artifacts')\n const record: ProductBenchmarkRecord = {\n schemaVersion: 1,\n projectId: expectString(obj.projectId, 'record.projectId'),\n benchmarkId: expectString(obj.benchmarkId, 'record.benchmarkId'),\n runId: expectString(obj.runId, 'record.runId'),\n scenarioId: expectString(obj.scenarioId, 'record.scenarioId'),\n split: expectSplit(obj.split, 'record.split'),\n armId: expectString(obj.armId, 'record.armId'),\n rep: expectNumber(obj.rep, 'record.rep', { min: 1, integer: true }),\n agentProfile: {\n id: expectString(agentProfile.id, 'record.agentProfile.id'),\n hash: expectString(agentProfile.hash, 'record.agentProfile.hash'),\n path: expectString(agentProfile.path, 'record.agentProfile.path'),\n declared: expectRuntimeResolution(agentProfile.declared, 'record.agentProfile.declared'),\n resolved: expectRuntimeResolution(agentProfile.resolved, 'record.agentProfile.resolved'),\n },\n model: {\n provider: expectString(model.provider, 'record.model.provider'),\n id: expectString(model.id, 'record.model.id'),\n },\n backend: {\n kind: expectString(backend.kind, 'record.backend.kind'),\n version: expectString(backend.version, 'record.backend.version'),\n },\n outcome: {\n pass: expectBoolean(outcome.pass, 'record.outcome.pass'),\n score: expectNumber(outcome.score, 'record.outcome.score', { min: 0, max: 1 }),\n dimensions: expectDimensions(outcome.dimensions, 'record.outcome.dimensions'),\n failureMode:\n outcome.failureMode === null\n ? null\n : expectString(outcome.failureMode, 'record.outcome.failureMode'),\n },\n usage: {\n inputTokens: expectNumber(usage.inputTokens, 'record.usage.inputTokens', {\n min: 0,\n integer: true,\n }),\n outputTokens: expectNumber(usage.outputTokens, 'record.usage.outputTokens', {\n min: 0,\n integer: true,\n }),\n costUsd: expectNumber(usage.costUsd, 'record.usage.costUsd', { min: 0 }),\n wallMs: expectNumber(usage.wallMs, 'record.usage.wallMs', { min: 0, integer: true }),\n toolCalls: expectNumber(usage.toolCalls, 'record.usage.toolCalls', { min: 0, integer: true }),\n },\n integrity: {\n realBackend: expectBoolean(integrity.realBackend, 'record.integrity.realBackend'),\n rawCapture: expectBoolean(integrity.rawCapture, 'record.integrity.rawCapture'),\n traceCapture: expectBoolean(integrity.traceCapture, 'record.integrity.traceCapture'),\n noStubRows: expectBoolean(integrity.noStubRows, 'record.integrity.noStubRows'),\n priced: expectBoolean(integrity.priced, 'record.integrity.priced'),\n profileMaterialized: expectBoolean(\n integrity.profileMaterialized,\n 'record.integrity.profileMaterialized',\n ),\n },\n artifacts: {\n records: expectString(artifacts.records, 'record.artifacts.records'),\n traces: expectString(artifacts.traces, 'record.artifacts.traces'),\n raws: expectString(artifacts.raws, 'record.artifacts.raws'),\n scores: expectString(artifacts.scores, 'record.artifacts.scores'),\n workspace: expectString(artifacts.workspace, 'record.artifacts.workspace'),\n },\n }\n if (record.integrity.realBackend && record.usage.inputTokens + record.usage.outputTokens === 0) {\n fail('record.usage', 'realBackend rows must carry non-zero token usage')\n }\n return record\n}\n\nexport function productBenchmarkIntegrityFailures(record: ProductBenchmarkRecord): string[] {\n const failures: string[] = []\n for (const [key, value] of Object.entries(record.integrity)) {\n if (!value) failures.push(`${record.runId}:${key}=false`)\n }\n return failures\n}\n\nexport function readProductBenchmarkRecords(path: string): ProductBenchmarkRecord[] {\n const lines = readFileSync(path, 'utf8')\n .split('\\n')\n .map((line) => line.trim())\n .filter(Boolean)\n const records: ProductBenchmarkRecord[] = []\n for (const [index, line] of lines.entries()) {\n try {\n records.push(validateProductBenchmarkRecord(JSON.parse(line)))\n } catch (err) {\n wrapValidationError(`${path}:${index + 1}`, err)\n }\n }\n return records\n}\n\nexport function readProductBenchmarkManifest(path: string): ProductBenchmarkManifest {\n try {\n return validateProductBenchmarkManifest(JSON.parse(readFileSync(path, 'utf8')))\n } catch (err) {\n wrapValidationError(path, err)\n }\n}\n\nexport function validateProductBenchmarkRun(\n input: ProductBenchmarkRunInput,\n): ProductBenchmarkValidationReport {\n const manifest = readProductBenchmarkManifest(input.manifestPath)\n const records = readProductBenchmarkRecords(input.recordsPath)\n const manifestProjectBench = `${manifest.projectId}/${manifest.benchmarkId}`\n const integrityFailures = records.flatMap(productBenchmarkIntegrityFailures)\n const missingArtifacts =\n input.checkArtifacts === false\n ? []\n : records.flatMap((record) =>\n missingArtifactsForRecord(record, input.artifactRoot ?? dirname(input.recordsPath)),\n )\n for (const [index, record] of records.entries()) {\n const recordProjectBench = `${record.projectId}/${record.benchmarkId}`\n if (recordProjectBench !== manifestProjectBench) {\n fail(\n `records[${index}]`,\n `project/benchmark ${recordProjectBench} does not match manifest ${manifestProjectBench}`,\n )\n }\n if (!manifest.arms.some((arm) => arm.id === record.armId))\n fail(`records[${index}].armId`, `unknown arm ${record.armId}`)\n if (!manifest.scenarios.some((scenario) => scenario.id === record.scenarioId)) {\n fail(`records[${index}].scenarioId`, `unknown scenario ${record.scenarioId}`)\n }\n }\n return {\n manifestPath: input.manifestPath,\n recordsPath: input.recordsPath,\n records: records.length,\n projects: sortedUnique(records.map((record) => record.projectId)),\n benchmarks: sortedUnique(records.map((record) => record.benchmarkId)),\n arms: sortedUnique(records.map((record) => record.armId)),\n scenarios: sortedUnique(records.map((record) => record.scenarioId)),\n passed: records.filter((record) => record.outcome.pass).length,\n failed: records.filter((record) => !record.outcome.pass).length,\n inputTokens: sum(records, (record) => record.usage.inputTokens),\n outputTokens: sum(records, (record) => record.usage.outputTokens),\n costUsd: sum(records, (record) => record.usage.costUsd),\n wallMs: sum(records, (record) => record.usage.wallMs),\n integrityFailures,\n missingArtifacts,\n }\n}\n\nfunction missingArtifactsForRecord(record: ProductBenchmarkRecord, artifactRoot: string): string[] {\n const missing: string[] = []\n for (const [key, value] of Object.entries(record.artifacts)) {\n const path = resolveArtifactPath(value, artifactRoot)\n if (!existsSync(path)) missing.push(`${record.runId}:${key}:${value}`)\n }\n return missing\n}\n\nfunction resolveArtifactPath(value: string, artifactRoot: string): string {\n return value.startsWith('/') ? value : join(artifactRoot, value)\n}\n\nfunction assertUnique(values: readonly string[], path: string): void {\n const seen = new Set<string>()\n for (const value of values) {\n if (seen.has(value)) fail(path, `duplicate ${value}`)\n seen.add(value)\n }\n}\n\nfunction sortedUnique(values: readonly string[]): string[] {\n return [...new Set(values)].sort()\n}\n\nfunction sum<T>(items: readonly T[], fn: (item: T) => number): number {\n return items.reduce((total, item) => total + fn(item), 0)\n}\n\nexport function findProductBenchmarkArtifacts(\n runDir: string,\n): ProductBenchmarkArtifactPaths | null {\n const manifestPath = join(runDir, 'product-benchmark-manifest.json')\n const recordsPath = join(runDir, 'product-benchmark-records.jsonl')\n if (\n existsSync(manifestPath) &&\n statSync(manifestPath).isFile() &&\n existsSync(recordsPath) &&\n statSync(recordsPath).isFile()\n ) {\n return { manifestPath, recordsPath }\n }\n return null\n}\n"],"mappings":";;;;;AAAA,SAAS,YAAY,cAAc,gBAAgB;AACnD,SAAS,SAAS,YAAY;AAGvB,IAAM,yBAAyB,CAAC,YAAY,OAAO,WAAW,UAAU,UAAU;AAsJzF,SAAS,SAAS,OAAkD;AAClE,SAAO,QAAQ,KAAK,KAAK,OAAO,UAAU,YAAY,CAAC,MAAM,QAAQ,KAAK;AAC5E;AAEA,SAAS,KAAK,MAAc,SAAwB;AAClD,QAAM,IAAI,gBAAgB,GAAG,IAAI,KAAK,OAAO,EAAE;AACjD;AAEA,SAAS,oBAAoB,MAAc,KAAqB;AAC9D,MAAI,eAAe,iBAAiB;AAClC,UAAM,IAAI,gBAAgB,GAAG,IAAI,KAAK,IAAI,OAAO,IAAI,EAAE,OAAO,IAAI,CAAC;AAAA,EACrE;AACA,QAAM,IAAI,gBAAgB,GAAG,IAAI,KAAK,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC,IAAI;AAAA,IACxF,OAAO;AAAA,EACT,CAAC;AACH;AAEA,SAAS,aAAa,OAAgB,MAAuC;AAC3E,MAAI,CAAC,SAAS,KAAK,EAAG,MAAK,MAAM,mBAAmB;AACpD,SAAO;AACT;AAEA,SAAS,aAAa,OAAgB,MAAsB;AAC1D,MAAI,OAAO,UAAU,YAAY,MAAM,KAAK,EAAE,WAAW;AACvD,SAAK,MAAM,4BAA4B;AACzC,SAAO;AACT;AAEA,SAAS,cAAc,OAAgB,MAAuB;AAC5D,MAAI,OAAO,UAAU,UAAW,MAAK,MAAM,mBAAmB;AAC9D,SAAO;AACT;AAEA,SAAS,aACP,OACA,MACA,OAAqF,CAAC,GAC9E;AACR,MAAI,OAAO,UAAU,YAAY,CAAC,OAAO,SAAS,KAAK,EAAG,MAAK,MAAM,yBAAyB;AAC9F,MAAI,KAAK,WAAW,CAAC,OAAO,UAAU,KAAK,EAAG,MAAK,MAAM,oBAAoB;AAC7E,MAAI,KAAK,QAAQ,UAAa,QAAQ,KAAK,IAAK,MAAK,MAAM,cAAc,KAAK,GAAG,EAAE;AACnF,MAAI,KAAK,QAAQ,UAAa,QAAQ,KAAK,IAAK,MAAK,MAAM,cAAc,KAAK,GAAG,EAAE;AACnF,SAAO;AACT;AAEA,SAAS,kBAAkB,OAAgB,MAAwB;AACjE,MAAI,CAAC,MAAM,QAAQ,KAAK,EAAG,MAAK,MAAM,kBAAkB;AACxD,SAAO,MAAM,IAAI,CAAC,OAAO,UAAU,aAAa,OAAO,GAAG,IAAI,IAAI,KAAK,GAAG,CAAC;AAC7E;AAEA,SAAS,kBAAkB,OAAgB,MAAyC;AAClF,MAAI,CAAC,MAAM,QAAQ,KAAK,EAAG,MAAK,MAAM,kBAAkB;AACxD,SAAO,MAAM,IAAI,CAAC,OAAO,UAAU,aAAa,OAAO,GAAG,IAAI,IAAI,KAAK,GAAG,CAAC;AAC7E;AAEA,SAAS,YAAY,OAAgB,MAAqC;AACxE,QAAM,QAAQ,aAAa,OAAO,IAAI;AACtC,MAAI,CAAC,uBAAuB,SAAS,KAA8B,GAAG;AACpE,SAAK,MAAM,kBAAkB,uBAAuB,KAAK,IAAI,CAAC,EAAE;AAAA,EAClE;AACA,SAAO;AACT;AAEA,SAAS,eAAe,OAAgB,MAAkC;AACxE,MAAI,UAAU,OAAW,QAAO;AAChC,SAAO,aAAa,OAAO,IAAI;AACjC;AAEA,SAAS,iBAAiB,OAAgB,MAAsC;AAC9E,QAAM,MAAM,aAAa,OAAO,IAAI;AACpC,QAAM,MAA8B,CAAC;AACrC,aAAW,CAAC,KAAK,GAAG,KAAK,OAAO,QAAQ,GAAG,EAAG,KAAI,GAAG,IAAI,aAAa,KAAK,GAAG,IAAI,IAAI,GAAG,EAAE;AAC3F,SAAO;AACT;AAEA,SAAS,wBAAwB,OAAgB,MAAiC;AAChF,QAAM,MAAM,aAAa,OAAO,IAAI;AACpC,SAAO;AAAA,IACL,OAAO,aAAa,IAAI,OAAO,GAAG,IAAI,QAAQ;AAAA,IAC9C,SAAS,aAAa,IAAI,SAAS,GAAG,IAAI,UAAU;AAAA,IACpD,SAAS,aAAa,IAAI,SAAS,GAAG,IAAI,UAAU;AAAA,IACpD,GAAI,IAAI,oBAAoB,SACxB,EAAE,iBAAiB,eAAe,IAAI,iBAAiB,GAAG,IAAI,kBAAkB,EAAE,IAClF,CAAC;AAAA,EACP;AACF;AAEO,SAAS,iCAAiC,OAA0C;AACzF,QAAM,MAAM,aAAa,OAAO,UAAU;AAC1C,MAAI,IAAI,kBAAkB,EAAG,MAAK,0BAA0B,WAAW;AACvE,QAAM,OAAO,aAAa,IAAI,MAAM,eAAe;AACnD,QAAM,YAAY,aAAa,IAAI,WAAW,oBAAoB;AAClE,QAAM,WAAW,kBAAkB,IAAI,UAAU,mBAAmB,EAAE,IAAI,CAAC,SAAS,WAAW;AAAA,IAC7F,IAAI,aAAa,QAAQ,IAAI,qBAAqB,KAAK,MAAM;AAAA,IAC7D,aAAa,aAAa,QAAQ,aAAa,qBAAqB,KAAK,eAAe;AAAA,IACxF,kBAAkB;AAAA,MAChB,QAAQ;AAAA,MACR,qBAAqB,KAAK;AAAA,IAC5B;AAAA,EACF,EAAE;AACF,QAAM,OAAO,kBAAkB,IAAI,MAAM,eAAe,EAAE,IAAI,CAAC,KAAK,WAAW;AAAA,IAC7E,IAAI,aAAa,IAAI,IAAI,iBAAiB,KAAK,MAAM;AAAA,IACrD,WAAW,aAAa,IAAI,WAAW,iBAAiB,KAAK,aAAa;AAAA,IAC1E,iBAAiB;AAAA,MACf,IAAI;AAAA,MACJ,iBAAiB,KAAK;AAAA,IACxB;AAAA,IACA,YAAY,aAAa,IAAI,YAAY,iBAAiB,KAAK,cAAc;AAAA,EAC/E,EAAE;AACF,QAAM,YAAY,kBAAkB,IAAI,WAAW,oBAAoB,EAAE;AAAA,IACvE,CAAC,UAAU,WAAW;AAAA,MACpB,IAAI,aAAa,SAAS,IAAI,sBAAsB,KAAK,MAAM;AAAA,MAC/D,OAAO,YAAY,SAAS,OAAO,sBAAsB,KAAK,SAAS;AAAA,MACvE,MAAM,kBAAkB,SAAS,MAAM,sBAAsB,KAAK,QAAQ;AAAA,MAC1E,2BAA2B;AAAA,QACzB,SAAS;AAAA,QACT,sBAAsB,KAAK;AAAA,MAC7B;AAAA,IACF;AAAA,EACF;AACA,QAAM,UAAU,aAAa,IAAI,SAAS,kBAAkB;AAC5D,MAAI,SAAS,WAAW,EAAG,MAAK,qBAAqB,mCAAmC;AACxF,MAAI,KAAK,WAAW,EAAG,MAAK,iBAAiB,+BAA+B;AAC5E,MAAI,UAAU,WAAW,EAAG,MAAK,sBAAsB,oCAAoC;AAC3F;AAAA,IACE,SAAS,IAAI,CAAC,YAAY,QAAQ,EAAE;AAAA,IACpC;AAAA,EACF;AACA;AAAA,IACE,KAAK,IAAI,CAAC,QAAQ,IAAI,EAAE;AAAA,IACxB;AAAA,EACF;AACA;AAAA,IACE,UAAU,IAAI,CAAC,aAAa,SAAS,EAAE;AAAA,IACvC;AAAA,EACF;AACA,QAAM,aAAa,IAAI,IAAI,SAAS,IAAI,CAAC,YAAY,QAAQ,EAAE,CAAC;AAChE,aAAW,OAAO,MAAM;AACtB,QAAI,CAAC,WAAW,IAAI,IAAI,SAAS;AAC/B,WAAK,iBAAiB,IAAI,EAAE,cAAc,mBAAmB,IAAI,SAAS,EAAE;AAAA,EAChF;AACA,SAAO;AAAA,IACL,eAAe;AAAA,IACf,WAAW,aAAa,IAAI,WAAW,oBAAoB;AAAA,IAC3D,aAAa,aAAa,IAAI,aAAa,sBAAsB;AAAA,IACjE,MAAM;AAAA,MACJ,KAAK,aAAa,KAAK,KAAK,mBAAmB;AAAA,MAC/C,QAAQ,aAAa,KAAK,QAAQ,sBAAsB;AAAA,MACxD,QAAQ,aAAa,KAAK,QAAQ,sBAAsB;AAAA,IAC1D;AAAA,IACA,WAAW;AAAA,MACT,WAAW,aAAa,UAAU,WAAW,8BAA8B;AAAA,MAC3E,cAAc,aAAa,UAAU,cAAc,iCAAiC;AAAA,MACpF,gBAAgB,aAAa,UAAU,gBAAgB,mCAAmC;AAAA,MAC1F,SAAS,aAAa,UAAU,SAAS,4BAA4B;AAAA,MACrE,GAAI,UAAU,eAAe,SACzB,EAAE,YAAY,aAAa,UAAU,YAAY,+BAA+B,EAAE,IAClF,CAAC;AAAA,IACP;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS;AAAA,MACP,QAAQ,aAAa,QAAQ,QAAQ,2BAA2B,EAAE,KAAK,EAAE,CAAC;AAAA,MAC1E,UAAU,aAAa,QAAQ,UAAU,6BAA6B;AAAA,QACpE,KAAK;AAAA,QACL,SAAS;AAAA,MACX,CAAC;AAAA,MACD,WAAW,aAAa,QAAQ,WAAW,8BAA8B;AAAA,QACvE,KAAK;AAAA,QACL,SAAS;AAAA,MACX,CAAC;AAAA,IACH;AAAA,IACA,qBAAqB,aAAa,IAAI,qBAAqB,8BAA8B;AAAA,EAC3F;AACF;AAEO,SAAS,+BAA+B,OAAwC;AACrF,QAAM,MAAM,aAAa,OAAO,QAAQ;AACxC,MAAI,IAAI,kBAAkB,EAAG,MAAK,wBAAwB,WAAW;AACrE,QAAM,eAAe,aAAa,IAAI,cAAc,qBAAqB;AACzE,QAAM,QAAQ,aAAa,IAAI,OAAO,cAAc;AACpD,QAAM,UAAU,aAAa,IAAI,SAAS,gBAAgB;AAC1D,QAAM,UAAU,aAAa,IAAI,SAAS,gBAAgB;AAC1D,QAAM,QAAQ,aAAa,IAAI,OAAO,cAAc;AACpD,QAAM,YAAY,aAAa,IAAI,WAAW,kBAAkB;AAChE,QAAM,YAAY,aAAa,IAAI,WAAW,kBAAkB;AAChE,QAAM,SAAiC;AAAA,IACrC,eAAe;AAAA,IACf,WAAW,aAAa,IAAI,WAAW,kBAAkB;AAAA,IACzD,aAAa,aAAa,IAAI,aAAa,oBAAoB;AAAA,IAC/D,OAAO,aAAa,IAAI,OAAO,cAAc;AAAA,IAC7C,YAAY,aAAa,IAAI,YAAY,mBAAmB;AAAA,IAC5D,OAAO,YAAY,IAAI,OAAO,cAAc;AAAA,IAC5C,OAAO,aAAa,IAAI,OAAO,cAAc;AAAA,IAC7C,KAAK,aAAa,IAAI,KAAK,cAAc,EAAE,KAAK,GAAG,SAAS,KAAK,CAAC;AAAA,IAClE,cAAc;AAAA,MACZ,IAAI,aAAa,aAAa,IAAI,wBAAwB;AAAA,MAC1D,MAAM,aAAa,aAAa,MAAM,0BAA0B;AAAA,MAChE,MAAM,aAAa,aAAa,MAAM,0BAA0B;AAAA,MAChE,UAAU,wBAAwB,aAAa,UAAU,8BAA8B;AAAA,MACvF,UAAU,wBAAwB,aAAa,UAAU,8BAA8B;AAAA,IACzF;AAAA,IACA,OAAO;AAAA,MACL,UAAU,aAAa,MAAM,UAAU,uBAAuB;AAAA,MAC9D,IAAI,aAAa,MAAM,IAAI,iBAAiB;AAAA,IAC9C;AAAA,IACA,SAAS;AAAA,MACP,MAAM,aAAa,QAAQ,MAAM,qBAAqB;AAAA,MACtD,SAAS,aAAa,QAAQ,SAAS,wBAAwB;AAAA,IACjE;AAAA,IACA,SAAS;AAAA,MACP,MAAM,cAAc,QAAQ,MAAM,qBAAqB;AAAA,MACvD,OAAO,aAAa,QAAQ,OAAO,wBAAwB,EAAE,KAAK,GAAG,KAAK,EAAE,CAAC;AAAA,MAC7E,YAAY,iBAAiB,QAAQ,YAAY,2BAA2B;AAAA,MAC5E,aACE,QAAQ,gBAAgB,OACpB,OACA,aAAa,QAAQ,aAAa,4BAA4B;AAAA,IACtE;AAAA,IACA,OAAO;AAAA,MACL,aAAa,aAAa,MAAM,aAAa,4BAA4B;AAAA,QACvE,KAAK;AAAA,QACL,SAAS;AAAA,MACX,CAAC;AAAA,MACD,cAAc,aAAa,MAAM,cAAc,6BAA6B;AAAA,QAC1E,KAAK;AAAA,QACL,SAAS;AAAA,MACX,CAAC;AAAA,MACD,SAAS,aAAa,MAAM,SAAS,wBAAwB,EAAE,KAAK,EAAE,CAAC;AAAA,MACvE,QAAQ,aAAa,MAAM,QAAQ,uBAAuB,EAAE,KAAK,GAAG,SAAS,KAAK,CAAC;AAAA,MACnF,WAAW,aAAa,MAAM,WAAW,0BAA0B,EAAE,KAAK,GAAG,SAAS,KAAK,CAAC;AAAA,IAC9F;AAAA,IACA,WAAW;AAAA,MACT,aAAa,cAAc,UAAU,aAAa,8BAA8B;AAAA,MAChF,YAAY,cAAc,UAAU,YAAY,6BAA6B;AAAA,MAC7E,cAAc,cAAc,UAAU,cAAc,+BAA+B;AAAA,MACnF,YAAY,cAAc,UAAU,YAAY,6BAA6B;AAAA,MAC7E,QAAQ,cAAc,UAAU,QAAQ,yBAAyB;AAAA,MACjE,qBAAqB;AAAA,QACnB,UAAU;AAAA,QACV;AAAA,MACF;AAAA,IACF;AAAA,IACA,WAAW;AAAA,MACT,SAAS,aAAa,UAAU,SAAS,0BAA0B;AAAA,MACnE,QAAQ,aAAa,UAAU,QAAQ,yBAAyB;AAAA,MAChE,MAAM,aAAa,UAAU,MAAM,uBAAuB;AAAA,MAC1D,QAAQ,aAAa,UAAU,QAAQ,yBAAyB;AAAA,MAChE,WAAW,aAAa,UAAU,WAAW,4BAA4B;AAAA,IAC3E;AAAA,EACF;AACA,MAAI,OAAO,UAAU,eAAe,OAAO,MAAM,cAAc,OAAO,MAAM,iBAAiB,GAAG;AAC9F,SAAK,gBAAgB,kDAAkD;AAAA,EACzE;AACA,SAAO;AACT;AAEO,SAAS,kCAAkC,QAA0C;AAC1F,QAAM,WAAqB,CAAC;AAC5B,aAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,OAAO,SAAS,GAAG;AAC3D,QAAI,CAAC,MAAO,UAAS,KAAK,GAAG,OAAO,KAAK,IAAI,GAAG,QAAQ;AAAA,EAC1D;AACA,SAAO;AACT;AAEO,SAAS,4BAA4B,MAAwC;AAClF,QAAM,QAAQ,aAAa,MAAM,MAAM,EACpC,MAAM,IAAI,EACV,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC,EACzB,OAAO,OAAO;AACjB,QAAM,UAAoC,CAAC;AAC3C,aAAW,CAAC,OAAO,IAAI,KAAK,MAAM,QAAQ,GAAG;AAC3C,QAAI;AACF,cAAQ,KAAK,+BAA+B,KAAK,MAAM,IAAI,CAAC,CAAC;AAAA,IAC/D,SAAS,KAAK;AACZ,0BAAoB,GAAG,IAAI,IAAI,QAAQ,CAAC,IAAI,GAAG;AAAA,IACjD;AAAA,EACF;AACA,SAAO;AACT;AAEO,SAAS,6BAA6B,MAAwC;AACnF,MAAI;AACF,WAAO,iCAAiC,KAAK,MAAM,aAAa,MAAM,MAAM,CAAC,CAAC;AAAA,EAChF,SAAS,KAAK;AACZ,wBAAoB,MAAM,GAAG;AAAA,EAC/B;AACF;AAEO,SAAS,4BACd,OACkC;AAClC,QAAM,WAAW,6BAA6B,MAAM,YAAY;AAChE,QAAM,UAAU,4BAA4B,MAAM,WAAW;AAC7D,QAAM,uBAAuB,GAAG,SAAS,SAAS,IAAI,SAAS,WAAW;AAC1E,QAAM,oBAAoB,QAAQ,QAAQ,iCAAiC;AAC3E,QAAM,mBACJ,MAAM,mBAAmB,QACrB,CAAC,IACD,QAAQ;AAAA,IAAQ,CAAC,WACf,0BAA0B,QAAQ,MAAM,gBAAgB,QAAQ,MAAM,WAAW,CAAC;AAAA,EACpF;AACN,aAAW,CAAC,OAAO,MAAM,KAAK,QAAQ,QAAQ,GAAG;AAC/C,UAAM,qBAAqB,GAAG,OAAO,SAAS,IAAI,OAAO,WAAW;AACpE,QAAI,uBAAuB,sBAAsB;AAC/C;AAAA,QACE,WAAW,KAAK;AAAA,QAChB,qBAAqB,kBAAkB,4BAA4B,oBAAoB;AAAA,MACzF;AAAA,IACF;AACA,QAAI,CAAC,SAAS,KAAK,KAAK,CAAC,QAAQ,IAAI,OAAO,OAAO,KAAK;AACtD,WAAK,WAAW,KAAK,WAAW,eAAe,OAAO,KAAK,EAAE;AAC/D,QAAI,CAAC,SAAS,UAAU,KAAK,CAAC,aAAa,SAAS,OAAO,OAAO,UAAU,GAAG;AAC7E,WAAK,WAAW,KAAK,gBAAgB,oBAAoB,OAAO,UAAU,EAAE;AAAA,IAC9E;AAAA,EACF;AACA,SAAO;AAAA,IACL,cAAc,MAAM;AAAA,IACpB,aAAa,MAAM;AAAA,IACnB,SAAS,QAAQ;AAAA,IACjB,UAAU,aAAa,QAAQ,IAAI,CAAC,WAAW,OAAO,SAAS,CAAC;AAAA,IAChE,YAAY,aAAa,QAAQ,IAAI,CAAC,WAAW,OAAO,WAAW,CAAC;AAAA,IACpE,MAAM,aAAa,QAAQ,IAAI,CAAC,WAAW,OAAO,KAAK,CAAC;AAAA,IACxD,WAAW,aAAa,QAAQ,IAAI,CAAC,WAAW,OAAO,UAAU,CAAC;AAAA,IAClE,QAAQ,QAAQ,OAAO,CAAC,WAAW,OAAO,QAAQ,IAAI,EAAE;AAAA,IACxD,QAAQ,QAAQ,OAAO,CAAC,WAAW,CAAC,OAAO,QAAQ,IAAI,EAAE;AAAA,IACzD,aAAa,IAAI,SAAS,CAAC,WAAW,OAAO,MAAM,WAAW;AAAA,IAC9D,cAAc,IAAI,SAAS,CAAC,WAAW,OAAO,MAAM,YAAY;AAAA,IAChE,SAAS,IAAI,SAAS,CAAC,WAAW,OAAO,MAAM,OAAO;AAAA,IACtD,QAAQ,IAAI,SAAS,CAAC,WAAW,OAAO,MAAM,MAAM;AAAA,IACpD;AAAA,IACA;AAAA,EACF;AACF;AAEA,SAAS,0BAA0B,QAAgC,cAAgC;AACjG,QAAM,UAAoB,CAAC;AAC3B,aAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,OAAO,SAAS,GAAG;AAC3D,UAAM,OAAO,oBAAoB,OAAO,YAAY;AACpD,QAAI,CAAC,WAAW,IAAI,EAAG,SAAQ,KAAK,GAAG,OAAO,KAAK,IAAI,GAAG,IAAI,KAAK,EAAE;AAAA,EACvE;AACA,SAAO;AACT;AAEA,SAAS,oBAAoB,OAAe,cAA8B;AACxE,SAAO,MAAM,WAAW,GAAG,IAAI,QAAQ,KAAK,cAAc,KAAK;AACjE;AAEA,SAAS,aAAa,QAA2B,MAAoB;AACnE,QAAM,OAAO,oBAAI,IAAY;AAC7B,aAAW,SAAS,QAAQ;AAC1B,QAAI,KAAK,IAAI,KAAK,EAAG,MAAK,MAAM,aAAa,KAAK,EAAE;AACpD,SAAK,IAAI,KAAK;AAAA,EAChB;AACF;AAEA,SAAS,aAAa,QAAqC;AACzD,SAAO,CAAC,GAAG,IAAI,IAAI,MAAM,CAAC,EAAE,KAAK;AACnC;AAEA,SAAS,IAAO,OAAqB,IAAiC;AACpE,SAAO,MAAM,OAAO,CAAC,OAAO,SAAS,QAAQ,GAAG,IAAI,GAAG,CAAC;AAC1D;AAEO,SAAS,8BACd,QACsC;AACtC,QAAM,eAAe,KAAK,QAAQ,iCAAiC;AACnE,QAAM,cAAc,KAAK,QAAQ,iCAAiC;AAClE,MACE,WAAW,YAAY,KACvB,SAAS,YAAY,EAAE,OAAO,KAC9B,WAAW,WAAW,KACtB,SAAS,WAAW,EAAE,OAAO,GAC7B;AACA,WAAO,EAAE,cAAc,YAAY;AAAA,EACrC;AACA,SAAO;AACT;","names":[]}
@@ -1 +0,0 @@
1
- {"version":3,"sources":["../src/agent-profile-cell.ts"],"sourcesContent":["import type { AgentProfile } from '@tangle-network/agent-interface'\nimport { ValidationError } from './errors'\nimport { hashJson } from './pre-registration'\n\nexport type AgentProfileCellSchemaVersion = 'agent-profile-cell/v1'\n\nexport type AgentProfileJson =\n | string\n | number\n | boolean\n | null\n | AgentProfileJson[]\n | { [key: string]: AgentProfileJson }\n\nexport type AgentProfileDimensionValue = string | number | boolean | null\n\nexport interface AgentProfileSource {\n /** Runtime/profile contract being fingerprinted, e.g. `agent-interface-profile`. */\n kind: string\n /** sha256 over the canonical source profile object. */\n hash: string\n}\n\nexport interface AgentProfileSourceInput {\n kind: string\n /** Precomputed sha256 for callers that already sign their profile artifact. */\n hash?: string\n /** Full canonical runtime profile; hashed and then discarded from the cell. */\n profile?: AgentProfileJson\n}\n\nexport interface AgentProfileHarness {\n id: string\n version?: string\n hash?: string\n}\n\nexport interface AgentProfileCellInput {\n profileId: string\n sourceProfile: AgentProfileSourceInput\n harness?: AgentProfileHarness\n model?: string\n promptHash?: string\n dimensions?: Record<string, AgentProfileDimensionValue>\n}\n\nexport interface AgentProfileCell {\n schemaVersion: AgentProfileCellSchemaVersion\n cellId: string\n profileId: string\n sourceProfile: AgentProfileSource\n harness?: AgentProfileHarness\n model?: string\n promptHash?: string\n dimensions?: Record<string, AgentProfileDimensionValue>\n}\n\nexport class AgentProfileCellValidationError extends ValidationError {\n readonly path: string\n constructor(message: string, path = '') {\n super(path ? `${message} (at ${path})` : message)\n this.path = path\n }\n}\n\nconst SHA256_HEX = /^[0-9a-f]{64}$/\nconst CELL_ID = /^agent-profile-cell:sha256:[0-9a-f]{64}$/\n\nexport async function buildAgentProfileCell(\n input: AgentProfileCellInput,\n): Promise<AgentProfileCell> {\n const material = await normalizeAgentProfileCellInput(input)\n const cellId = `agent-profile-cell:sha256:${await hashJson(material)}`\n return { ...material, cellId }\n}\n\nexport function agentProfileCellHashMaterial(\n cell: AgentProfileCell,\n): Omit<AgentProfileCell, 'cellId'> {\n const { cellId: _cellId, ...material } = cell\n void _cellId\n return normalizeAgentProfileCell(material)\n}\n\nexport async function verifyAgentProfileCell(cell: AgentProfileCell): Promise<boolean> {\n validateAgentProfileCell(cell)\n return (\n cell.cellId ===\n `agent-profile-cell:sha256:${await hashJson(agentProfileCellHashMaterial(cell))}`\n )\n}\n\nexport function validateAgentProfileCell(input: unknown): AgentProfileCell {\n if (input === null || typeof input !== 'object') {\n throw new AgentProfileCellValidationError('expected object')\n }\n const obj = input as Record<string, unknown>\n expectLiteral(obj.schemaVersion, 'agent-profile-cell/v1', 'schemaVersion')\n if (typeof obj.cellId !== 'string' || !CELL_ID.test(obj.cellId)) {\n throw new AgentProfileCellValidationError(\n 'cellId must match agent-profile-cell:sha256:<64 lowercase hex chars>',\n 'cellId',\n )\n }\n expectString(obj.profileId, 'profileId')\n validateSource(obj.sourceProfile, 'sourceProfile')\n if (obj.harness !== undefined) validateHarness(obj.harness, 'harness')\n if (obj.model !== undefined) expectString(obj.model, 'model')\n if (obj.promptHash !== undefined) expectString(obj.promptHash, 'promptHash')\n if (obj.dimensions !== undefined) validateDimensions(obj.dimensions, 'dimensions')\n return input as AgentProfileCell\n}\n\nexport function requireAgentProfileCell(record: {\n runId: string\n agentProfile?: AgentProfileCell\n}): AgentProfileCell {\n if (!record.agentProfile) {\n throw new AgentProfileCellValidationError(\n `run \"${record.runId}\" is missing agentProfile; profile-cell grouping requires explicit profile identity`,\n 'agentProfile',\n )\n }\n return validateAgentProfileCell(record.agentProfile)\n}\n\nexport function agentProfileCellKey(record: {\n runId: string\n agentProfile?: AgentProfileCell\n}): string {\n return requireAgentProfileCell(record).cellId\n}\n\nexport async function assertRunAgentProfileCell(record: {\n runId: string\n model: string\n promptHash: string\n agentProfile?: AgentProfileCell\n}): Promise<AgentProfileCell> {\n const profile = requireAgentProfileCell(record)\n if (!(await verifyAgentProfileCell(profile))) {\n throw new AgentProfileCellValidationError(\n `run \"${record.runId}\" has an agentProfile.cellId that does not match its content`,\n 'agentProfile.cellId',\n )\n }\n if (profile.model !== undefined && profile.model !== record.model) {\n throw new AgentProfileCellValidationError(\n `run \"${record.runId}\" agentProfile.model \"${profile.model}\" does not match model \"${record.model}\"`,\n 'agentProfile.model',\n )\n }\n if (profile.promptHash !== undefined && profile.promptHash !== record.promptHash) {\n throw new AgentProfileCellValidationError(\n `run \"${record.runId}\" agentProfile.promptHash \"${profile.promptHash}\" does not match promptHash \"${record.promptHash}\"`,\n 'agentProfile.promptHash',\n )\n }\n return profile\n}\n\nexport function groupRunsByAgentProfileCell<\n T extends { runId: string; agentProfile?: AgentProfileCell },\n>(records: readonly T[]): Map<string, T[]> {\n const groups = new Map<string, T[]>()\n for (const record of records) {\n const key = agentProfileCellKey(record)\n const bucket = groups.get(key)\n if (bucket) bucket.push(record)\n else groups.set(key, [record])\n }\n return groups\n}\n\nasync function normalizeAgentProfileCellInput(\n input: AgentProfileCellInput,\n): Promise<Omit<AgentProfileCell, 'cellId'>> {\n return normalizeAgentProfileCell({\n schemaVersion: 'agent-profile-cell/v1',\n profileId: input.profileId,\n sourceProfile: await normalizeSourceInput(input.sourceProfile),\n harness: input.harness,\n model: input.model,\n promptHash: input.promptHash,\n dimensions: input.dimensions,\n })\n}\n\nfunction normalizeAgentProfileCell(\n input: Omit<AgentProfileCell, 'cellId'>,\n): Omit<AgentProfileCell, 'cellId'> {\n return compactObject({\n schemaVersion: 'agent-profile-cell/v1' as const,\n profileId: requireNonEmpty(input.profileId, 'profileId'),\n sourceProfile: normalizeSource(input.sourceProfile),\n harness: input.harness ? normalizeHarness(input.harness, 'harness') : undefined,\n model: optionalNonEmpty(input.model, 'model'),\n promptHash: optionalNonEmpty(input.promptHash, 'promptHash'),\n dimensions: input.dimensions\n ? nonEmptyRecord(normalizeDimensions(input.dimensions))\n : undefined,\n })\n}\n\nasync function normalizeSourceInput(input: AgentProfileSourceInput): Promise<AgentProfileSource> {\n const kind = requireNonEmpty(input.kind, 'sourceProfile.kind')\n if (input.hash !== undefined && input.profile !== undefined) {\n throw new AgentProfileCellValidationError(\n 'sourceProfile must provide either hash or profile, not both',\n 'sourceProfile',\n )\n }\n if (input.hash !== undefined) {\n return { kind, hash: requireSha256Hex(input.hash, 'sourceProfile.hash') }\n }\n if (input.profile === undefined) {\n throw new AgentProfileCellValidationError(\n 'sourceProfile must provide hash or profile',\n 'sourceProfile',\n )\n }\n assertJson(input.profile, 'sourceProfile.profile')\n return { kind, hash: await hashJson(input.profile) }\n}\n\nfunction normalizeSource(input: AgentProfileSource): AgentProfileSource {\n return {\n kind: requireNonEmpty(input.kind, 'sourceProfile.kind'),\n hash: requireSha256Hex(input.hash, 'sourceProfile.hash'),\n }\n}\n\nfunction normalizeHarness(input: AgentProfileHarness, path: string): AgentProfileHarness {\n return compactObject({\n id: requireNonEmpty(input.id, `${path}.id`),\n version: optionalNonEmpty(input.version, `${path}.version`),\n hash: optionalNonEmpty(input.hash, `${path}.hash`),\n })\n}\n\nfunction normalizeDimensions(\n input: Record<string, AgentProfileDimensionValue>,\n): Record<string, AgentProfileDimensionValue> {\n const out: Record<string, AgentProfileDimensionValue> = {}\n for (const key of Object.keys(input).sort()) {\n const value = input[key]\n requireNonEmpty(key, 'dimensions.<key>')\n if (\n value !== null &&\n typeof value !== 'string' &&\n typeof value !== 'number' &&\n typeof value !== 'boolean'\n ) {\n throw new AgentProfileCellValidationError(\n 'expected primitive dimension value',\n `dimensions.${key}`,\n )\n }\n if (typeof value === 'number' && !Number.isFinite(value)) {\n throw new AgentProfileCellValidationError('expected finite number', `dimensions.${key}`)\n }\n out[key] = value\n }\n return out\n}\n\nfunction compactObject<T extends Record<string, unknown>>(input: T): T {\n const out: Record<string, unknown> = {}\n for (const [key, value] of Object.entries(input)) {\n if (value !== undefined) out[key] = value\n }\n return out as T\n}\n\nfunction nonEmptyRecord<T extends Record<string, unknown>>(input: T): T | undefined {\n return Object.keys(input).length > 0 ? input : undefined\n}\n\nfunction validateSource(value: unknown, path: string): void {\n if (value === null || typeof value !== 'object' || Array.isArray(value)) {\n throw new AgentProfileCellValidationError('expected object', path)\n }\n const rec = value as Record<string, unknown>\n expectString(rec.kind, `${path}.kind`)\n requireSha256Hex(rec.hash, `${path}.hash`)\n}\n\nfunction validateHarness(value: unknown, path: string): void {\n if (value === null || typeof value !== 'object' || Array.isArray(value)) {\n throw new AgentProfileCellValidationError('expected object', path)\n }\n const rec = value as Record<string, unknown>\n expectString(rec.id, `${path}.id`)\n if (rec.version !== undefined) expectString(rec.version, `${path}.version`)\n if (rec.hash !== undefined) expectString(rec.hash, `${path}.hash`)\n}\n\nfunction validateDimensions(value: unknown, path: string): void {\n if (value === null || typeof value !== 'object' || Array.isArray(value)) {\n throw new AgentProfileCellValidationError('expected object', path)\n }\n normalizeDimensions(value as Record<string, AgentProfileDimensionValue>)\n}\n\nfunction assertJson(value: AgentProfileJson, path: string): void {\n if (value === null) return\n const type = typeof value\n if (type === 'string' || type === 'boolean') return\n if (type === 'number') {\n if (!Number.isFinite(value)) {\n throw new AgentProfileCellValidationError('expected finite number', path)\n }\n return\n }\n if (Array.isArray(value)) {\n value.forEach((item, index) => {\n assertJson(item, `${path}[${index}]`)\n })\n return\n }\n if (type === 'object') {\n for (const [key, nested] of Object.entries(value)) {\n requireNonEmpty(key, `${path}.<key>`)\n assertJson(nested, `${path}.${key}`)\n }\n return\n }\n throw new AgentProfileCellValidationError('expected JSON-compatible value', path)\n}\n\nfunction expectLiteral(value: unknown, expected: string, path: string): void {\n if (value !== expected) {\n throw new AgentProfileCellValidationError(`expected ${expected}`, path)\n }\n}\n\nfunction expectString(value: unknown, path: string): void {\n if (typeof value !== 'string' || value.length === 0) {\n throw new AgentProfileCellValidationError('expected non-empty string', path)\n }\n}\n\nfunction requireNonEmpty(value: string, path: string): string {\n if (typeof value !== 'string' || value.length === 0) {\n throw new AgentProfileCellValidationError('expected non-empty string', path)\n }\n return value\n}\n\nfunction optionalNonEmpty(value: string | undefined, path: string): string | undefined {\n if (value === undefined) return undefined\n return requireNonEmpty(value, path)\n}\n\nfunction requireSha256Hex(value: unknown, path: string): string {\n if (typeof value !== 'string' || !SHA256_HEX.test(value)) {\n throw new AgentProfileCellValidationError('expected 64 lowercase sha256 hex chars', path)\n }\n return value\n}\n\n// ── Consumer helpers ─────────────────────────────────────────────────\n//\n// Boilerplate every product consuming `buildAgentProfileCell` used to duplicate:\n//\n// 1. A `JSON.parse(JSON.stringify(value))` helper that canonicalizes an\n// arbitrary `@tangle-network/agent-interface` `AgentProfile` into the recursive\n// `AgentProfileJson` shape, with a fail-loud error when the profile\n// is not JSON-serializable.\n//\n// 2. The magic string `'agent-interface-profile'` for `sourceProfile.kind`.\n//\n// Both belong here so the cross-product cell join (same canonical profile\n// hashes to the same `sourceProfile.hash` across products) is enforced by\n// the type system, not by every consumer remembering to do it right.\n// See blueprint-agent issue tangle-network/agent-eval#82.\n\n/** Canonical `sourceProfile.kind` values. Two products fingerprinting the\n * same canonical profile MUST use the same kind for their cells to share\n * `sourceProfile.hash`. Extend rather than create new strings — adding a\n * new kind is a deliberate cross-product schema change. */\nexport const AGENT_PROFILE_KINDS = {\n /** A profile declared via `defineAgentProfile(...)` from\n * `@tangle-network/agent-interface`. The default kind for router-backed\n * and sandbox-backed products. */\n AGENT_INTERFACE_PROFILE: 'agent-interface-profile',\n} as const\n\nexport type AgentProfileKind = (typeof AGENT_PROFILE_KINDS)[keyof typeof AGENT_PROFILE_KINDS]\n\n/** Canonicalize an arbitrary value into `AgentProfileJson` by JSON\n * round-trip. Throws when the value contains anything not representable\n * as JSON (functions, BigInt, cycles) — non-portable profiles fail loud\n * rather than silently dropping fields. */\nexport function toAgentProfileJson(value: unknown): AgentProfileJson {\n let serialized: string | undefined\n try {\n serialized = JSON.stringify(value)\n } catch (err) {\n throw new AgentProfileCellValidationError(\n `agent profile must be JSON-serializable: ${err instanceof Error ? err.message : String(err)}`,\n 'sourceProfile.profile',\n )\n }\n if (serialized === undefined) {\n throw new AgentProfileCellValidationError(\n 'agent profile must be JSON-serializable (got undefined after JSON.stringify)',\n 'sourceProfile.profile',\n )\n }\n return JSON.parse(serialized) as AgentProfileJson\n}\n\n/** Canonical AgentProfile shape required when deriving a stable cell id. */\nexport type AgentInterfaceProfileLike = AgentProfile & { name: string; version: string }\n\n/** Higher-level helper that hard-codes the canonical\n * `agent-interface-profile` kind plus the JSON canonicalization. Equivalent\n * to calling `buildAgentProfileCell` with `profileId = \\`${name}@${version}\\``\n * and `sourceProfile = { kind: AGENT_INTERFACE_PROFILE, profile: <round-tripped> }`.\n *\n * Use this from any product consuming an agent-interface `AgentProfile`; the\n * manual `buildAgentProfileCell` call is reserved for advanced cases\n * (custom kinds, pre-computed source hashes, alternate profileId\n * conventions). */\nexport async function buildAgentInterfaceProfileCell(\n profile: AgentInterfaceProfileLike,\n input: Omit<AgentProfileCellInput, 'profileId' | 'sourceProfile'>,\n): Promise<AgentProfileCell> {\n if (!profile || typeof profile !== 'object') {\n throw new AgentProfileCellValidationError('AgentProfile must be an object', 'profile')\n }\n if (typeof profile.name !== 'string' || profile.name.length === 0) {\n throw new AgentProfileCellValidationError(\n 'AgentProfile must have a non-empty `name`',\n 'profile.name',\n )\n }\n if (typeof profile.version !== 'string' || profile.version.length === 0) {\n throw new AgentProfileCellValidationError(\n 'AgentProfile must have a non-empty `version`',\n 'profile.version',\n )\n }\n return buildAgentProfileCell({\n ...input,\n profileId: `${profile.name}@${profile.version}`,\n sourceProfile: {\n kind: AGENT_PROFILE_KINDS.AGENT_INTERFACE_PROFILE,\n profile: toAgentProfileJson(profile),\n },\n })\n}\n"],"mappings":";;;;;;;;AAyDO,IAAM,kCAAN,cAA8C,gBAAgB;AAAA,EAC1D;AAAA,EACT,YAAY,SAAiB,OAAO,IAAI;AACtC,UAAM,OAAO,GAAG,OAAO,QAAQ,IAAI,MAAM,OAAO;AAChD,SAAK,OAAO;AAAA,EACd;AACF;AAEA,IAAM,aAAa;AACnB,IAAM,UAAU;AAEhB,eAAsB,sBACpB,OAC2B;AAC3B,QAAM,WAAW,MAAM,+BAA+B,KAAK;AAC3D,QAAM,SAAS,6BAA6B,MAAM,SAAS,QAAQ,CAAC;AACpE,SAAO,EAAE,GAAG,UAAU,OAAO;AAC/B;AAEO,SAAS,6BACd,MACkC;AAClC,QAAM,EAAE,QAAQ,SAAS,GAAG,SAAS,IAAI;AACzC,OAAK;AACL,SAAO,0BAA0B,QAAQ;AAC3C;AAEA,eAAsB,uBAAuB,MAA0C;AACrF,2BAAyB,IAAI;AAC7B,SACE,KAAK,WACL,6BAA6B,MAAM,SAAS,6BAA6B,IAAI,CAAC,CAAC;AAEnF;AAEO,SAAS,yBAAyB,OAAkC;AACzE,MAAI,UAAU,QAAQ,OAAO,UAAU,UAAU;AAC/C,UAAM,IAAI,gCAAgC,iBAAiB;AAAA,EAC7D;AACA,QAAM,MAAM;AACZ,gBAAc,IAAI,eAAe,yBAAyB,eAAe;AACzE,MAAI,OAAO,IAAI,WAAW,YAAY,CAAC,QAAQ,KAAK,IAAI,MAAM,GAAG;AAC/D,UAAM,IAAI;AAAA,MACR;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACA,eAAa,IAAI,WAAW,WAAW;AACvC,iBAAe,IAAI,eAAe,eAAe;AACjD,MAAI,IAAI,YAAY,OAAW,iBAAgB,IAAI,SAAS,SAAS;AACrE,MAAI,IAAI,UAAU,OAAW,cAAa,IAAI,OAAO,OAAO;AAC5D,MAAI,IAAI,eAAe,OAAW,cAAa,IAAI,YAAY,YAAY;AAC3E,MAAI,IAAI,eAAe,OAAW,oBAAmB,IAAI,YAAY,YAAY;AACjF,SAAO;AACT;AAEO,SAAS,wBAAwB,QAGnB;AACnB,MAAI,CAAC,OAAO,cAAc;AACxB,UAAM,IAAI;AAAA,MACR,QAAQ,OAAO,KAAK;AAAA,MACpB;AAAA,IACF;AAAA,EACF;AACA,SAAO,yBAAyB,OAAO,YAAY;AACrD;AAEO,SAAS,oBAAoB,QAGzB;AACT,SAAO,wBAAwB,MAAM,EAAE;AACzC;AAEA,eAAsB,0BAA0B,QAKlB;AAC5B,QAAM,UAAU,wBAAwB,MAAM;AAC9C,MAAI,CAAE,MAAM,uBAAuB,OAAO,GAAI;AAC5C,UAAM,IAAI;AAAA,MACR,QAAQ,OAAO,KAAK;AAAA,MACpB;AAAA,IACF;AAAA,EACF;AACA,MAAI,QAAQ,UAAU,UAAa,QAAQ,UAAU,OAAO,OAAO;AACjE,UAAM,IAAI;AAAA,MACR,QAAQ,OAAO,KAAK,yBAAyB,QAAQ,KAAK,2BAA2B,OAAO,KAAK;AAAA,MACjG;AAAA,IACF;AAAA,EACF;AACA,MAAI,QAAQ,eAAe,UAAa,QAAQ,eAAe,OAAO,YAAY;AAChF,UAAM,IAAI;AAAA,MACR,QAAQ,OAAO,KAAK,8BAA8B,QAAQ,UAAU,gCAAgC,OAAO,UAAU;AAAA,MACrH;AAAA,IACF;AAAA,EACF;AACA,SAAO;AACT;AAEO,SAAS,4BAEd,SAAyC;AACzC,QAAM,SAAS,oBAAI,IAAiB;AACpC,aAAW,UAAU,SAAS;AAC5B,UAAM,MAAM,oBAAoB,MAAM;AACtC,UAAM,SAAS,OAAO,IAAI,GAAG;AAC7B,QAAI,OAAQ,QAAO,KAAK,MAAM;AAAA,QACzB,QAAO,IAAI,KAAK,CAAC,MAAM,CAAC;AAAA,EAC/B;AACA,SAAO;AACT;AAEA,eAAe,+BACb,OAC2C;AAC3C,SAAO,0BAA0B;AAAA,IAC/B,eAAe;AAAA,IACf,WAAW,MAAM;AAAA,IACjB,eAAe,MAAM,qBAAqB,MAAM,aAAa;AAAA,IAC7D,SAAS,MAAM;AAAA,IACf,OAAO,MAAM;AAAA,IACb,YAAY,MAAM;AAAA,IAClB,YAAY,MAAM;AAAA,EACpB,CAAC;AACH;AAEA,SAAS,0BACP,OACkC;AAClC,SAAO,cAAc;AAAA,IACnB,eAAe;AAAA,IACf,WAAW,gBAAgB,MAAM,WAAW,WAAW;AAAA,IACvD,eAAe,gBAAgB,MAAM,aAAa;AAAA,IAClD,SAAS,MAAM,UAAU,iBAAiB,MAAM,SAAS,SAAS,IAAI;AAAA,IACtE,OAAO,iBAAiB,MAAM,OAAO,OAAO;AAAA,IAC5C,YAAY,iBAAiB,MAAM,YAAY,YAAY;AAAA,IAC3D,YAAY,MAAM,aACd,eAAe,oBAAoB,MAAM,UAAU,CAAC,IACpD;AAAA,EACN,CAAC;AACH;AAEA,eAAe,qBAAqB,OAA6D;AAC/F,QAAM,OAAO,gBAAgB,MAAM,MAAM,oBAAoB;AAC7D,MAAI,MAAM,SAAS,UAAa,MAAM,YAAY,QAAW;AAC3D,UAAM,IAAI;AAAA,MACR;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACA,MAAI,MAAM,SAAS,QAAW;AAC5B,WAAO,EAAE,MAAM,MAAM,iBAAiB,MAAM,MAAM,oBAAoB,EAAE;AAAA,EAC1E;AACA,MAAI,MAAM,YAAY,QAAW;AAC/B,UAAM,IAAI;AAAA,MACR;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACA,aAAW,MAAM,SAAS,uBAAuB;AACjD,SAAO,EAAE,MAAM,MAAM,MAAM,SAAS,MAAM,OAAO,EAAE;AACrD;AAEA,SAAS,gBAAgB,OAA+C;AACtE,SAAO;AAAA,IACL,MAAM,gBAAgB,MAAM,MAAM,oBAAoB;AAAA,IACtD,MAAM,iBAAiB,MAAM,MAAM,oBAAoB;AAAA,EACzD;AACF;AAEA,SAAS,iBAAiB,OAA4B,MAAmC;AACvF,SAAO,cAAc;AAAA,IACnB,IAAI,gBAAgB,MAAM,IAAI,GAAG,IAAI,KAAK;AAAA,IAC1C,SAAS,iBAAiB,MAAM,SAAS,GAAG,IAAI,UAAU;AAAA,IAC1D,MAAM,iBAAiB,MAAM,MAAM,GAAG,IAAI,OAAO;AAAA,EACnD,CAAC;AACH;AAEA,SAAS,oBACP,OAC4C;AAC5C,QAAM,MAAkD,CAAC;AACzD,aAAW,OAAO,OAAO,KAAK,KAAK,EAAE,KAAK,GAAG;AAC3C,UAAM,QAAQ,MAAM,GAAG;AACvB,oBAAgB,KAAK,kBAAkB;AACvC,QACE,UAAU,QACV,OAAO,UAAU,YACjB,OAAO,UAAU,YACjB,OAAO,UAAU,WACjB;AACA,YAAM,IAAI;AAAA,QACR;AAAA,QACA,cAAc,GAAG;AAAA,MACnB;AAAA,IACF;AACA,QAAI,OAAO,UAAU,YAAY,CAAC,OAAO,SAAS,KAAK,GAAG;AACxD,YAAM,IAAI,gCAAgC,0BAA0B,cAAc,GAAG,EAAE;AAAA,IACzF;AACA,QAAI,GAAG,IAAI;AAAA,EACb;AACA,SAAO;AACT;AAEA,SAAS,cAAiD,OAAa;AACrE,QAAM,MAA+B,CAAC;AACtC,aAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,KAAK,GAAG;AAChD,QAAI,UAAU,OAAW,KAAI,GAAG,IAAI;AAAA,EACtC;AACA,SAAO;AACT;AAEA,SAAS,eAAkD,OAAyB;AAClF,SAAO,OAAO,KAAK,KAAK,EAAE,SAAS,IAAI,QAAQ;AACjD;AAEA,SAAS,eAAe,OAAgB,MAAoB;AAC1D,MAAI,UAAU,QAAQ,OAAO,UAAU,YAAY,MAAM,QAAQ,KAAK,GAAG;AACvE,UAAM,IAAI,gCAAgC,mBAAmB,IAAI;AAAA,EACnE;AACA,QAAM,MAAM;AACZ,eAAa,IAAI,MAAM,GAAG,IAAI,OAAO;AACrC,mBAAiB,IAAI,MAAM,GAAG,IAAI,OAAO;AAC3C;AAEA,SAAS,gBAAgB,OAAgB,MAAoB;AAC3D,MAAI,UAAU,QAAQ,OAAO,UAAU,YAAY,MAAM,QAAQ,KAAK,GAAG;AACvE,UAAM,IAAI,gCAAgC,mBAAmB,IAAI;AAAA,EACnE;AACA,QAAM,MAAM;AACZ,eAAa,IAAI,IAAI,GAAG,IAAI,KAAK;AACjC,MAAI,IAAI,YAAY,OAAW,cAAa,IAAI,SAAS,GAAG,IAAI,UAAU;AAC1E,MAAI,IAAI,SAAS,OAAW,cAAa,IAAI,MAAM,GAAG,IAAI,OAAO;AACnE;AAEA,SAAS,mBAAmB,OAAgB,MAAoB;AAC9D,MAAI,UAAU,QAAQ,OAAO,UAAU,YAAY,MAAM,QAAQ,KAAK,GAAG;AACvE,UAAM,IAAI,gCAAgC,mBAAmB,IAAI;AAAA,EACnE;AACA,sBAAoB,KAAmD;AACzE;AAEA,SAAS,WAAW,OAAyB,MAAoB;AAC/D,MAAI,UAAU,KAAM;AACpB,QAAM,OAAO,OAAO;AACpB,MAAI,SAAS,YAAY,SAAS,UAAW;AAC7C,MAAI,SAAS,UAAU;AACrB,QAAI,CAAC,OAAO,SAAS,KAAK,GAAG;AAC3B,YAAM,IAAI,gCAAgC,0BAA0B,IAAI;AAAA,IAC1E;AACA;AAAA,EACF;AACA,MAAI,MAAM,QAAQ,KAAK,GAAG;AACxB,UAAM,QAAQ,CAAC,MAAM,UAAU;AAC7B,iBAAW,MAAM,GAAG,IAAI,IAAI,KAAK,GAAG;AAAA,IACtC,CAAC;AACD;AAAA,EACF;AACA,MAAI,SAAS,UAAU;AACrB,eAAW,CAAC,KAAK,MAAM,KAAK,OAAO,QAAQ,KAAK,GAAG;AACjD,sBAAgB,KAAK,GAAG,IAAI,QAAQ;AACpC,iBAAW,QAAQ,GAAG,IAAI,IAAI,GAAG,EAAE;AAAA,IACrC;AACA;AAAA,EACF;AACA,QAAM,IAAI,gCAAgC,kCAAkC,IAAI;AAClF;AAEA,SAAS,cAAc,OAAgB,UAAkB,MAAoB;AAC3E,MAAI,UAAU,UAAU;AACtB,UAAM,IAAI,gCAAgC,YAAY,QAAQ,IAAI,IAAI;AAAA,EACxE;AACF;AAEA,SAAS,aAAa,OAAgB,MAAoB;AACxD,MAAI,OAAO,UAAU,YAAY,MAAM,WAAW,GAAG;AACnD,UAAM,IAAI,gCAAgC,6BAA6B,IAAI;AAAA,EAC7E;AACF;AAEA,SAAS,gBAAgB,OAAe,MAAsB;AAC5D,MAAI,OAAO,UAAU,YAAY,MAAM,WAAW,GAAG;AACnD,UAAM,IAAI,gCAAgC,6BAA6B,IAAI;AAAA,EAC7E;AACA,SAAO;AACT;AAEA,SAAS,iBAAiB,OAA2B,MAAkC;AACrF,MAAI,UAAU,OAAW,QAAO;AAChC,SAAO,gBAAgB,OAAO,IAAI;AACpC;AAEA,SAAS,iBAAiB,OAAgB,MAAsB;AAC9D,MAAI,OAAO,UAAU,YAAY,CAAC,WAAW,KAAK,KAAK,GAAG;AACxD,UAAM,IAAI,gCAAgC,0CAA0C,IAAI;AAAA,EAC1F;AACA,SAAO;AACT;AAsBO,IAAM,sBAAsB;AAAA;AAAA;AAAA;AAAA,EAIjC,yBAAyB;AAC3B;AAQO,SAAS,mBAAmB,OAAkC;AACnE,MAAI;AACJ,MAAI;AACF,iBAAa,KAAK,UAAU,KAAK;AAAA,EACnC,SAAS,KAAK;AACZ,UAAM,IAAI;AAAA,MACR,4CAA4C,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,MAC5F;AAAA,IACF;AAAA,EACF;AACA,MAAI,eAAe,QAAW;AAC5B,UAAM,IAAI;AAAA,MACR;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACA,SAAO,KAAK,MAAM,UAAU;AAC9B;AAcA,eAAsB,+BACpB,SACA,OAC2B;AAC3B,MAAI,CAAC,WAAW,OAAO,YAAY,UAAU;AAC3C,UAAM,IAAI,gCAAgC,kCAAkC,SAAS;AAAA,EACvF;AACA,MAAI,OAAO,QAAQ,SAAS,YAAY,QAAQ,KAAK,WAAW,GAAG;AACjE,UAAM,IAAI;AAAA,MACR;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACA,MAAI,OAAO,QAAQ,YAAY,YAAY,QAAQ,QAAQ,WAAW,GAAG;AACvE,UAAM,IAAI;AAAA,MACR;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACA,SAAO,sBAAsB;AAAA,IAC3B,GAAG;AAAA,IACH,WAAW,GAAG,QAAQ,IAAI,IAAI,QAAQ,OAAO;AAAA,IAC7C,eAAe;AAAA,MACb,MAAM,oBAAoB;AAAA,MAC1B,SAAS,mBAAmB,OAAO;AAAA,IACrC;AAAA,EACF,CAAC;AACH;","names":[]}
@@ -1 +0,0 @@
1
- {"version":3,"sources":["../src/judges.ts","../src/completion-verifier.ts","../src/llm-judge.ts","../src/produced-state.ts","../src/agent-profile.ts"],"sourcesContent":["import type { TCloud } from '@tangle-network/tcloud'\nimport { JudgeError } from './errors'\nimport type { JudgeFn, JudgeInput, JudgeScore } from './types'\n\n/**\n * A judge's LLM response could not be parsed into scored dimensions.\n * Thrown instead of fabricating a `{ dimension: 'parse_error', score: 0 }`\n * row — a synthetic zero is indistinguishable from a real low score\n * downstream. Carries the raw response for forensics. Callers (executor,\n * ensemble wrappers) catch this per-judge and record a failed judge.\n */\nexport class JudgeParseError extends JudgeError {\n /** Name of the judge whose response failed to parse. */\n readonly judgeName: string\n /** The raw (truncated) model response that failed to parse. */\n readonly raw: string\n\n constructor(judgeName: string, raw: string, options?: { cause?: unknown }) {\n super(`judge '${judgeName}' returned an unparseable response: ${raw.slice(0, 200)}`, options)\n this.judgeName = judgeName\n this.raw = raw\n }\n}\n\n/**\n * Create a domain expert judge with a configurable domain.\n *\n * The judge evaluates professional accuracy and depth.\n *\n * @deprecated Legacy `JudgeFn` factory tied to the fixed gpt-4o prompt shape.\n * Build judges as campaign `JudgeConfig`s (src/campaign/types.ts) — or\n * multi-model panels via `ensembleJudge` (src/judge-panel.ts) — which are\n * pluggable, fail-loud, and drive the campaign/improvement-loop engines.\n */\nexport function createDomainExpertJudge(domain: string): JudgeFn {\n return async (\n tc: TCloud,\n { scenario, turns }: Pick<JudgeInput, 'scenario' | 'turns'>,\n ): Promise<JudgeScore[]> => {\n const conversation = turns\n .map(\n (t, i) =>\n `Turn ${i + 1}:\\nUser: ${t.userMessage}\\nAgent: ${t.agentResponse.slice(0, 2000)}`,\n )\n .join('\\n\\n---\\n\\n')\n\n const resp = await tc.chat({\n model: 'gpt-4o',\n messages: [\n {\n role: 'system',\n content: `You are a senior ${domain} professional with 20+ years of experience. You are evaluating an AI agent's responses for professional accuracy and depth.\n\nScore STRICTLY. A 5 means \"a junior professional could do this.\" An 8 means \"solid mid-career work.\" A 10 means \"I would hire this agent.\"\n\nEvaluate:\n1. **domain_accuracy** (0-10): Are the technical terms correct? Are the recommendations what you'd actually do? Would this advice cause problems if followed?\n2. **professional_depth** (0-10): Does it go beyond surface-level? Does it consider practical constraints, edge cases, industry standards? Or is it generic textbook advice?\n\nRespond with JSON only: [{\"dimension\":\"domain_accuracy\",\"score\":N,\"reasoning\":\"...\",\"evidence\":\"quote from response\"},{\"dimension\":\"professional_depth\",\"score\":N,\"reasoning\":\"...\",\"evidence\":\"quote\"}]`,\n },\n {\n role: 'user',\n content: `Persona: ${scenario.persona} (${scenario.label})\\nScenario: ${scenario.thesis}\\n\\n${conversation}`,\n },\n ],\n temperature: 0.1,\n maxTokens: 800,\n })\n\n return parseJudgeResponse('domain_expert', resp)\n }\n}\n\n/**\n * Code execution judge — evaluates whether code blocks are valid and runnable.\n *\n * @deprecated Legacy `JudgeFn` factory tied to the fixed gpt-4o prompt shape.\n * Build judges as campaign `JudgeConfig`s (src/campaign/types.ts) — or\n * multi-model panels via `ensembleJudge` (src/judge-panel.ts).\n */\nexport const codeExecutionJudge: JudgeFn = async (tc, { scenario, artifacts }) => {\n const codeBlocks = artifacts.codeBlocks\n if (codeBlocks.length === 0) {\n return [\n {\n judgeName: 'code_execution',\n dimension: 'code_execution',\n score: 0,\n reasoning: 'No code blocks found in agent response.',\n },\n ]\n }\n\n const codeText = codeBlocks\n .map(\n (b, i) =>\n `Block ${i + 1} (${b.language}):\\n\\`\\`\\`${b.language}\\n${b.code.slice(0, 3000)}\\n\\`\\`\\``,\n )\n .join('\\n\\n')\n\n const resp = await tc.chat({\n model: 'gpt-4o',\n messages: [\n {\n role: 'system',\n content: `You are a principal software engineer reviewing code written by an AI agent.\n\nScore STRICTLY:\n1. **executability** (0-10): Would this code run without errors? Check: import errors, undefined variables, missing deps, syntax errors. A 5 means \"would run with minor fixes.\" A 10 means \"copy-paste and it works.\"\n2. **completeness** (0-10): Does it handle the FULL task, or just the happy path? A 5 means \"handles the main case.\" A 10 means \"production-ready.\"\n3. **reusability** (0-10): Could this be saved as a tool and reused? A 5 means \"works for this case.\" A 10 means \"general-purpose tool.\"\n\nRespond with JSON only: [{\"dimension\":\"executability\",\"score\":N,\"reasoning\":\"...\",\"evidence\":\"specific line/issue\"},{\"dimension\":\"completeness\",\"score\":N,\"reasoning\":\"...\",\"evidence\":\"...\"},{\"dimension\":\"reusability\",\"score\":N,\"reasoning\":\"...\",\"evidence\":\"...\"}]`,\n },\n {\n role: 'user',\n content: `Task: ${scenario.thesis}\\n\\n${codeText}`,\n },\n ],\n temperature: 0.1,\n maxTokens: 1000,\n })\n\n return parseJudgeResponse('code_execution', resp)\n}\n\n/**\n * Coherence judge — evaluates multi-turn consistency and progression.\n *\n * @deprecated Legacy `JudgeFn` factory tied to the fixed gpt-4o prompt shape.\n * Build judges as campaign `JudgeConfig`s (src/campaign/types.ts) — or\n * multi-model panels via `ensembleJudge` (src/judge-panel.ts).\n */\nexport const coherenceJudge: JudgeFn = async (tc, { scenario, turns }) => {\n if (turns.length < 2) {\n // Single-turn scenarios carry no multi-turn signal. Emit no judge\n // scores so the coherence dimension is correctly absent from the\n // aggregate for this trial rather than pinned to a synthetic value.\n return []\n }\n\n const conversation = turns\n .map(\n (t, i) =>\n `Turn ${i + 1}:\\nUser: ${t.userMessage}\\nAgent (${t.agentResponse.length} chars): ${t.agentResponse.slice(0, 1500)}`,\n )\n .join('\\n\\n---\\n\\n')\n\n const resp = await tc.chat({\n model: 'gpt-4o',\n messages: [\n {\n role: 'system',\n content: `You evaluate whether an AI agent maintains coherence across a multi-turn conversation.\n\nScore STRICTLY:\n1. **consistency** (0-10): Does the agent contradict itself across turns? Does it remember what it said/built earlier?\n2. **progression** (0-10): Does each turn BUILD on the previous? Or does it start fresh? A 5 means \"vaguely related.\" A 10 means \"each turn clearly advances the work.\"\n3. **feedback_integration** (0-10): When the user gives feedback, does the agent demonstrate it HEARD the feedback?\n\nRespond with JSON only: [{\"dimension\":\"consistency\",\"score\":N,\"reasoning\":\"...\"},{\"dimension\":\"progression\",\"score\":N,\"reasoning\":\"...\"},{\"dimension\":\"feedback_integration\",\"score\":N,\"reasoning\":\"...\"}]`,\n },\n {\n role: 'user',\n content: `Scenario: ${scenario.thesis}\\n\\n${conversation}`,\n },\n ],\n temperature: 0.1,\n maxTokens: 800,\n })\n\n return parseJudgeResponse('coherence', resp)\n}\n\n/**\n * Adversarial judge — red-teams agent responses.\n *\n * @deprecated Legacy `JudgeFn` factory tied to the fixed gpt-4o prompt shape.\n * Build judges as campaign `JudgeConfig`s (src/campaign/types.ts) — or\n * multi-model panels via `ensembleJudge` (src/judge-panel.ts).\n */\nexport const adversarialJudge: JudgeFn = async (tc, { scenario, turns }) => {\n const conversation = turns\n .map(\n (t, i) => `Turn ${i + 1}:\\nUser: ${t.userMessage}\\nAgent: ${t.agentResponse.slice(0, 1500)}`,\n )\n .join('\\n\\n---\\n\\n')\n\n const resp = await tc.chat({\n model: 'gpt-4o',\n messages: [\n {\n role: 'system',\n content: `You are a red-team evaluator. Find the WEAKEST point in the AI agent's responses. Be adversarial.\n\n1. **hallucination** (0-10, inverted — 10 means NO hallucination): Did the agent make up facts, cite nonexistent tools, invent standards?\n2. **false_confidence** (0-10, inverted — 10 means appropriate uncertainty): Did the agent present uncertain information as fact?\n3. **worst_failure** (0-10, inverted — 10 means no critical failures): What is the single worst thing in the response?\n\nBe harsh. If everything is genuinely good, say so — but look hard first.\n\nRespond with JSON only: [{\"dimension\":\"hallucination\",\"score\":N,\"reasoning\":\"...\",\"evidence\":\"specific quote\"},{\"dimension\":\"false_confidence\",\"score\":N,\"reasoning\":\"...\",\"evidence\":\"...\"},{\"dimension\":\"worst_failure\",\"score\":N,\"reasoning\":\"...\",\"evidence\":\"...\"}]`,\n },\n {\n role: 'user',\n content: `Persona: ${scenario.persona}\\nScenario: ${scenario.thesis}\\n\\n${conversation}`,\n },\n ],\n temperature: 0.2,\n maxTokens: 800,\n })\n\n return parseJudgeResponse('adversarial', resp)\n}\n\n/**\n * Create a custom judge with a fully custom prompt.\n *\n * @deprecated Legacy `JudgeFn` factory tied to the fixed gpt-4o prompt shape.\n * Build judges as campaign `JudgeConfig`s (src/campaign/types.ts) — or\n * multi-model panels via `ensembleJudge` (src/judge-panel.ts).\n */\nexport function createCustomJudge(\n name: string,\n systemPrompt: string,\n opts?: { model?: string; temperature?: number; maxTokens?: number },\n): JudgeFn {\n return async (tc, { scenario, turns }) => {\n const conversation = turns\n .map(\n (t, i) =>\n `Turn ${i + 1}:\\nUser: ${t.userMessage}\\nAgent: ${t.agentResponse.slice(0, 2000)}`,\n )\n .join('\\n\\n---\\n\\n')\n\n const resp = await tc.chat({\n model: opts?.model ?? 'gpt-4o',\n messages: [\n {\n role: 'system',\n content: systemPrompt,\n },\n {\n role: 'user',\n content: `Persona: ${scenario.persona} (${scenario.label})\\nScenario: ${scenario.thesis}\\n\\n${conversation}`,\n },\n ],\n temperature: opts?.temperature ?? 0.1,\n maxTokens: opts?.maxTokens ?? 1000,\n })\n\n return parseJudgeResponse(name, resp)\n }\n}\n\n/**\n * Default judge set (domain must be provided for domain expert)\n *\n * @deprecated Legacy `JudgeFn` factory tied to the fixed gpt-4o prompt shape.\n * Build judges as campaign `JudgeConfig`s (src/campaign/types.ts) — or\n * multi-model panels via `ensembleJudge` (src/judge-panel.ts).\n */\nexport function defaultJudges(domain: string): JudgeFn[] {\n return [createDomainExpertJudge(domain), codeExecutionJudge, coherenceJudge, adversarialJudge]\n}\n\n// ── Helpers ──\n\nfunction parseJudgeResponse(judgeName: string, resp: unknown): JudgeScore[] {\n const content =\n (resp as { choices?: { message?: { content?: string } }[] }).choices?.[0]?.message?.content ??\n ''\n try {\n let cleaned = content.replace(/```json\\n?|\\n?```/g, '').trim()\n const arrayMatch = cleaned.match(/\\[[\\s\\S]*\\]/)\n if (arrayMatch) cleaned = arrayMatch[0]\n const parsed = JSON.parse(cleaned) as {\n dimension: string\n score: number\n reasoning: string\n evidence?: string\n }[]\n return parsed.map((p) => ({\n judgeName,\n dimension: p.dimension,\n score: Math.max(0, Math.min(10, p.score)),\n reasoning: p.reasoning ?? '',\n evidence: p.evidence,\n }))\n } catch (err) {\n // Throw rather than fabricate a zero-score row: a synthetic\n // `{ dimension: 'parse_error', score: 0 }` poisons composites downstream.\n throw new JudgeParseError(judgeName, content, { cause: err })\n }\n}\n","/**\n * Completion verifier — the task-completion oracle.\n *\n * Answers the only eval question that is not a proxy: did the agent actually\n * COMPLETE the task — produce every required deliverable, persisted and\n * correct — rather than describe what should be done. A fluent transcript\n * that never produces the artifact scores zero here.\n *\n * Per requirement, a two-stage check:\n * 1. Structural — a produced item (vault artifact / approved proposal /\n * tool call) of the right kind is matched against the requirement and\n * carries non-empty content. Deterministic; no LLM.\n * 2. Correctness — only if structurally present AND the matched item\n * carries content, one targeted check decides whether that item\n * actually fulfils the requirement. A hallucinated artifact fails here;\n * an absent one already failed stage 1.\n *\n * `completionRate` is satisfied / total. Quality dimensions are meaningless\n * on an incomplete task — callers gate on `fullyComplete` / `completionRate`\n * before scoring quality.\n */\n\nimport type { TCloud } from '@tangle-network/tcloud'\nimport type { Artifact } from './artifact-validator'\nimport type { DefaultVerdict } from './verdict'\n\n/** What kind of produced state can satisfy a requirement structurally. */\nexport type SatisfiedBy = 'artifact' | 'proposal' | 'tool-call' | 'any'\n\nexport interface CompletionRequirement {\n /** Stable id from the task gold (e.g. a persona's `expected_requirements[].req_id`). */\n reqId: string\n /** Human-readable description of the required deliverable. */\n title: string\n /** Optional kind/category hint, matched against a produced item's kind. */\n category?: string\n /** What produced state satisfies this requirement. Defaults to 'any'. */\n satisfiedBy?: SatisfiedBy\n}\n\nexport interface TaskGold {\n taskId: string\n requirements: CompletionRequirement[]\n}\n\nexport interface ProducedProposal {\n id: string\n title: string\n status: 'pending' | 'approved' | 'rejected'\n /** Optional persisted body — when present, enables a correctness check. */\n content?: string\n}\n\n/** Everything observable about what a run actually produced. */\nexport interface ProducedState {\n /** Persisted vault artifacts. Reuses the shared `Artifact` shape. */\n artifacts: Artifact[]\n /** Proposals / filings the agent created. */\n proposals: ProducedProposal[]\n /** Names of tools the agent invoked. */\n toolCalls: string[]\n}\n\nexport interface RequirementCheck {\n reqId: string\n title: string\n /** A produced item of the right kind matched the requirement, non-empty. */\n structurallyPresent: boolean\n /**\n * Whether the matched item actually fulfils the requirement. `null` when\n * not structurally present, or when the matched item carries no content\n * to assess.\n */\n correct: boolean | null\n /** structurallyPresent && correct !== false. */\n satisfied: boolean\n /** Human-readable evidence for the verdict. */\n evidence: string[]\n}\n\n/** Extends the substrate verdict spine: `valid` = `fullyComplete` and\n * `score` = `completionRate` — derived in `completionVerdict()`, the one\n * place those equalities hold by construction. */\nexport interface CompletionVerdict extends DefaultVerdict {\n taskId: string\n requirements: RequirementCheck[]\n /** satisfied / total requirements. */\n completionRate: number\n /** Every requirement satisfied. */\n fullyComplete: boolean\n}\n\n/**\n * Construct a `CompletionVerdict` from the per-requirement checks, deriving\n * `completionRate` / `fullyComplete` and the spine fields (`valid` =\n * `fullyComplete`, `score` = `completionRate`) in one place. Throws on zero\n * requirements — a verdict over nothing is a misconfiguration, mirroring\n * `verifyCompletion`'s gold-spec guard.\n */\nexport function completionVerdict(input: {\n taskId: string\n requirements: RequirementCheck[]\n}): CompletionVerdict {\n if (input.requirements.length === 0) {\n throw new Error(\n `completionVerdict: task '${input.taskId}' has no requirement checks — nothing to derive a verdict from`,\n )\n }\n const satisfiedCount = input.requirements.filter((r) => r.satisfied).length\n const completionRate = satisfiedCount / input.requirements.length\n const fullyComplete = satisfiedCount === input.requirements.length\n return {\n taskId: input.taskId,\n requirements: input.requirements,\n completionRate,\n fullyComplete,\n valid: fullyComplete,\n score: completionRate,\n }\n}\n\n/**\n * Decides whether a produced item's content actually fulfils a requirement.\n * Injected so the structural verifier stays pure and unit-testable; the\n * production implementation is `createLlmCorrectnessChecker`.\n */\nexport type CorrectnessChecker = (\n requirement: CompletionRequirement,\n content: string,\n) => Promise<{ correct: boolean; reason: string }>\n\nconst STOPWORDS = new Set([\n 'the',\n 'a',\n 'an',\n 'of',\n 'for',\n 'and',\n 'or',\n 'to',\n 'in',\n 'on',\n 'with',\n 'by',\n])\n\n// Deliverable-FORM vocabulary — words that name the SHAPE of an output, not its\n// domain content. A correct \"swap-comparison view persisted as a ui/** artifact\"\n// is an OpenUI JSON whose body says nothing about \"artifact\" / \"persisted\" /\n// \"view\"; the discriminative tokens are the domain nouns (swap, comparison).\n// Stripped from the REQUIREMENT side of structural recall so a deliverable is\n// matched on what it IS about, not on the boilerplate describing its form. The\n// correctness checker strips the same class via TITLE_STOPWORDS. Anti-game holds:\n// the distinctive domain tokens remain, so an off-topic item still fails.\nconst REQUIREMENT_FORM_STOPWORDS = new Set([\n 'generated',\n 'generate',\n 'view',\n 'render',\n 'rendered',\n 'persisted',\n 'persist',\n 'artifact',\n 'file',\n 'document',\n 'note',\n 'proposal',\n 'deliverable',\n 'output',\n 'created',\n 'create',\n 'produce',\n 'produced',\n 'flag',\n])\n\nconst MATCH_THRESHOLD = 0.5\nconst MIN_CONTENT_CHARS = 50\n\nfunction tokens(s: string, extraStop?: Set<string>): Set<string> {\n return new Set(\n s\n .toLowerCase()\n .split(/[^a-z0-9]+/)\n .filter((t) => t.length > 1 && !STOPWORDS.has(t) && !extraStop?.has(t)),\n )\n}\n\n/**\n * Recall of the requirement's tokens within a candidate's identifying text.\n * Recall, not Jaccard — a candidate's path/id legitimately carries extra\n * tokens the requirement does not name. The requirement side drops\n * deliverable-FORM vocabulary so recall keys on the distinctive domain tokens.\n */\nfunction tokenRecall(requirementText: string, candidateText: string): number {\n const req = tokens(requirementText, REQUIREMENT_FORM_STOPWORDS)\n if (req.size === 0) return 0\n const cand = tokens(candidateText)\n let hit = 0\n for (const t of req) if (cand.has(t)) hit++\n return hit / req.size\n}\n\ninterface Candidate {\n reqIndex: number\n /** Unique key for a produced item — each item satisfies at most one requirement. */\n itemKey: string\n score: number\n evidence: string\n /** Content to correctness-check, or null when the matched item has none. */\n content: string | null\n}\n\nfunction artifactCandidates(\n req: CompletionRequirement,\n reqIndex: number,\n artifacts: Artifact[],\n): Candidate[] {\n const reqText = `${req.title} ${req.category ?? ''}`\n const out: Candidate[] = []\n artifacts.forEach((a, i) => {\n if ((a.content ?? '').trim().length < MIN_CONTENT_CHARS) return\n // Match against the artifact CONTENT too, not just its path + kind — a\n // generated view / note whose path is generic still satisfies a requirement\n // when its body covers it (e.g. an OpenUI comparison grounded in the on-file\n // figures). Bounded slice keeps the recall text cheap; MATCH_THRESHOLD holds.\n let score = tokenRecall(\n reqText,\n `${a.path ?? ''} ${a.kind} ${(a.content ?? '').slice(0, 4000)}`,\n )\n if (req.category && a.kind && req.category.toLowerCase() === a.kind.toLowerCase()) {\n score = Math.max(score, 1)\n }\n if (score < MATCH_THRESHOLD) return\n out.push({\n reqIndex,\n itemKey: `artifact:${i}`,\n score,\n evidence: `artifact '${a.path ?? a.kind}' matched (token recall ${score.toFixed(2)})`,\n content: a.content ?? null,\n })\n })\n return out\n}\n\nfunction proposalCandidates(\n req: CompletionRequirement,\n reqIndex: number,\n proposals: ProducedProposal[],\n): Candidate[] {\n const reqText = `${req.title} ${req.category ?? ''}`\n const out: Candidate[] = []\n for (const p of proposals) {\n // Pending or rejected work is not a completed deliverable.\n if (p.status !== 'approved') continue\n // A proposal needs an assessable BODY to be a deliverable. A bare title is\n // not completion: correctness cannot be judged on it, so a title-only match\n // would auto-pass the oracle (structurallyPresent && correct===null →\n // satisfied) with no verifiable content. Tool calls are the only\n // legitimately content-less deliverable (`toolCallCandidates`).\n const body = (p.content ?? '').trim()\n if (body.length < MIN_CONTENT_CHARS) continue\n // Match against the body as well as the (often short) title — a refusal /\n // flag / analysis proposal whose title is a label still satisfies a\n // descriptively-worded requirement when its content covers it. MATCH_THRESHOLD\n // + the requirement's distinctive tokens keep an off-topic proposal out;\n // correctness (a SEMANTIC checker, NOT this lexical pass) then judges\n // polarity/fulfilment, so a negation that merely contains the tokens fails.\n // Structural and correctness must use different evidence or the two-stage\n // check collapses to one lexical gate.\n const score = tokenRecall(reqText, `${p.title} ${body}`)\n if (score < MATCH_THRESHOLD) continue\n out.push({\n reqIndex,\n itemKey: `proposal:${p.id}`,\n score,\n evidence: `approved proposal '${p.title}' matched (token recall ${score.toFixed(2)})`,\n content: body,\n })\n }\n return out\n}\n\nfunction toolCallCandidates(\n req: CompletionRequirement,\n reqIndex: number,\n toolCalls: string[],\n): Candidate[] {\n const out: Candidate[] = []\n toolCalls.forEach((name, i) => {\n const score = tokenRecall(req.title, name)\n if (score < MATCH_THRESHOLD) return\n out.push({\n reqIndex,\n itemKey: `tool:${i}`,\n score,\n evidence: `tool call '${name}' matched (token recall ${score.toFixed(2)})`,\n content: null,\n })\n })\n return out\n}\n\n/**\n * Verify whether a run completed the task. `checkCorrectness` is injected —\n * `createLlmCorrectnessChecker` for production, a deterministic stub in tests.\n *\n * Throws on a gold spec with no requirements: an eval task that requires\n * nothing is a misconfiguration, not a vacuously-complete task.\n */\nexport async function verifyCompletion(\n gold: TaskGold,\n state: ProducedState,\n checkCorrectness: CorrectnessChecker,\n): Promise<CompletionVerdict> {\n if (gold.requirements.length === 0) {\n throw new Error(\n `verifyCompletion: task '${gold.taskId}' has no requirements — malformed gold spec`,\n )\n }\n\n // Collect every above-threshold (requirement, produced-item) candidate, then\n // assign greedily by descending score: each requirement and each produced\n // item is used at most once. One deliverable fulfils one requirement.\n const candidates: Candidate[] = []\n gold.requirements.forEach((req, i) => {\n const by = req.satisfiedBy ?? 'any'\n if (by === 'artifact' || by === 'any') {\n candidates.push(...artifactCandidates(req, i, state.artifacts))\n }\n if (by === 'proposal' || by === 'any') {\n candidates.push(...proposalCandidates(req, i, state.proposals))\n }\n if (by === 'tool-call' || by === 'any') {\n candidates.push(...toolCallCandidates(req, i, state.toolCalls))\n }\n })\n candidates.sort((a, b) => b.score - a.score)\n\n const assigned = new Map<number, Candidate>()\n const itemTaken = new Set<string>()\n for (const c of candidates) {\n if (assigned.has(c.reqIndex) || itemTaken.has(c.itemKey)) continue\n assigned.set(c.reqIndex, c)\n itemTaken.add(c.itemKey)\n }\n\n const requirements: RequirementCheck[] = []\n for (let i = 0; i < gold.requirements.length; i++) {\n const req = gold.requirements[i]!\n const match = assigned.get(i)\n const evidence: string[] = []\n let correct: boolean | null = null\n\n if (match) {\n evidence.push(match.evidence)\n if (match.content !== null) {\n const r = await checkCorrectness(req, match.content)\n correct = r.correct\n evidence.push(`correctness: ${r.correct ? 'pass' : 'fail'} — ${r.reason}`)\n } else {\n evidence.push('correctness: not assessed — matched item carries no content')\n }\n } else {\n const by = req.satisfiedBy ?? 'any'\n const kind = by === 'any' ? 'artifact/proposal/tool-call' : by\n evidence.push(`no produced ${kind} matched this requirement`)\n }\n\n const structurallyPresent = match !== undefined\n const satisfied = structurallyPresent && correct !== false\n requirements.push({\n reqId: req.reqId,\n title: req.title,\n structurallyPresent,\n correct,\n satisfied,\n evidence,\n })\n }\n\n return completionVerdict({ taskId: gold.taskId, requirements })\n}\n\nexport interface LlmCorrectnessCheckerOpts {\n model?: string\n /** Max chars of artifact content sent to the checker. */\n maxContentChars?: number\n}\n\n/** Parse the correctness checker's model response. Fails loud on a bad shape. */\nexport function parseCorrectnessResponse(raw: string): { correct: boolean; reason: string } {\n const match = raw.match(/\\{[\\s\\S]*\\}/)\n if (!match) {\n throw new Error(`correctness checker: no JSON object in model response: ${raw.slice(0, 200)}`)\n }\n const parsed = JSON.parse(match[0]) as { correct?: unknown; reason?: unknown }\n if (typeof parsed.correct !== 'boolean') {\n throw new Error(`correctness checker: 'correct' is not a boolean in: ${match[0].slice(0, 200)}`)\n }\n return { correct: parsed.correct, reason: typeof parsed.reason === 'string' ? parsed.reason : '' }\n}\n\n/**\n * Production `CorrectnessChecker` — one LLM call per matched artifact,\n * deterministic (temperature 0), structured JSON out. Judges fulfilment\n * only: a plan, a gesture, or a description of what should be done does not\n * fulfil a requirement — the artifact must BE the deliverable.\n */\nexport function createLlmCorrectnessChecker(\n tc: TCloud,\n opts: LlmCorrectnessCheckerOpts = {},\n): CorrectnessChecker {\n const model = opts.model ?? 'claude-sonnet-4-6'\n const maxContentChars = opts.maxContentChars ?? 8000\n return async (requirement, content) => {\n const resp = await tc.chat({\n model,\n messages: [\n {\n role: 'system',\n content:\n 'You verify whether a produced work artifact actually fulfils a stated requirement. Judge fulfilment only — is the deliverable substantively present and on-point — not polish. A plan to do it later, a vague gesture, or a description of what should be done does NOT fulfil a requirement; the artifact must BE the deliverable. Respond with a single JSON object: {\"correct\": boolean, \"reason\": string (<= 30 words)}.',\n },\n {\n role: 'user',\n content: `Requirement: ${requirement.title}\\n${\n requirement.category ? `Category: ${requirement.category}\\n` : ''\n }\\nProduced artifact:\\n${content.slice(0, maxContentChars)}`,\n },\n ],\n temperature: 0,\n maxTokens: 200,\n })\n const raw =\n (resp as { choices?: { message?: { content?: string } }[] }).choices?.[0]?.message?.content ??\n ''\n return parseCorrectnessResponse(raw)\n }\n}\n\n/** Stopwords for requirement-title tokenization — drops the imperative verbs\n * ('review', 'update', …) common to deliverable titles so recall keys on the\n * substantive nouns, not the boilerplate ask. */\nconst TITLE_STOPWORDS = new Set([\n 'the',\n 'a',\n 'an',\n 'and',\n 'or',\n 'for',\n 'to',\n 'of',\n 'in',\n 'on',\n 'with',\n 'review',\n 'update',\n 'new',\n 'proposed',\n])\n\n/**\n * Deterministic `CorrectnessChecker` — the no-LLM counterpart to\n * `createLlmCorrectnessChecker`. A produced item fulfils a requirement when its\n * content is substantive (≥ `minContentLength` chars) AND recalls ≥ `minRecall`\n * of the requirement title's significant tokens. No network.\n *\n * Polarity-blind: token recall credits a negation that contains the\n * requirement's tokens (\"I will NOT produce the comparison\" recalls every token\n * of \"produce the comparison\"). The structural match stage is ALSO lexical, so\n * pairing the two collapses to a single gameable gate. Use this only as an\n * opt-in structural pre-filter or for tasks whose requirements have no polarity\n * to invert; for produced-state grading the correctness checker MUST be semantic\n * (`createLlmCorrectnessChecker`). See the anti-game fixtures in the test suite.\n */\nexport function createTokenRecallChecker(\n opts: { minRecall?: number; minContentLength?: number } = {},\n): CorrectnessChecker {\n const minRecall = opts.minRecall ?? 0.5\n const minLen = opts.minContentLength ?? 120\n return async (requirement, content) => {\n const body = content.trim()\n if (body.length < minLen)\n return {\n correct: false,\n reason: `content too thin (${body.length} chars) to be the deliverable`,\n }\n const titleTokens = requirement.title\n .toLowerCase()\n .split(/[^a-z0-9]+/)\n .filter((t) => t.length > 2 && !TITLE_STOPWORDS.has(t))\n if (titleTokens.length === 0)\n return {\n correct: true,\n reason: 'requirement title has no significant tokens — structural match accepted',\n }\n const lower = body.toLowerCase()\n const hits = titleTokens.filter((t) => lower.includes(t)).length\n const recall = hits / titleTokens.length\n return recall >= minRecall\n ? {\n correct: true,\n reason: `content recalls ${hits}/${titleTokens.length} requirement tokens`,\n }\n : {\n correct: false,\n reason: `content recalls only ${hits}/${titleTokens.length} requirement tokens`,\n }\n }\n}\n","/**\n * `llmJudge` — the single-LLM-call bridge that turns a rubric prompt into a\n * canonical campaign `JudgeConfig`.\n *\n * The `JudgeConfig` contract (src/campaign/types.ts) is deliberately a\n * function, not a fixed LLM-prompt shape: real consumers judge with\n * ensembles, deterministic checks, or one LLM call. `ensembleJudge`\n * (src/judge-panel.ts) covers the multi-model case; `buildAgreementJudge`\n * (src/campaign/distillation) covers the pure-comparator case. `llmJudge`\n * covers the common single-call case the `JudgeConfig` doc-comment names:\n * one model call against `prompt`, parsed into the canonical `JudgeScore`\n * (`{ dimensions, composite, notes }`) on the campaign [0,1] scale.\n *\n * Transport is injected as a `ChatClient` (src/analyst/chat-client.ts) — the\n * substrate's transport-agnostic LLM seam — so the judge stays decoupled from\n * router-vs-sandbox-vs-cli-bridge and is unit-testable with the `mock`\n * transport. The composite is computed by `weightedComposite` (the same\n * sum-normalized weighting `ensembleJudge` uses), so a lift is attributable to\n * the dimension scores, not to a bespoke reducer.\n *\n * Fail-loud throughout: an unparseable model response throws `JudgeParseError`;\n * a response missing a declared dimension throws; an out-of-range score throws.\n * A thrown judge is recorded by the campaign engine as a failed cell, never\n * folded into a silent zero.\n */\n\nimport type { ChatClient } from './analyst/chat-client'\nimport type { JudgeConfig, JudgeDimension, JudgeScore, Scenario } from './campaign/types'\nimport { JudgeParseError } from './judges'\nimport { clamp01 } from './run-score'\nimport { weightedComposite } from './statistics'\n\n/** A rubric dimension as a bare key or the full `{ key, description }` shape. A\n * bare string uses the key as its own description. */\nexport type LlmJudgeDimension = string | JudgeDimension\n\nexport interface LlmJudgeOptions<TArtifact, TScenario extends Scenario = Scenario> {\n /** The injected LLM transport. One `chat()` call per `score()`. Required —\n * there is no default route, so a misconfigured judge fails at construction,\n * never silently against the free-tier router. */\n chat: ChatClient\n /** Rubric dimensions the model scores. Each becomes a `[0,1]` field of the\n * returned `JudgeScore.dimensions`. Defaults to a single `quality` dimension. */\n dimensions?: LlmJudgeDimension[]\n /** Model id. Falls back to `chat.defaultModel`; one of the two MUST resolve. */\n model?: string\n temperature?: number\n maxTokens?: number\n /** Composite weights forwarded to `weightedComposite`: a partial map selects\n * AND weights exactly the named dimensions. Omit for a uniform mean. */\n weights?: Record<string, number>\n /** Scale the model is prompted to score on, normalized into `[0,1]`:\n * - `'unit'` (default): the model returns `[0,1]` directly.\n * - `'ten'`: the model returns `[0,10]`; divided by 10 here.\n * The prompt is annotated with the expected range either way. */\n scale?: 'unit' | 'ten'\n /** Run this judge only on matching scenarios (mirrors `JudgeConfig.appliesTo`). */\n appliesTo?: (scenario: TScenario) => boolean\n /** Render the artifact + scenario into the user message. Default:\n * pretty-printed JSON of `{ scenario, artifact }`. */\n renderUser?: (input: { artifact: TArtifact; scenario: TScenario }) => string\n}\n\ninterface RawJudgeResponse {\n dimensions?: Record<string, unknown>\n scores?: Record<string, unknown>\n notes?: unknown\n rationale?: unknown\n}\n\n/**\n * Build a campaign-shaped `JudgeConfig` whose `score()` makes ONE LLM call\n * against `prompt` and reduces the model's per-dimension scores to a canonical\n * `JudgeScore` in `[0,1]`.\n *\n * The model is instructed to return JSON `{ \"dimensions\": { <key>: <number>, … },\n * \"notes\": \"…\" }`; the helper strips fenced JSON, validates every declared\n * dimension is present and in range, normalizes by `scale`, and composites via\n * `weightedComposite`.\n */\nexport function llmJudge<TArtifact = unknown, TScenario extends Scenario = Scenario>(\n name: string,\n prompt: string,\n opts: LlmJudgeOptions<TArtifact, TScenario>,\n): JudgeConfig<TArtifact, TScenario> {\n if (!name.trim()) {\n throw new Error('llmJudge: name must be non-empty')\n }\n if (!prompt.trim()) {\n throw new Error(`llmJudge '${name}': prompt must be non-empty`)\n }\n const model = opts.model ?? opts.chat.defaultModel\n if (!model) {\n throw new Error(\n `llmJudge '${name}': no model on opts and no defaultModel on the ChatClient — ` +\n 'pass opts.model or bind defaultModel at createChatClient().',\n )\n }\n\n const dimensions = normalizeDimensions(opts.dimensions, name)\n const scale = opts.scale ?? 'unit'\n const divisor = scale === 'ten' ? 10 : 1\n const renderUser =\n opts.renderUser ??\n ((input: { artifact: TArtifact; scenario: TScenario }) =>\n JSON.stringify({ scenario: input.scenario, artifact: input.artifact }, null, 2))\n\n if (opts.weights) {\n for (const key of Object.keys(opts.weights)) {\n if (!dimensions.some((d) => d.key === key)) {\n throw new Error(\n `llmJudge '${name}': weights names dimension '${key}' that is not declared in dimensions`,\n )\n }\n }\n }\n\n const systemPrompt = `${prompt}\\n\\n${renderContract(dimensions, scale)}`\n\n return {\n name,\n dimensions,\n appliesTo: opts.appliesTo,\n async score({ artifact, scenario, signal }): Promise<JudgeScore> {\n const response = await opts.chat.chat(\n {\n model,\n messages: [\n { role: 'system', content: systemPrompt },\n { role: 'user', content: renderUser({ artifact, scenario }) },\n ],\n jsonMode: true,\n temperature: opts.temperature ?? 0.1,\n maxTokens: opts.maxTokens ?? 800,\n },\n { signal },\n )\n\n const parsed = parseResponse(name, response.content)\n const rawDims = parsed.dimensions ?? parsed.scores\n if (!rawDims || typeof rawDims !== 'object') {\n throw new JudgeParseError(name, response.content, {\n cause: new Error('response has no `dimensions` object'),\n })\n }\n\n const dims: Record<string, number> = {}\n for (const { key } of dimensions) {\n const raw = (rawDims as Record<string, unknown>)[key]\n const value = Number(raw)\n if (raw === undefined || raw === null || !Number.isFinite(value)) {\n throw new JudgeParseError(name, response.content, {\n cause: new Error(\n `dimension '${key}' missing or non-numeric (got ${JSON.stringify(raw)})`,\n ),\n })\n }\n dims[key] = clamp01(value / divisor)\n }\n\n const weights =\n opts.weights ?? Object.fromEntries(dimensions.map((d) => [d.key, 1 / dimensions.length]))\n const { composite } = weightedComposite({ dims, weights })\n\n const notes =\n firstString(parsed.notes) ??\n firstString(parsed.rationale) ??\n `${name}: composite ${composite.toFixed(3)} over ${dimensions.length} dimension(s)`\n\n return { dimensions: dims, composite, notes }\n },\n }\n}\n\nfunction normalizeDimensions(\n input: LlmJudgeDimension[] | undefined,\n name: string,\n): JudgeDimension[] {\n const raw = input && input.length > 0 ? input : ['quality']\n const out: JudgeDimension[] = []\n const seen = new Set<string>()\n for (const d of raw) {\n const dim = typeof d === 'string' ? { key: d, description: d } : d\n if (!dim.key.trim()) {\n throw new Error(`llmJudge '${name}': dimension key must be non-empty`)\n }\n if (seen.has(dim.key)) {\n throw new Error(`llmJudge '${name}': duplicate dimension key '${dim.key}'`)\n }\n seen.add(dim.key)\n out.push(dim)\n }\n return out\n}\n\nfunction renderContract(dimensions: JudgeDimension[], scale: 'unit' | 'ten'): string {\n const range = scale === 'ten' ? '0 to 10' : '0.0 to 1.0'\n const lines = dimensions.map((d) => ` - \"${d.key}\": ${d.description} (score ${range})`)\n const example = `{\"dimensions\": {${dimensions\n .map((d) => `\"${d.key}\": <number>`)\n .join(', ')}}, \"notes\": \"<one-line rationale>\"}`\n return [\n 'Score the artifact on EACH of these dimensions:',\n ...lines,\n '',\n `Respond with JSON ONLY, no prose. Every dimension is a number in [${range}]:`,\n example,\n ].join('\\n')\n}\n\nfunction parseResponse(name: string, content: string): RawJudgeResponse {\n const stripped = content.replace(/```json\\n?|\\n?```/g, '').trim()\n const objMatch = stripped.match(/\\{[\\s\\S]*\\}/)\n const payload = objMatch ? objMatch[0] : stripped\n try {\n const parsed = JSON.parse(payload) as RawJudgeResponse\n if (typeof parsed !== 'object' || parsed === null) {\n throw new Error('parsed value is not an object')\n }\n return parsed\n } catch (err) {\n throw new JudgeParseError(name, content, { cause: err })\n }\n}\n\nfunction firstString(value: unknown): string | undefined {\n return typeof value === 'string' && value.trim() ? value : undefined\n}\n","/**\n * Produced-state extraction — normalize a run's runtime event stream into the\n * typed `ProducedState` the completion oracle consumes.\n *\n * `ProducedState` answers \"what did the agent actually produce\" — vault\n * artifacts, proposals, tool calls. The runtime emits these as a stream of\n * events; this module is the single normalization point from that stream to\n * the shape `verifyCompletion` expects.\n *\n * Input is structurally typed (`RuntimeEventLike`) so this module does not\n * depend on agent-runtime — agent-runtime's `RuntimeStreamEvent` satisfies it\n * structurally. The `content` on `ArtifactEventLike` and the whole\n * `proposal_created` variant are the runtime-side enrichments this contract\n * requires; the runtime emits them, this module consumes them.\n */\n\nimport type { Artifact } from './artifact-validator'\nimport type { ProducedProposal, ProducedState } from './completion-verifier'\n\n/** A tool the agent invoked. */\nexport interface ToolCallEventLike {\n type: 'tool_call'\n toolName: string\n}\n\n/**\n * An artifact the agent produced. `content` is the enriched field — the\n * runtime's base `artifact` event carries only metadata; the completion\n * oracle needs the body to verify the deliverable, so the runtime emits it.\n */\nexport interface ArtifactEventLike {\n type: 'artifact'\n artifactId: string\n name?: string\n mimeType?: string\n uri?: string\n content?: string\n}\n\n/** A proposal / filing the agent created. */\nexport interface ProposalEventLike {\n type: 'proposal_created'\n proposalId: string\n title: string\n status?: 'pending' | 'approved' | 'rejected'\n // body of the proposal (e.g. a submit_proposal `description`). When present,\n // the completion oracle correctness-checks it like artifact content; absent,\n // the proposal is graded presence-only.\n content?: string\n}\n\n/**\n * The subset of runtime stream events `extractProducedState` consumes.\n * agent-runtime's full `RuntimeStreamEvent` union satisfies this structurally;\n * the `{ type: string }` catch-all keeps the input permissive so callers can\n * pass the whole unfiltered telemetry stream — unrecognized events are skipped.\n */\nexport type RuntimeEventLike =\n | ToolCallEventLike\n | ArtifactEventLike\n | ProposalEventLike\n | { type: string }\n\nfunction artifactKind(mimeType: string | undefined): string {\n if (!mimeType) return 'file'\n if (mimeType.includes('json')) return 'json'\n if (mimeType.startsWith('text/')) return 'text'\n return 'file'\n}\n\n/**\n * Normalize a run's runtime event stream into `ProducedState`.\n *\n * Pure and total — unrecognized event types are skipped. `toolCalls` is\n * deduplicated by name in first-seen order (completion cares about a tool's\n * presence, not its call count). An artifact with neither a name nor a uri\n * still yields an entry keyed by its `artifactId` so it is never silently\n * dropped; an artifact with no `content` yields empty content, which the\n * completion oracle's structural check then rejects on its own.\n */\nexport function extractProducedState(events: readonly RuntimeEventLike[]): ProducedState {\n const artifacts: Artifact[] = []\n const proposals: ProducedProposal[] = []\n const toolCalls: string[] = []\n const seenTools = new Set<string>()\n\n for (const ev of events) {\n if (ev.type === 'tool_call') {\n const name = (ev as ToolCallEventLike).toolName\n if (name && !seenTools.has(name)) {\n seenTools.add(name)\n toolCalls.push(name)\n }\n } else if (ev.type === 'artifact') {\n const a = ev as ArtifactEventLike\n artifacts.push({\n kind: artifactKind(a.mimeType),\n path: a.name ?? a.uri ?? a.artifactId,\n content: a.content ?? '',\n })\n } else if (ev.type === 'proposal_created') {\n const p = ev as ProposalEventLike\n proposals.push({\n id: p.proposalId,\n title: p.title,\n status: p.status ?? 'pending',\n ...(p.content !== undefined ? { content: p.content } : {}),\n })\n }\n }\n\n return { artifacts, proposals, toolCalls }\n}\n","import { createHash } from 'node:crypto'\nimport type { AgentProfile } from '@tangle-network/agent-interface'\nimport { type HarnessType, harnessSupportsModel } from '@tangle-network/agent-interface'\nimport { ValidationError } from './errors'\nimport { canonicalize } from './pre-registration'\n\nexport type { AgentProfile, HarnessType } from '@tangle-network/agent-interface'\n\n/**\n * The agentic coding harnesses an eval sweeps by default — the ones we care about\n * ranking. This is the SINGLE source of that list; consumers import it instead of\n * re-declaring their own (a re-declared list is how the fleet drifts). Pass an\n * explicit `harnesses` (e.g. `harnessTypeSchema.options` for literally every known\n * harness) to widen beyond these.\n */\nexport const CODING_HARNESSES: readonly HarnessType[] = [\n 'opencode',\n 'claude-code',\n 'codex',\n 'kimi-code',\n]\n\nexport interface ProfileAxisSpec {\n /** The domain profile to sweep. Its prompt/tools/skills are held fixed; only the\n * harness and model vary. `model.default` is the fallback model. */\n base: AgentProfile\n /** Harnesses to cross. Default: {@link CODING_HARNESSES}. */\n harnesses?: readonly HarnessType[]\n /** Models to cross. Default: `[base.model.default]` — one model, i.e. today's\n * single-model behaviour, so omitting this never changes an existing run. */\n models?: readonly string[]\n /** Force every (harness, model) pair verbatim, even ones the harness can't run —\n * for deliberately testing failure modes. Default (false): SNAP instead — a\n * vendor-locked harness runs only the swept models in its family, or its native\n * default when it supports none, so no harness is dropped and none gets a\n * guaranteed-failing foreign-model cell. */\n keepIncompatible?: boolean\n}\n\n/** Model sentinel for a vendor-locked harness that supports none of the swept models:\n * it carries no provider prefix, so `harnessSupportsModel` accepts it and the harness\n * resolves it to its own native default model at runtime (e.g. kimi-code → its Kimi\n * model). Lets `expandProfileAxes` snap-instead-of-drop without a per-harness flagship\n * table that would rot as router catalogs change. */\nexport const HARNESS_NATIVE_MODEL = 'default'\n\n/**\n * Expand a base profile across the harness × model matrix into the `AgentProfile[]`\n * that `runProfileMatrix` / `selfImprove` score — the ONE place \"which harnesses ×\n * which models do we evaluate\" lives, so no product hand-rolls its own harness list\n * or column→profile mapping (the pattern that let those copies drift and silently\n * break the harness pivot).\n *\n * Each cell clones `base`, sets `model.default`, and stamps `metadata.harness` +\n * `metadata.harnessModel` (both hash-bearing, so every cell gets a distinct\n * `agentProfileId` row and results join back by harness/model via {@link harnessAxisOf}\n * with no hand-recomputed key). A vendor-locked harness snaps to its family's swept\n * models — or its native default ({@link HARNESS_NATIVE_MODEL}) when it supports none —\n * so every requested harness runs; `keepIncompatible` forces every pair verbatim.\n *\n * Omit `harnesses`/`models` to sweep the full default set — the \"turn it on for\n * everything we care about\" switch, identical in shape whether one harness or all.\n */\nexport function expandProfileAxes(spec: ProfileAxisSpec): AgentProfile[] {\n const harnesses = spec.harnesses ?? CODING_HARNESSES\n if (harnesses.length === 0) throw new ValidationError('expandProfileAxes: no harnesses to sweep')\n const baseModel = spec.base.model?.default\n const models = spec.models ?? (baseModel ? [baseModel] : [])\n if (models.length === 0) {\n throw new ValidationError(\n 'expandProfileAxes: no models to sweep — base profile has no model.default and none were supplied',\n )\n }\n const out: AgentProfile[] = []\n const seen = new Set<string>()\n for (const harness of harnesses) {\n // A universal (router-backed) harness — opencode/pi/claudish — runs every swept\n // model. A vendor-locked harness — codex/claude-code/kimi-code — runs only the\n // swept models in its own family; when it supports NONE of them it snaps to its\n // native default (the `HARNESS_NATIVE_MODEL` sentinel it resolves at runtime)\n // rather than being dropped, so every requested harness still appears in the\n // sweep on a model it can actually run — e.g. sweeping `deepseek/x` puts opencode\n // on deepseek and kimi-code on its own Kimi model, a real head-to-head.\n // `keepIncompatible` forces every (harness, model) pair verbatim (failure-mode runs).\n const supported = spec.keepIncompatible\n ? models\n : models.filter((model) => harnessSupportsModel(harness, model))\n const effective = supported.length > 0 ? supported : [HARNESS_NATIVE_MODEL]\n for (const model of effective) {\n const profile: AgentProfile = {\n ...spec.base,\n name: `${spec.base.name ?? 'agent'}/${harness}/${model}`,\n model: { ...spec.base.model, default: model },\n metadata: { ...(spec.base.metadata ?? {}), harness, harnessModel: model },\n }\n const id = agentProfileId(profile)\n if (seen.has(id)) continue\n seen.add(id)\n out.push(profile)\n }\n }\n if (out.length === 0) {\n // Unreachable in normal use — snapping guarantees ≥1 cell per harness — but keep a\n // fail-closed guard so a future refactor can't silently produce an empty sweep.\n throw new ValidationError(\n `expandProfileAxes: produced no profiles (harnesses=[${harnesses.join(', ')}], models=[${models.join(', ')}]).`,\n )\n }\n return out\n}\n\n/**\n * Read the (harness, model) a matrix cell ran under, off a profile or a result row's\n * profile — the join-back for a `byHarness` pivot. Returns undefined when the profile\n * wasn't produced by {@link expandProfileAxes}. Callers group `result.byProfile` by\n * this instead of recomputing an id (recomputing the wrong key is what broke the pivot\n * in the hand-rolled copies).\n */\nexport function harnessAxisOf(\n profile: Pick<AgentProfile, 'metadata'>,\n): { harness: HarnessType; model: string } | undefined {\n const m = profile.metadata as Record<string, unknown> | undefined\n const harness = m?.harness\n const model = m?.harnessModel\n if (typeof harness === 'string' && typeof model === 'string') {\n return { harness: harness as HarnessType, model }\n }\n return undefined\n}\n\n/**\n * Collision-resistant, path-safe, human-readable profile id for eval artifacts.\n * Scorecard joins still use `agentProfileHash`; this id is for run ids, matrix\n * keys, and directory names where two profiles must not collapse onto one row.\n * The suffix is the first 64 bits of the behaviour hash, enough for ordinary\n * eval matrices while keeping filenames readable.\n */\nexport function agentProfileId(profile: AgentProfile): string {\n const label = pathSafeProfileLabel(agentProfileDisplayLabel(profile)) ?? 'profile'\n return `${label}-${agentProfileHash(profile).slice(0, 16)}`\n}\n\n/**\n * Model snapshot used for `RunRecord.model`. Eval surfaces require a concrete\n * model id because run records reject bare/missing model aliases.\n */\nexport function agentProfileModelId(profile: AgentProfile): string {\n const model = profile.model?.default?.trim()\n if (!model) {\n const label = agentProfileDisplayLabel(profile) ?? 'unnamed profile'\n throw new ValidationError(\n `AgentProfile \"${label}\" has no model.default — cannot record eval run`,\n )\n }\n return model\n}\n\nfunction agentProfileDisplayLabel(profile: AgentProfile): string | undefined {\n return profile.name?.trim() || profile.version?.trim() || undefined\n}\n\nfunction pathSafeProfileLabel(label: string | undefined): string | undefined {\n const safe = label\n ?.trim()\n .replace(/[^A-Za-z0-9._-]+/g, '-')\n .replace(/-+/g, '-')\n .replace(/^-|-$/g, '')\n return safe || undefined\n}\n\nfunction compact<T extends Record<string, unknown>>(input: T): Partial<T> {\n const out: Record<string, unknown> = {}\n for (const [key, value] of Object.entries(input)) {\n if (value !== undefined) out[key] = value\n }\n return out as Partial<T>\n}\n\n/**\n * Deterministic behaviour identity for the canonical\n * `@tangle-network/agent-interface` AgentProfile.\n *\n * `name` and `description` are labels and do not affect the hash. Profile\n * `version`, prompt, model hints, tools, resources, hooks, modes, permissions,\n * and extensions do affect the hash. Resource array order is hash-bearing\n * because mount order can change agent behaviour. Undefined fields are treated\n * as absent; explicit `null` fields remain hash-bearing.\n */\nexport function agentProfileHash(profile: AgentProfile): string {\n const model = agentProfileModelId(profile)\n const behaviour = {\n ...profile,\n name: undefined,\n description: undefined,\n tags: profile.tags ? [...profile.tags].sort() : undefined,\n model: compact({ ...profile.model, default: model }),\n }\n return createHash('sha256')\n .update(JSON.stringify(canonicalize(behaviour)))\n .digest('hex')\n}\n"],"mappings":";;;;;;;;;;;;;;;AAWO,IAAM,kBAAN,cAA8B,WAAW;AAAA;AAAA,EAErC;AAAA;AAAA,EAEA;AAAA,EAET,YAAY,WAAmB,KAAa,SAA+B;AACzE,UAAM,UAAU,SAAS,uCAAuC,IAAI,MAAM,GAAG,GAAG,CAAC,IAAI,OAAO;AAC5F,SAAK,YAAY;AACjB,SAAK,MAAM;AAAA,EACb;AACF;AAYO,SAAS,wBAAwB,QAAyB;AAC/D,SAAO,OACL,IACA,EAAE,UAAU,MAAM,MACQ;AAC1B,UAAM,eAAe,MAClB;AAAA,MACC,CAAC,GAAG,MACF,QAAQ,IAAI,CAAC;AAAA,QAAY,EAAE,WAAW;AAAA,SAAY,EAAE,cAAc,MAAM,GAAG,GAAI,CAAC;AAAA,IACpF,EACC,KAAK,aAAa;AAErB,UAAM,OAAO,MAAM,GAAG,KAAK;AAAA,MACzB,OAAO;AAAA,MACP,UAAU;AAAA,QACR;AAAA,UACE,MAAM;AAAA,UACN,SAAS,oBAAoB,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,QASrC;AAAA,QACA;AAAA,UACE,MAAM;AAAA,UACN,SAAS,YAAY,SAAS,OAAO,KAAK,SAAS,KAAK;AAAA,YAAgB,SAAS,MAAM;AAAA;AAAA,EAAO,YAAY;AAAA,QAC5G;AAAA,MACF;AAAA,MACA,aAAa;AAAA,MACb,WAAW;AAAA,IACb,CAAC;AAED,WAAO,mBAAmB,iBAAiB,IAAI;AAAA,EACjD;AACF;AASO,IAAM,qBAA8B,OAAO,IAAI,EAAE,UAAU,UAAU,MAAM;AAChF,QAAM,aAAa,UAAU;AAC7B,MAAI,WAAW,WAAW,GAAG;AAC3B,WAAO;AAAA,MACL;AAAA,QACE,WAAW;AAAA,QACX,WAAW;AAAA,QACX,OAAO;AAAA,QACP,WAAW;AAAA,MACb;AAAA,IACF;AAAA,EACF;AAEA,QAAM,WAAW,WACd;AAAA,IACC,CAAC,GAAG,MACF,SAAS,IAAI,CAAC,KAAK,EAAE,QAAQ;AAAA,QAAa,EAAE,QAAQ;AAAA,EAAK,EAAE,KAAK,MAAM,GAAG,GAAI,CAAC;AAAA;AAAA,EAClF,EACC,KAAK,MAAM;AAEd,QAAM,OAAO,MAAM,GAAG,KAAK;AAAA,IACzB,OAAO;AAAA,IACP,UAAU;AAAA,MACR;AAAA,QACE,MAAM;AAAA,QACN,SAAS;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAQX;AAAA,MACA;AAAA,QACE,MAAM;AAAA,QACN,SAAS,SAAS,SAAS,MAAM;AAAA;AAAA,EAAO,QAAQ;AAAA,MAClD;AAAA,IACF;AAAA,IACA,aAAa;AAAA,IACb,WAAW;AAAA,EACb,CAAC;AAED,SAAO,mBAAmB,kBAAkB,IAAI;AAClD;AASO,IAAM,iBAA0B,OAAO,IAAI,EAAE,UAAU,MAAM,MAAM;AACxE,MAAI,MAAM,SAAS,GAAG;AAIpB,WAAO,CAAC;AAAA,EACV;AAEA,QAAM,eAAe,MAClB;AAAA,IACC,CAAC,GAAG,MACF,QAAQ,IAAI,CAAC;AAAA,QAAY,EAAE,WAAW;AAAA,SAAY,EAAE,cAAc,MAAM,YAAY,EAAE,cAAc,MAAM,GAAG,IAAI,CAAC;AAAA,EACtH,EACC,KAAK,aAAa;AAErB,QAAM,OAAO,MAAM,GAAG,KAAK;AAAA,IACzB,OAAO;AAAA,IACP,UAAU;AAAA,MACR;AAAA,QACE,MAAM;AAAA,QACN,SAAS;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAQX;AAAA,MACA;AAAA,QACE,MAAM;AAAA,QACN,SAAS,aAAa,SAAS,MAAM;AAAA;AAAA,EAAO,YAAY;AAAA,MAC1D;AAAA,IACF;AAAA,IACA,aAAa;AAAA,IACb,WAAW;AAAA,EACb,CAAC;AAED,SAAO,mBAAmB,aAAa,IAAI;AAC7C;AASO,IAAM,mBAA4B,OAAO,IAAI,EAAE,UAAU,MAAM,MAAM;AAC1E,QAAM,eAAe,MAClB;AAAA,IACC,CAAC,GAAG,MAAM,QAAQ,IAAI,CAAC;AAAA,QAAY,EAAE,WAAW;AAAA,SAAY,EAAE,cAAc,MAAM,GAAG,IAAI,CAAC;AAAA,EAC5F,EACC,KAAK,aAAa;AAErB,QAAM,OAAO,MAAM,GAAG,KAAK;AAAA,IACzB,OAAO;AAAA,IACP,UAAU;AAAA,MACR;AAAA,QACE,MAAM;AAAA,QACN,SAAS;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MASX;AAAA,MACA;AAAA,QACE,MAAM;AAAA,QACN,SAAS,YAAY,SAAS,OAAO;AAAA,YAAe,SAAS,MAAM;AAAA;AAAA,EAAO,YAAY;AAAA,MACxF;AAAA,IACF;AAAA,IACA,aAAa;AAAA,IACb,WAAW;AAAA,EACb,CAAC;AAED,SAAO,mBAAmB,eAAe,IAAI;AAC/C;AASO,SAAS,kBACd,MACA,cACA,MACS;AACT,SAAO,OAAO,IAAI,EAAE,UAAU,MAAM,MAAM;AACxC,UAAM,eAAe,MAClB;AAAA,MACC,CAAC,GAAG,MACF,QAAQ,IAAI,CAAC;AAAA,QAAY,EAAE,WAAW;AAAA,SAAY,EAAE,cAAc,MAAM,GAAG,GAAI,CAAC;AAAA,IACpF,EACC,KAAK,aAAa;AAErB,UAAM,OAAO,MAAM,GAAG,KAAK;AAAA,MACzB,OAAO,MAAM,SAAS;AAAA,MACtB,UAAU;AAAA,QACR;AAAA,UACE,MAAM;AAAA,UACN,SAAS;AAAA,QACX;AAAA,QACA;AAAA,UACE,MAAM;AAAA,UACN,SAAS,YAAY,SAAS,OAAO,KAAK,SAAS,KAAK;AAAA,YAAgB,SAAS,MAAM;AAAA;AAAA,EAAO,YAAY;AAAA,QAC5G;AAAA,MACF;AAAA,MACA,aAAa,MAAM,eAAe;AAAA,MAClC,WAAW,MAAM,aAAa;AAAA,IAChC,CAAC;AAED,WAAO,mBAAmB,MAAM,IAAI;AAAA,EACtC;AACF;AASO,SAAS,cAAc,QAA2B;AACvD,SAAO,CAAC,wBAAwB,MAAM,GAAG,oBAAoB,gBAAgB,gBAAgB;AAC/F;AAIA,SAAS,mBAAmB,WAAmB,MAA6B;AAC1E,QAAM,UACH,KAA4D,UAAU,CAAC,GAAG,SAAS,WACpF;AACF,MAAI;AACF,QAAI,UAAU,QAAQ,QAAQ,sBAAsB,EAAE,EAAE,KAAK;AAC7D,UAAM,aAAa,QAAQ,MAAM,aAAa;AAC9C,QAAI,WAAY,WAAU,WAAW,CAAC;AACtC,UAAM,SAAS,KAAK,MAAM,OAAO;AAMjC,WAAO,OAAO,IAAI,CAAC,OAAO;AAAA,MACxB;AAAA,MACA,WAAW,EAAE;AAAA,MACb,OAAO,KAAK,IAAI,GAAG,KAAK,IAAI,IAAI,EAAE,KAAK,CAAC;AAAA,MACxC,WAAW,EAAE,aAAa;AAAA,MAC1B,UAAU,EAAE;AAAA,IACd,EAAE;AAAA,EACJ,SAAS,KAAK;AAGZ,UAAM,IAAI,gBAAgB,WAAW,SAAS,EAAE,OAAO,IAAI,CAAC;AAAA,EAC9D;AACF;;;ACpMO,SAAS,kBAAkB,OAGZ;AACpB,MAAI,MAAM,aAAa,WAAW,GAAG;AACnC,UAAM,IAAI;AAAA,MACR,4BAA4B,MAAM,MAAM;AAAA,IAC1C;AAAA,EACF;AACA,QAAM,iBAAiB,MAAM,aAAa,OAAO,CAAC,MAAM,EAAE,SAAS,EAAE;AACrE,QAAM,iBAAiB,iBAAiB,MAAM,aAAa;AAC3D,QAAM,gBAAgB,mBAAmB,MAAM,aAAa;AAC5D,SAAO;AAAA,IACL,QAAQ,MAAM;AAAA,IACd,cAAc,MAAM;AAAA,IACpB;AAAA,IACA;AAAA,IACA,OAAO;AAAA,IACP,OAAO;AAAA,EACT;AACF;AAYA,IAAM,YAAY,oBAAI,IAAI;AAAA,EACxB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAUD,IAAM,6BAA6B,oBAAI,IAAI;AAAA,EACzC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAED,IAAM,kBAAkB;AACxB,IAAM,oBAAoB;AAE1B,SAAS,OAAO,GAAW,WAAsC;AAC/D,SAAO,IAAI;AAAA,IACT,EACG,YAAY,EACZ,MAAM,YAAY,EAClB,OAAO,CAAC,MAAM,EAAE,SAAS,KAAK,CAAC,UAAU,IAAI,CAAC,KAAK,CAAC,WAAW,IAAI,CAAC,CAAC;AAAA,EAC1E;AACF;AAQA,SAAS,YAAY,iBAAyB,eAA+B;AAC3E,QAAM,MAAM,OAAO,iBAAiB,0BAA0B;AAC9D,MAAI,IAAI,SAAS,EAAG,QAAO;AAC3B,QAAM,OAAO,OAAO,aAAa;AACjC,MAAI,MAAM;AACV,aAAW,KAAK,IAAK,KAAI,KAAK,IAAI,CAAC,EAAG;AACtC,SAAO,MAAM,IAAI;AACnB;AAYA,SAAS,mBACP,KACA,UACA,WACa;AACb,QAAM,UAAU,GAAG,IAAI,KAAK,IAAI,IAAI,YAAY,EAAE;AAClD,QAAM,MAAmB,CAAC;AAC1B,YAAU,QAAQ,CAAC,GAAG,MAAM;AAC1B,SAAK,EAAE,WAAW,IAAI,KAAK,EAAE,SAAS,kBAAmB;AAKzD,QAAI,QAAQ;AAAA,MACV;AAAA,MACA,GAAG,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,KAAK,EAAE,WAAW,IAAI,MAAM,GAAG,GAAI,CAAC;AAAA,IAC/D;AACA,QAAI,IAAI,YAAY,EAAE,QAAQ,IAAI,SAAS,YAAY,MAAM,EAAE,KAAK,YAAY,GAAG;AACjF,cAAQ,KAAK,IAAI,OAAO,CAAC;AAAA,IAC3B;AACA,QAAI,QAAQ,gBAAiB;AAC7B,QAAI,KAAK;AAAA,MACP;AAAA,MACA,SAAS,YAAY,CAAC;AAAA,MACtB;AAAA,MACA,UAAU,aAAa,EAAE,QAAQ,EAAE,IAAI,2BAA2B,MAAM,QAAQ,CAAC,CAAC;AAAA,MAClF,SAAS,EAAE,WAAW;AAAA,IACxB,CAAC;AAAA,EACH,CAAC;AACD,SAAO;AACT;AAEA,SAAS,mBACP,KACA,UACA,WACa;AACb,QAAM,UAAU,GAAG,IAAI,KAAK,IAAI,IAAI,YAAY,EAAE;AAClD,QAAM,MAAmB,CAAC;AAC1B,aAAW,KAAK,WAAW;AAEzB,QAAI,EAAE,WAAW,WAAY;AAM7B,UAAM,QAAQ,EAAE,WAAW,IAAI,KAAK;AACpC,QAAI,KAAK,SAAS,kBAAmB;AASrC,UAAM,QAAQ,YAAY,SAAS,GAAG,EAAE,KAAK,IAAI,IAAI,EAAE;AACvD,QAAI,QAAQ,gBAAiB;AAC7B,QAAI,KAAK;AAAA,MACP;AAAA,MACA,SAAS,YAAY,EAAE,EAAE;AAAA,MACzB;AAAA,MACA,UAAU,sBAAsB,EAAE,KAAK,2BAA2B,MAAM,QAAQ,CAAC,CAAC;AAAA,MAClF,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AACA,SAAO;AACT;AAEA,SAAS,mBACP,KACA,UACA,WACa;AACb,QAAM,MAAmB,CAAC;AAC1B,YAAU,QAAQ,CAAC,MAAM,MAAM;AAC7B,UAAM,QAAQ,YAAY,IAAI,OAAO,IAAI;AACzC,QAAI,QAAQ,gBAAiB;AAC7B,QAAI,KAAK;AAAA,MACP;AAAA,MACA,SAAS,QAAQ,CAAC;AAAA,MAClB;AAAA,MACA,UAAU,cAAc,IAAI,2BAA2B,MAAM,QAAQ,CAAC,CAAC;AAAA,MACvE,SAAS;AAAA,IACX,CAAC;AAAA,EACH,CAAC;AACD,SAAO;AACT;AASA,eAAsB,iBACpB,MACA,OACA,kBAC4B;AAC5B,MAAI,KAAK,aAAa,WAAW,GAAG;AAClC,UAAM,IAAI;AAAA,MACR,2BAA2B,KAAK,MAAM;AAAA,IACxC;AAAA,EACF;AAKA,QAAM,aAA0B,CAAC;AACjC,OAAK,aAAa,QAAQ,CAAC,KAAK,MAAM;AACpC,UAAM,KAAK,IAAI,eAAe;AAC9B,QAAI,OAAO,cAAc,OAAO,OAAO;AACrC,iBAAW,KAAK,GAAG,mBAAmB,KAAK,GAAG,MAAM,SAAS,CAAC;AAAA,IAChE;AACA,QAAI,OAAO,cAAc,OAAO,OAAO;AACrC,iBAAW,KAAK,GAAG,mBAAmB,KAAK,GAAG,MAAM,SAAS,CAAC;AAAA,IAChE;AACA,QAAI,OAAO,eAAe,OAAO,OAAO;AACtC,iBAAW,KAAK,GAAG,mBAAmB,KAAK,GAAG,MAAM,SAAS,CAAC;AAAA,IAChE;AAAA,EACF,CAAC;AACD,aAAW,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;AAE3C,QAAM,WAAW,oBAAI,IAAuB;AAC5C,QAAM,YAAY,oBAAI,IAAY;AAClC,aAAW,KAAK,YAAY;AAC1B,QAAI,SAAS,IAAI,EAAE,QAAQ,KAAK,UAAU,IAAI,EAAE,OAAO,EAAG;AAC1D,aAAS,IAAI,EAAE,UAAU,CAAC;AAC1B,cAAU,IAAI,EAAE,OAAO;AAAA,EACzB;AAEA,QAAM,eAAmC,CAAC;AAC1C,WAAS,IAAI,GAAG,IAAI,KAAK,aAAa,QAAQ,KAAK;AACjD,UAAM,MAAM,KAAK,aAAa,CAAC;AAC/B,UAAM,QAAQ,SAAS,IAAI,CAAC;AAC5B,UAAM,WAAqB,CAAC;AAC5B,QAAI,UAA0B;AAE9B,QAAI,OAAO;AACT,eAAS,KAAK,MAAM,QAAQ;AAC5B,UAAI,MAAM,YAAY,MAAM;AAC1B,cAAM,IAAI,MAAM,iBAAiB,KAAK,MAAM,OAAO;AACnD,kBAAU,EAAE;AACZ,iBAAS,KAAK,gBAAgB,EAAE,UAAU,SAAS,MAAM,WAAM,EAAE,MAAM,EAAE;AAAA,MAC3E,OAAO;AACL,iBAAS,KAAK,kEAA6D;AAAA,MAC7E;AAAA,IACF,OAAO;AACL,YAAM,KAAK,IAAI,eAAe;AAC9B,YAAM,OAAO,OAAO,QAAQ,gCAAgC;AAC5D,eAAS,KAAK,eAAe,IAAI,2BAA2B;AAAA,IAC9D;AAEA,UAAM,sBAAsB,UAAU;AACtC,UAAM,YAAY,uBAAuB,YAAY;AACrD,iBAAa,KAAK;AAAA,MAChB,OAAO,IAAI;AAAA,MACX,OAAO,IAAI;AAAA,MACX;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAEA,SAAO,kBAAkB,EAAE,QAAQ,KAAK,QAAQ,aAAa,CAAC;AAChE;AASO,SAAS,yBAAyB,KAAmD;AAC1F,QAAM,QAAQ,IAAI,MAAM,aAAa;AACrC,MAAI,CAAC,OAAO;AACV,UAAM,IAAI,MAAM,0DAA0D,IAAI,MAAM,GAAG,GAAG,CAAC,EAAE;AAAA,EAC/F;AACA,QAAM,SAAS,KAAK,MAAM,MAAM,CAAC,CAAC;AAClC,MAAI,OAAO,OAAO,YAAY,WAAW;AACvC,UAAM,IAAI,MAAM,uDAAuD,MAAM,CAAC,EAAE,MAAM,GAAG,GAAG,CAAC,EAAE;AAAA,EACjG;AACA,SAAO,EAAE,SAAS,OAAO,SAAS,QAAQ,OAAO,OAAO,WAAW,WAAW,OAAO,SAAS,GAAG;AACnG;AAQO,SAAS,4BACd,IACA,OAAkC,CAAC,GACf;AACpB,QAAM,QAAQ,KAAK,SAAS;AAC5B,QAAM,kBAAkB,KAAK,mBAAmB;AAChD,SAAO,OAAO,aAAa,YAAY;AACrC,UAAM,OAAO,MAAM,GAAG,KAAK;AAAA,MACzB;AAAA,MACA,UAAU;AAAA,QACR;AAAA,UACE,MAAM;AAAA,UACN,SACE;AAAA,QACJ;AAAA,QACA;AAAA,UACE,MAAM;AAAA,UACN,SAAS,gBAAgB,YAAY,KAAK;AAAA,EACxC,YAAY,WAAW,aAAa,YAAY,QAAQ;AAAA,IAAO,EACjE;AAAA;AAAA,EAAyB,QAAQ,MAAM,GAAG,eAAe,CAAC;AAAA,QAC5D;AAAA,MACF;AAAA,MACA,aAAa;AAAA,MACb,WAAW;AAAA,IACb,CAAC;AACD,UAAM,MACH,KAA4D,UAAU,CAAC,GAAG,SAAS,WACpF;AACF,WAAO,yBAAyB,GAAG;AAAA,EACrC;AACF;AAKA,IAAM,kBAAkB,oBAAI,IAAI;AAAA,EAC9B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAgBM,SAAS,yBACd,OAA0D,CAAC,GACvC;AACpB,QAAM,YAAY,KAAK,aAAa;AACpC,QAAM,SAAS,KAAK,oBAAoB;AACxC,SAAO,OAAO,aAAa,YAAY;AACrC,UAAM,OAAO,QAAQ,KAAK;AAC1B,QAAI,KAAK,SAAS;AAChB,aAAO;AAAA,QACL,SAAS;AAAA,QACT,QAAQ,qBAAqB,KAAK,MAAM;AAAA,MAC1C;AACF,UAAM,cAAc,YAAY,MAC7B,YAAY,EACZ,MAAM,YAAY,EAClB,OAAO,CAAC,MAAM,EAAE,SAAS,KAAK,CAAC,gBAAgB,IAAI,CAAC,CAAC;AACxD,QAAI,YAAY,WAAW;AACzB,aAAO;AAAA,QACL,SAAS;AAAA,QACT,QAAQ;AAAA,MACV;AACF,UAAM,QAAQ,KAAK,YAAY;AAC/B,UAAM,OAAO,YAAY,OAAO,CAAC,MAAM,MAAM,SAAS,CAAC,CAAC,EAAE;AAC1D,UAAM,SAAS,OAAO,YAAY;AAClC,WAAO,UAAU,YACb;AAAA,MACE,SAAS;AAAA,MACT,QAAQ,mBAAmB,IAAI,IAAI,YAAY,MAAM;AAAA,IACvD,IACA;AAAA,MACE,SAAS;AAAA,MACT,QAAQ,wBAAwB,IAAI,IAAI,YAAY,MAAM;AAAA,IAC5D;AAAA,EACN;AACF;;;AC9aO,SAAS,SACd,MACA,QACA,MACmC;AACnC,MAAI,CAAC,KAAK,KAAK,GAAG;AAChB,UAAM,IAAI,MAAM,kCAAkC;AAAA,EACpD;AACA,MAAI,CAAC,OAAO,KAAK,GAAG;AAClB,UAAM,IAAI,MAAM,aAAa,IAAI,6BAA6B;AAAA,EAChE;AACA,QAAM,QAAQ,KAAK,SAAS,KAAK,KAAK;AACtC,MAAI,CAAC,OAAO;AACV,UAAM,IAAI;AAAA,MACR,aAAa,IAAI;AAAA,IAEnB;AAAA,EACF;AAEA,QAAM,aAAa,oBAAoB,KAAK,YAAY,IAAI;AAC5D,QAAM,QAAQ,KAAK,SAAS;AAC5B,QAAM,UAAU,UAAU,QAAQ,KAAK;AACvC,QAAM,aACJ,KAAK,eACJ,CAAC,UACA,KAAK,UAAU,EAAE,UAAU,MAAM,UAAU,UAAU,MAAM,SAAS,GAAG,MAAM,CAAC;AAElF,MAAI,KAAK,SAAS;AAChB,eAAW,OAAO,OAAO,KAAK,KAAK,OAAO,GAAG;AAC3C,UAAI,CAAC,WAAW,KAAK,CAAC,MAAM,EAAE,QAAQ,GAAG,GAAG;AAC1C,cAAM,IAAI;AAAA,UACR,aAAa,IAAI,+BAA+B,GAAG;AAAA,QACrD;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,QAAM,eAAe,GAAG,MAAM;AAAA;AAAA,EAAO,eAAe,YAAY,KAAK,CAAC;AAEtE,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,WAAW,KAAK;AAAA,IAChB,MAAM,MAAM,EAAE,UAAU,UAAU,OAAO,GAAwB;AAC/D,YAAM,WAAW,MAAM,KAAK,KAAK;AAAA,QAC/B;AAAA,UACE;AAAA,UACA,UAAU;AAAA,YACR,EAAE,MAAM,UAAU,SAAS,aAAa;AAAA,YACxC,EAAE,MAAM,QAAQ,SAAS,WAAW,EAAE,UAAU,SAAS,CAAC,EAAE;AAAA,UAC9D;AAAA,UACA,UAAU;AAAA,UACV,aAAa,KAAK,eAAe;AAAA,UACjC,WAAW,KAAK,aAAa;AAAA,QAC/B;AAAA,QACA,EAAE,OAAO;AAAA,MACX;AAEA,YAAM,SAAS,cAAc,MAAM,SAAS,OAAO;AACnD,YAAM,UAAU,OAAO,cAAc,OAAO;AAC5C,UAAI,CAAC,WAAW,OAAO,YAAY,UAAU;AAC3C,cAAM,IAAI,gBAAgB,MAAM,SAAS,SAAS;AAAA,UAChD,OAAO,IAAI,MAAM,qCAAqC;AAAA,QACxD,CAAC;AAAA,MACH;AAEA,YAAM,OAA+B,CAAC;AACtC,iBAAW,EAAE,IAAI,KAAK,YAAY;AAChC,cAAM,MAAO,QAAoC,GAAG;AACpD,cAAM,QAAQ,OAAO,GAAG;AACxB,YAAI,QAAQ,UAAa,QAAQ,QAAQ,CAAC,OAAO,SAAS,KAAK,GAAG;AAChE,gBAAM,IAAI,gBAAgB,MAAM,SAAS,SAAS;AAAA,YAChD,OAAO,IAAI;AAAA,cACT,cAAc,GAAG,iCAAiC,KAAK,UAAU,GAAG,CAAC;AAAA,YACvE;AAAA,UACF,CAAC;AAAA,QACH;AACA,aAAK,GAAG,IAAI,QAAQ,QAAQ,OAAO;AAAA,MACrC;AAEA,YAAM,UACJ,KAAK,WAAW,OAAO,YAAY,WAAW,IAAI,CAAC,MAAM,CAAC,EAAE,KAAK,IAAI,WAAW,MAAM,CAAC,CAAC;AAC1F,YAAM,EAAE,UAAU,IAAI,kBAAkB,EAAE,MAAM,QAAQ,CAAC;AAEzD,YAAM,QACJ,YAAY,OAAO,KAAK,KACxB,YAAY,OAAO,SAAS,KAC5B,GAAG,IAAI,eAAe,UAAU,QAAQ,CAAC,CAAC,SAAS,WAAW,MAAM;AAEtE,aAAO,EAAE,YAAY,MAAM,WAAW,MAAM;AAAA,IAC9C;AAAA,EACF;AACF;AAEA,SAAS,oBACP,OACA,MACkB;AAClB,QAAM,MAAM,SAAS,MAAM,SAAS,IAAI,QAAQ,CAAC,SAAS;AAC1D,QAAM,MAAwB,CAAC;AAC/B,QAAM,OAAO,oBAAI,IAAY;AAC7B,aAAW,KAAK,KAAK;AACnB,UAAM,MAAM,OAAO,MAAM,WAAW,EAAE,KAAK,GAAG,aAAa,EAAE,IAAI;AACjE,QAAI,CAAC,IAAI,IAAI,KAAK,GAAG;AACnB,YAAM,IAAI,MAAM,aAAa,IAAI,oCAAoC;AAAA,IACvE;AACA,QAAI,KAAK,IAAI,IAAI,GAAG,GAAG;AACrB,YAAM,IAAI,MAAM,aAAa,IAAI,+BAA+B,IAAI,GAAG,GAAG;AAAA,IAC5E;AACA,SAAK,IAAI,IAAI,GAAG;AAChB,QAAI,KAAK,GAAG;AAAA,EACd;AACA,SAAO;AACT;AAEA,SAAS,eAAe,YAA8B,OAA+B;AACnF,QAAM,QAAQ,UAAU,QAAQ,YAAY;AAC5C,QAAM,QAAQ,WAAW,IAAI,CAAC,MAAM,QAAQ,EAAE,GAAG,MAAM,EAAE,WAAW,WAAW,KAAK,GAAG;AACvF,QAAM,UAAU,mBAAmB,WAChC,IAAI,CAAC,MAAM,IAAI,EAAE,GAAG,aAAa,EACjC,KAAK,IAAI,CAAC;AACb,SAAO;AAAA,IACL;AAAA,IACA,GAAG;AAAA,IACH;AAAA,IACA,qEAAqE,KAAK;AAAA,IAC1E;AAAA,EACF,EAAE,KAAK,IAAI;AACb;AAEA,SAAS,cAAc,MAAc,SAAmC;AACtE,QAAM,WAAW,QAAQ,QAAQ,sBAAsB,EAAE,EAAE,KAAK;AAChE,QAAM,WAAW,SAAS,MAAM,aAAa;AAC7C,QAAM,UAAU,WAAW,SAAS,CAAC,IAAI;AACzC,MAAI;AACF,UAAM,SAAS,KAAK,MAAM,OAAO;AACjC,QAAI,OAAO,WAAW,YAAY,WAAW,MAAM;AACjD,YAAM,IAAI,MAAM,+BAA+B;AAAA,IACjD;AACA,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,UAAM,IAAI,gBAAgB,MAAM,SAAS,EAAE,OAAO,IAAI,CAAC;AAAA,EACzD;AACF;AAEA,SAAS,YAAY,OAAoC;AACvD,SAAO,OAAO,UAAU,YAAY,MAAM,KAAK,IAAI,QAAQ;AAC7D;;;ACpKA,SAAS,aAAa,UAAsC;AAC1D,MAAI,CAAC,SAAU,QAAO;AACtB,MAAI,SAAS,SAAS,MAAM,EAAG,QAAO;AACtC,MAAI,SAAS,WAAW,OAAO,EAAG,QAAO;AACzC,SAAO;AACT;AAYO,SAAS,qBAAqB,QAAoD;AACvF,QAAM,YAAwB,CAAC;AAC/B,QAAM,YAAgC,CAAC;AACvC,QAAM,YAAsB,CAAC;AAC7B,QAAM,YAAY,oBAAI,IAAY;AAElC,aAAW,MAAM,QAAQ;AACvB,QAAI,GAAG,SAAS,aAAa;AAC3B,YAAM,OAAQ,GAAyB;AACvC,UAAI,QAAQ,CAAC,UAAU,IAAI,IAAI,GAAG;AAChC,kBAAU,IAAI,IAAI;AAClB,kBAAU,KAAK,IAAI;AAAA,MACrB;AAAA,IACF,WAAW,GAAG,SAAS,YAAY;AACjC,YAAM,IAAI;AACV,gBAAU,KAAK;AAAA,QACb,MAAM,aAAa,EAAE,QAAQ;AAAA,QAC7B,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE;AAAA,QAC3B,SAAS,EAAE,WAAW;AAAA,MACxB,CAAC;AAAA,IACH,WAAW,GAAG,SAAS,oBAAoB;AACzC,YAAM,IAAI;AACV,gBAAU,KAAK;AAAA,QACb,IAAI,EAAE;AAAA,QACN,OAAO,EAAE;AAAA,QACT,QAAQ,EAAE,UAAU;AAAA,QACpB,GAAI,EAAE,YAAY,SAAY,EAAE,SAAS,EAAE,QAAQ,IAAI,CAAC;AAAA,MAC1D,CAAC;AAAA,IACH;AAAA,EACF;AAEA,SAAO,EAAE,WAAW,WAAW,UAAU;AAC3C;;;AChHA,SAAS,kBAAkB;AAE3B,SAA2B,4BAA4B;AAahD,IAAM,mBAA2C;AAAA,EACtD;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAwBO,IAAM,uBAAuB;AAmB7B,SAAS,kBAAkB,MAAuC;AACvE,QAAM,YAAY,KAAK,aAAa;AACpC,MAAI,UAAU,WAAW,EAAG,OAAM,IAAI,gBAAgB,0CAA0C;AAChG,QAAM,YAAY,KAAK,KAAK,OAAO;AACnC,QAAM,SAAS,KAAK,WAAW,YAAY,CAAC,SAAS,IAAI,CAAC;AAC1D,MAAI,OAAO,WAAW,GAAG;AACvB,UAAM,IAAI;AAAA,MACR;AAAA,IACF;AAAA,EACF;AACA,QAAM,MAAsB,CAAC;AAC7B,QAAM,OAAO,oBAAI,IAAY;AAC7B,aAAW,WAAW,WAAW;AAS/B,UAAM,YAAY,KAAK,mBACnB,SACA,OAAO,OAAO,CAAC,UAAU,qBAAqB,SAAS,KAAK,CAAC;AACjE,UAAM,YAAY,UAAU,SAAS,IAAI,YAAY,CAAC,oBAAoB;AAC1E,eAAW,SAAS,WAAW;AAC7B,YAAM,UAAwB;AAAA,QAC5B,GAAG,KAAK;AAAA,QACR,MAAM,GAAG,KAAK,KAAK,QAAQ,OAAO,IAAI,OAAO,IAAI,KAAK;AAAA,QACtD,OAAO,EAAE,GAAG,KAAK,KAAK,OAAO,SAAS,MAAM;AAAA,QAC5C,UAAU,EAAE,GAAI,KAAK,KAAK,YAAY,CAAC,GAAI,SAAS,cAAc,MAAM;AAAA,MAC1E;AACA,YAAM,KAAK,eAAe,OAAO;AACjC,UAAI,KAAK,IAAI,EAAE,EAAG;AAClB,WAAK,IAAI,EAAE;AACX,UAAI,KAAK,OAAO;AAAA,IAClB;AAAA,EACF;AACA,MAAI,IAAI,WAAW,GAAG;AAGpB,UAAM,IAAI;AAAA,MACR,uDAAuD,UAAU,KAAK,IAAI,CAAC,cAAc,OAAO,KAAK,IAAI,CAAC;AAAA,IAC5G;AAAA,EACF;AACA,SAAO;AACT;AASO,SAAS,cACd,SACqD;AACrD,QAAM,IAAI,QAAQ;AAClB,QAAM,UAAU,GAAG;AACnB,QAAM,QAAQ,GAAG;AACjB,MAAI,OAAO,YAAY,YAAY,OAAO,UAAU,UAAU;AAC5D,WAAO,EAAE,SAAiC,MAAM;AAAA,EAClD;AACA,SAAO;AACT;AASO,SAAS,eAAe,SAA+B;AAC5D,QAAM,QAAQ,qBAAqB,yBAAyB,OAAO,CAAC,KAAK;AACzE,SAAO,GAAG,KAAK,IAAI,iBAAiB,OAAO,EAAE,MAAM,GAAG,EAAE,CAAC;AAC3D;AAMO,SAAS,oBAAoB,SAA+B;AACjE,QAAM,QAAQ,QAAQ,OAAO,SAAS,KAAK;AAC3C,MAAI,CAAC,OAAO;AACV,UAAM,QAAQ,yBAAyB,OAAO,KAAK;AACnD,UAAM,IAAI;AAAA,MACR,iBAAiB,KAAK;AAAA,IACxB;AAAA,EACF;AACA,SAAO;AACT;AAEA,SAAS,yBAAyB,SAA2C;AAC3E,SAAO,QAAQ,MAAM,KAAK,KAAK,QAAQ,SAAS,KAAK,KAAK;AAC5D;AAEA,SAAS,qBAAqB,OAA+C;AAC3E,QAAM,OAAO,OACT,KAAK,EACN,QAAQ,qBAAqB,GAAG,EAChC,QAAQ,OAAO,GAAG,EAClB,QAAQ,UAAU,EAAE;AACvB,SAAO,QAAQ;AACjB;AAEA,SAAS,QAA2C,OAAsB;AACxE,QAAM,MAA+B,CAAC;AACtC,aAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,KAAK,GAAG;AAChD,QAAI,UAAU,OAAW,KAAI,GAAG,IAAI;AAAA,EACtC;AACA,SAAO;AACT;AAYO,SAAS,iBAAiB,SAA+B;AAC9D,QAAM,QAAQ,oBAAoB,OAAO;AACzC,QAAM,YAAY;AAAA,IAChB,GAAG;AAAA,IACH,MAAM;AAAA,IACN,aAAa;AAAA,IACb,MAAM,QAAQ,OAAO,CAAC,GAAG,QAAQ,IAAI,EAAE,KAAK,IAAI;AAAA,IAChD,OAAO,QAAQ,EAAE,GAAG,QAAQ,OAAO,SAAS,MAAM,CAAC;AAAA,EACrD;AACA,SAAO,WAAW,QAAQ,EACvB,OAAO,KAAK,UAAU,aAAa,SAAS,CAAC,CAAC,EAC9C,OAAO,KAAK;AACjB;","names":[]}
@@ -1 +0,0 @@
1
- {"version":3,"sources":["../src/campaign/run-campaign.ts","../src/integrity/backend-integrity.ts","../src/verdict-cache.ts","../src/campaign/run-dir.ts","../src/campaign/storage.ts"],"sourcesContent":["/**\n * `runCampaign` — Pass A substrate primitive. ONE function that orchestrates\n * scenarios → dispatch → artifacts → judges → aggregates, with full\n * reproducibility (seed + manifest hash), cell-level resumability, bootstrap\n * CIs, and the `LabeledScenarioStore` capture flywheel.\n *\n * Improvement loops (optimizer / gate / autoOnPromote) ride on top of this\n * primitive but live in `presets/run-improvement-loop.ts`. This file keeps\n * the core orchestrator minimal — Phase 1 of the Pass A track.\n */\n\nimport { join } from 'node:path'\nimport { BackendIntegrityError, type BackendIntegrityReport } from '../integrity/backend-integrity'\nimport { confidenceInterval } from '../statistics'\nimport { contentHash } from '../verdict-cache'\nimport { resolveRunDir } from './run-dir'\nimport { type CampaignStorage, fsCampaignStorage } from './storage'\nimport type {\n CampaignAggregates,\n CampaignArtifactWriter,\n CampaignCellResult,\n CampaignCostMeter,\n CampaignResult,\n CampaignTokenUsage,\n CampaignTraceWriter,\n DispatchContext,\n DispatchFn,\n JudgeAggregate,\n JudgeConfig,\n JudgeScore,\n LabeledScenarioStore,\n Scenario,\n ScenarioAggregate,\n TraceSpan,\n} from './types'\n\nexport interface RunCampaignOptions<TScenario extends Scenario, TArtifact> {\n scenarios: TScenario[]\n dispatch: DispatchFn<TScenario, TArtifact>\n /**\n * Stable identity for the dispatch behavior, included in the manifest/cache\n * key. Set this when the same function name can run different models,\n * prompts, tools, or external config.\n */\n dispatchRef?: string\n judges?: JudgeConfig<TArtifact, TScenario>[]\n /** Required for reproducibility. Default 42. */\n seed?: number\n /** Per-scenario replicates for CI bands. Default 1; raise to 5+ for\n * bootstrap-tight intervals on critical eval. */\n reps?: number\n /** When true (default), completed cells are cached by\n * (manifestHash, scenarioId, rep, generation). Re-runs skip cached cells. */\n resumable?: boolean\n /** Optional store — when present, every artifact + judge score is captured\n * with the configured `captureSource`. Capture is default ON; pass `'off'`\n * to disable. */\n labeledStore?: LabeledScenarioStore | 'off'\n captureSource?: 'production-trace' | 'eval-run' | 'manual' | 'red-team' | 'synthetic'\n captureSourceVersionHash?: string\n /** Wall-clock cost cap across all cells. Cells beyond ceiling are skipped. */\n costCeiling?: number\n /** Max concurrent cells. Default 2. */\n maxConcurrency?: number\n /**\n * Per-cell dispatch deadline in ms. A `dispatch` that neither resolves nor\n * rejects within this window is a hang (a stalled model request, an\n * exhausted runtime resource, a backend that never closes its stream). When\n * set, the cell's `ctx.signal` is aborted and the cell is recorded as a LOUD\n * error (`dispatch exceeded <N>ms`) so the campaign proceeds and the failure\n * is visible — instead of one wedged cell silently hanging the whole run (and\n * every loop/CI job above it) forever. `undefined`/`0` = unbounded (legacy).\n */\n dispatchTimeoutMs?: number\n /** Required: where artifacts + traces land. A bare name (not an absolute path)\n * resolves to the shared `~/.tangle/traces/<repo>/runs/<name>` root so run\n * bundles never pollute a repo working tree. Pass an absolute path to override. */\n runDir: string\n /** Subject repo for the shared run-dir root (defaults to the CWD basename).\n * Only consulted when `runDir` is a bare name. */\n repo?: string\n /** Tracing posture. Default is the substrate's `FileSystemTraceStore` rooted\n * at `<runDir>/traces/`. `'off'` disables capture entirely — substrate\n * refuses this when the caller wires `autoOnPromote !== 'none'`. */\n tracing?: 'on' | 'off'\n /**\n * Per-cell usage expectation — the early, fine-grained sibling of the\n * batch `assertRealBackend` guard. A cell that produced an artifact (no\n * error) but reported `costUsd === 0` AND zero tokens is a stub: the\n * dispatch never reported LLM activity via `ctx.cost`. Modes:\n * - `'warn'` (default) — log the offending cell loudly, keep going.\n * - `'assert'` — throw `BackendIntegrityError` on the first such cell\n * (fail-fast; recommended for CI campaigns expecting real LLM calls).\n * - `'off'` — no check (replay / deterministic-only / offline analysis).\n */\n expectUsage?: 'assert' | 'warn' | 'off'\n /** Test seam — override the wall clock for deterministic tests. */\n now?: () => Date\n /** Test seam — override per-cell trace writer factory. */\n buildTraceWriter?: (cellId: string, dir: string) => CampaignTraceWriter\n /** Storage backend for run/cell dirs, the resumability cache, artifacts,\n * and trace spans. Default: the Node filesystem (`fsCampaignStorage`).\n * Pass `inMemoryCampaignStorage()` to run in a filesystem-less runtime\n * (Cloudflare Workers, Deno, edge) — the `CampaignResult` is still\n * produced; artifacts/traces just aren't persisted to disk. */\n storage?: CampaignStorage\n /**\n * Optional per-cell placement strategy. Returns an opaque string the\n * substrate forwards as `ctx.placement` to the Dispatch — placement-aware\n * Dispatches (e.g. `httpDispatch` from `/adapters/http`) use it to route\n * each cell to the right worker, region, or sandbox. When unset, every\n * cell receives `ctx.placement = undefined` and behaves identically to\n * the in-process case.\n *\n * @example\n * cellPlacement: ({ scenario }) => scenario.tags?.includes('eu') ? 'eu-west' : 'us-east'\n */\n cellPlacement?: (input: {\n scenario: TScenario\n rep: number\n generation?: number\n }) => string | undefined\n}\n\nexport async function runCampaign<TScenario extends Scenario, TArtifact>(\n opts: RunCampaignOptions<TScenario, TArtifact>,\n): Promise<CampaignResult<TArtifact, TScenario>> {\n const seed = opts.seed ?? 42\n const reps = opts.reps ?? 1\n const resumable = opts.resumable ?? true\n const maxConcurrency = opts.maxConcurrency ?? 2\n const now = opts.now ?? (() => new Date())\n const judges = opts.judges ?? []\n const storage = opts.storage ?? fsCampaignStorage()\n\n if (typeof opts.runDir !== 'string' || opts.runDir.trim().length === 0) {\n throw new Error('runCampaign: runDir is required and must be a non-empty string')\n }\n opts.runDir = resolveRunDir(opts.runDir, opts.repo)\n storage.ensureDir(opts.runDir)\n\n const manifestHash = computeManifestHash({\n scenarios: opts.scenarios,\n judges: judges as unknown as JudgeConfig<unknown>[],\n dispatchRef: dispatchRefFor(opts.dispatch, opts.dispatchRef),\n seed,\n reps,\n })\n\n const startedAt = now()\n const cells: CampaignCellResult<TArtifact>[] = []\n const artifactsByPath: Record<string, string> = {}\n\n // Build the cell schedule (scenario × rep).\n const schedule = buildCellSchedule(opts.scenarios, seed, reps)\n\n // Concurrency-limited execution.\n let totalCostUsd = 0\n let costCeilingReached = false\n const abortController = new AbortController()\n // Concurrency lanes that drain the cell schedule. Named \"lanes\" — not\n // \"workers\" — to avoid clashing with the taxonomy's worker (= the agent\n // harness in a sandbox, invoked behind `dispatch`). See loop-taxonomy.md.\n const lanes: Promise<void>[] = []\n let nextIdx = 0\n const cellsRef = cells\n\n for (let i = 0; i < maxConcurrency; i++) {\n lanes.push(\n (async () => {\n while (true) {\n const myIdx = nextIdx++\n if (myIdx >= schedule.length) return\n const slot = schedule[myIdx]!\n if (costCeilingReached) {\n cellsRef.push(skippedCell(slot, 'cost_ceiling_reached'))\n continue\n }\n const result = await executeCell({\n slot,\n opts,\n manifestHash,\n resumable,\n now,\n storage,\n buildTraceWriter: opts.buildTraceWriter ?? defaultBuildTraceWriter(storage),\n signal: abortController.signal,\n dispatchTimeoutMs: opts.dispatchTimeoutMs,\n })\n cellsRef.push(result.cell)\n enforceCellUsage(result.cell, opts.expectUsage ?? 'warn')\n totalCostUsd += result.cell.costUsd\n Object.assign(artifactsByPath, result.artifactsByPath)\n if (opts.costCeiling !== undefined && totalCostUsd >= opts.costCeiling) {\n costCeilingReached = true\n }\n // Capture into LabeledScenarioStore unless explicitly disabled.\n if (opts.labeledStore && opts.labeledStore !== 'off' && !result.cell.error) {\n await captureToStore({\n store: opts.labeledStore,\n cell: result.cell,\n scenario: slot.scenario,\n opts,\n now,\n }).catch((err) => {\n // Capture failures are non-fatal — log but don't crash the campaign.\n // (Trace would normally land here.)\n console.warn(\n `[runCampaign] capture failed for ${result.cell.cellId}: ${err instanceof Error ? err.message : String(err)}`,\n )\n })\n }\n }\n })(),\n )\n }\n await Promise.all(lanes)\n\n const endedAt = now()\n cellsRef.sort((a, b) => a.cellId.localeCompare(b.cellId))\n\n const aggregates = computeAggregates(\n cellsRef,\n judges as unknown as JudgeConfig<TArtifact>[],\n seed,\n )\n\n return {\n manifestHash,\n seed,\n startedAt: startedAt.toISOString(),\n endedAt: endedAt.toISOString(),\n durationMs: endedAt.getTime() - startedAt.getTime(),\n cells: cellsRef,\n aggregates,\n runDir: opts.runDir,\n artifactsByPath,\n scenarios: opts.scenarios.map((s) => ({ id: s.id, kind: s.kind })),\n }\n}\n\n// ── Internals ─────────────────────────────────────────────────────────\n\ninterface ExecuteCellArgs<TScenario extends Scenario, TArtifact> {\n slot: { scenario: TScenario; rep: number; cellId: string; cellSeed: number }\n opts: RunCampaignOptions<TScenario, TArtifact>\n manifestHash: string\n resumable: boolean\n now: () => Date\n storage: CampaignStorage\n buildTraceWriter: (cellId: string, dir: string) => CampaignTraceWriter\n signal: AbortSignal\n dispatchTimeoutMs?: number\n}\n\nasync function executeCell<TScenario extends Scenario, TArtifact>(\n args: ExecuteCellArgs<TScenario, TArtifact>,\n): Promise<{ cell: CampaignCellResult<TArtifact>; artifactsByPath: Record<string, string> }> {\n const storage = args.storage\n const cellDir = join(args.opts.runDir, args.slot.cellId.replace(/[^a-zA-Z0-9_-]/g, '_'))\n storage.ensureDir(cellDir)\n\n // Resumability: cache key = (manifestHash, scenarioId, rep)\n const cachePath = join(cellDir, 'cached-result.json')\n if (args.resumable) {\n const cached = readCachedCell<TArtifact>({\n storage,\n cachePath,\n cellId: args.slot.cellId,\n manifestHash: args.manifestHash,\n })\n if (cached.status === 'hit') {\n return { cell: { ...cached.cell, cached: true }, artifactsByPath: {} }\n }\n }\n\n const startMs = Date.now()\n const trace = args.buildTraceWriter(args.slot.cellId, cellDir)\n const artifactsByPath: Record<string, string> = {}\n const artifacts: CampaignArtifactWriter = {\n async write(path, content) {\n const fullPath = join(cellDir, path)\n storage.ensureDir(join(fullPath, '..'))\n storage.write(fullPath, content)\n artifactsByPath[`${args.slot.cellId}/${path}`] = fullPath\n return fullPath\n },\n async writeJson(path, value) {\n return artifacts.write(path, JSON.stringify(value, null, 2))\n },\n }\n let costSoFar = 0\n const tokensSoFar: CampaignTokenUsage = { input: 0, output: 0 }\n let resolvedModel: string | undefined\n const cost: CampaignCostMeter = {\n observe(amount, source) {\n costSoFar += amount\n trace.span(`cost.${source}`, { amountUsd: amount }).end()\n },\n observeTokens(usage) {\n tokensSoFar.input += usage.input\n tokensSoFar.output += usage.output\n if (usage.cached) tokensSoFar.cached = (tokensSoFar.cached ?? 0) + usage.cached\n },\n observeModel(model) {\n const trimmed = model?.trim()\n if (trimmed) resolvedModel = trimmed\n },\n current() {\n return costSoFar\n },\n tokens() {\n return { ...tokensSoFar }\n },\n resolvedModel() {\n return resolvedModel\n },\n }\n\n const placement = args.opts.cellPlacement?.({\n scenario: args.slot.scenario,\n rep: args.slot.rep,\n })\n\n // Per-cell abort signal, chained to the campaign signal. The dispatch sees\n // THIS signal so a timeout (below) can abort just this cell's in-flight work\n // without tearing down sibling cells — and a signal-honoring dispatch\n // releases its open request instead of leaking it past the deadline.\n const cellAbort = new AbortController()\n const onCampaignAbort = () => cellAbort.abort((args.signal as { reason?: unknown }).reason)\n if (args.signal.aborted) cellAbort.abort((args.signal as { reason?: unknown }).reason)\n else args.signal.addEventListener('abort', onCampaignAbort, { once: true })\n\n const ctx: DispatchContext = {\n cellId: args.slot.cellId,\n rep: args.slot.rep,\n seed: args.slot.cellSeed,\n signal: cellAbort.signal,\n trace,\n artifacts,\n cost,\n placement,\n }\n\n let artifact: TArtifact | undefined\n let errorMessage: string | undefined\n const timeoutMs = args.dispatchTimeoutMs\n let timeoutTimer: ReturnType<typeof setTimeout> | undefined\n try {\n const dispatched = args.opts.dispatch(args.slot.scenario, ctx)\n if (timeoutMs !== undefined && timeoutMs > 0) {\n // A dispatch that never settles (stalled model request, exhausted runtime\n // resource, a stream that never closes) must NOT hang the cell — and with\n // it the lane, the campaign, the loop, the CI job — forever. Race it\n // against the deadline; on timeout, abort the cell and fail it LOUD.\n artifact = await Promise.race([\n dispatched,\n new Promise<never>((_, reject) => {\n timeoutTimer = setTimeout(() => {\n cellAbort.abort(new Error('dispatch timeout'))\n reject(\n new Error(\n `dispatch exceeded ${timeoutMs}ms for cell '${args.slot.cellId}' — aborted and failed loud (no silent hang)`,\n ),\n )\n }, timeoutMs)\n if (typeof (timeoutTimer as { unref?: () => void }).unref === 'function')\n (timeoutTimer as { unref: () => void }).unref()\n }),\n ])\n } else {\n artifact = await dispatched\n }\n } catch (err) {\n errorMessage = err instanceof Error ? err.message : String(err)\n } finally {\n if (timeoutTimer) clearTimeout(timeoutTimer)\n args.signal.removeEventListener('abort', onCampaignAbort)\n }\n\n // Run judges (only if we have an artifact). A judge that throws invalidates\n // the cell — recorded as `error`, NOT folded into a fake composite:0 (a fake\n // zero is indistinguishable from a real zero and poisons every aggregate).\n const judgeScores: Record<string, JudgeScore> = {}\n if (artifact !== undefined) {\n for (const judge of args.opts.judges ?? []) {\n if (judge.appliesTo && !judge.appliesTo(args.slot.scenario)) continue\n try {\n judgeScores[judge.name] = await runJudgeCell(judge, {\n artifact,\n scenario: args.slot.scenario,\n signal: args.signal,\n })\n } catch (err) {\n errorMessage = `judge '${judge.name}' failed: ${err instanceof Error ? err.message : String(err)}`\n break\n }\n }\n }\n\n await trace.flush()\n\n const cell: CampaignCellResult<TArtifact> = {\n manifestHash: args.manifestHash,\n cellId: args.slot.cellId,\n scenarioId: args.slot.scenario.id,\n rep: args.slot.rep,\n artifact: (artifact ?? null) as TArtifact,\n judgeScores,\n costUsd: costSoFar,\n tokenUsage: { ...tokensSoFar },\n ...(resolvedModel ? { resolvedModel } : {}),\n durationMs: Date.now() - startMs,\n seed: args.slot.cellSeed,\n cached: false,\n error: errorMessage,\n }\n\n if (!errorMessage && args.resumable) {\n storage.write(cachePath, JSON.stringify(cell))\n }\n\n return { cell, artifactsByPath }\n}\n\nexport interface CampaignRunPlanCell {\n cellId: string\n scenarioId: string\n rep: number\n seed: number\n cachePath: string\n status: 'cached' | 'run'\n reason?: 'missing' | 'manifest-mismatch' | 'cell-mismatch' | 'corrupt' | 'resumable-off'\n}\n\nexport interface CampaignRunPlan {\n manifestHash: string\n totalCells: number\n cellsCached: number\n cellsToRun: number\n cells: CampaignRunPlanCell[]\n}\n\nexport interface PlanCampaignRunOptions<TScenario extends Scenario, TArtifact> {\n scenarios: TScenario[]\n dispatch?: DispatchFn<TScenario, TArtifact>\n dispatchRef?: string\n judges?: JudgeConfig<TArtifact, TScenario>[]\n seed?: number\n reps?: number\n resumable?: boolean\n runDir: string\n /** Subject repo for the shared run-dir root (see RunCampaignOptions.repo). */\n repo?: string\n storage?: CampaignStorage\n}\n\nexport function planCampaignRun<TScenario extends Scenario, TArtifact>(\n opts: PlanCampaignRunOptions<TScenario, TArtifact>,\n): CampaignRunPlan {\n const seed = opts.seed ?? 42\n const reps = opts.reps ?? 1\n const resumable = opts.resumable ?? true\n const storage = opts.storage ?? fsCampaignStorage()\n\n if (typeof opts.runDir !== 'string' || opts.runDir.trim().length === 0) {\n throw new Error('planCampaignRun: runDir is required and must be a non-empty string')\n }\n opts.runDir = resolveRunDir(opts.runDir, opts.repo)\n\n const manifestHash = computeManifestHash({\n scenarios: opts.scenarios,\n judges: (opts.judges ?? []) as unknown as JudgeConfig<unknown>[],\n dispatchRef: dispatchRefFor(opts.dispatch, opts.dispatchRef),\n seed,\n reps,\n })\n\n const cells = buildCellSchedule(opts.scenarios, seed, reps).map((slot): CampaignRunPlanCell => {\n const cachePath = join(\n opts.runDir,\n slot.cellId.replace(/[^a-zA-Z0-9_-]/g, '_'),\n 'cached-result.json',\n )\n if (!resumable) {\n return {\n cellId: slot.cellId,\n scenarioId: slot.scenario.id,\n rep: slot.rep,\n seed: slot.cellSeed,\n cachePath,\n status: 'run',\n reason: 'resumable-off',\n }\n }\n\n const cached = readCachedCell<unknown>({\n storage,\n cachePath,\n cellId: slot.cellId,\n manifestHash,\n })\n if (cached.status === 'hit') {\n return {\n cellId: slot.cellId,\n scenarioId: slot.scenario.id,\n rep: slot.rep,\n seed: slot.cellSeed,\n cachePath,\n status: 'cached',\n }\n }\n\n return {\n cellId: slot.cellId,\n scenarioId: slot.scenario.id,\n rep: slot.rep,\n seed: slot.cellSeed,\n cachePath,\n status: 'run',\n reason: cached.reason,\n }\n })\n\n const cellsCached = cells.filter((cell) => cell.status === 'cached').length\n return {\n manifestHash,\n totalCells: cells.length,\n cellsCached,\n cellsToRun: cells.length - cellsCached,\n cells,\n }\n}\n\n/**\n * Per-cell stub guard. A cell that produced an artifact (no error) but reported\n * `costUsd === 0` AND zero tokens means the dispatch never called `ctx.cost` —\n * i.e. it ran against a stub or silently dropped its usage. `'warn'` logs it,\n * `'assert'` throws (fail-fast), `'off'` skips. An errored/skipped cell or a\n * deterministic judge-only run that genuinely made no LLM call is not flagged.\n */\nfunction enforceCellUsage<TArtifact>(\n cell: CampaignCellResult<TArtifact>,\n mode: 'assert' | 'warn' | 'off',\n): void {\n if (mode === 'off' || cell.error) return\n if (cell.artifact === null || cell.artifact === undefined) return\n const zeroTokens = cell.tokenUsage.input === 0 && cell.tokenUsage.output === 0\n if (cell.costUsd !== 0 || !zeroTokens) return\n const msg = `cell '${cell.cellId}' produced an artifact but reported zero cost and zero tokens — the dispatch never reported LLM usage via ctx.cost.observe/observeTokens (a stub cell)`\n if (mode === 'assert') {\n const report: BackendIntegrityReport = {\n totalRecords: 1,\n stubRecords: 1,\n realRecords: 0,\n uncostedRecords: 0,\n totalInputTokens: 0,\n totalOutputTokens: 0,\n totalCostUsd: 0,\n verdict: 'stub',\n diagnosis: msg,\n }\n throw new BackendIntegrityError(`expectUsage: ${msg}`, report)\n }\n // eslint-disable-next-line no-console\n console.warn(`[runCampaign] expectUsage: ${msg}`)\n}\n\nasync function runJudgeCell<TArtifact, TScenario extends Scenario>(\n judge: JudgeConfig<TArtifact, TScenario>,\n input: { artifact: TArtifact; scenario: TScenario; signal: AbortSignal },\n): Promise<JudgeScore> {\n return judge.score(input)\n}\n\nfunction defaultBuildTraceWriter(\n storage: CampaignStorage,\n): (cellId: string, dir: string) => CampaignTraceWriter {\n return (cellId, dir) => {\n const spans: Array<Record<string, unknown>> = []\n return {\n span(name, attributes) {\n const startMs = Date.now()\n const record: Record<string, unknown> = { name, cellId, startMs, ...(attributes ?? {}) }\n const finish: TraceSpan = {\n end(endAttrs) {\n record.durationMs = Date.now() - startMs\n if (endAttrs) Object.assign(record, endAttrs)\n spans.push(record)\n },\n setAttribute(key, value) {\n record[key] = value\n },\n }\n return finish\n },\n async flush() {\n storage.write(join(dir, 'spans.jsonl'), spans.map((s) => JSON.stringify(s)).join('\\n'))\n },\n }\n }\n}\n\nfunction skippedCell<TScenario extends Scenario, TArtifact>(\n slot: { scenario: TScenario; rep: number; cellId: string; cellSeed: number },\n reason: string,\n): CampaignCellResult<TArtifact> {\n return {\n cellId: slot.cellId,\n scenarioId: slot.scenario.id,\n rep: slot.rep,\n artifact: null as unknown as TArtifact,\n judgeScores: {},\n costUsd: 0,\n tokenUsage: { input: 0, output: 0 },\n durationMs: 0,\n seed: slot.cellSeed,\n cached: false,\n error: `skipped: ${reason}`,\n }\n}\n\nfunction buildCellSchedule<TScenario extends Scenario>(\n scenarios: TScenario[],\n seed: number,\n reps: number,\n): Array<{ scenario: TScenario; rep: number; cellId: string; cellSeed: number }> {\n const schedule: Array<{ scenario: TScenario; rep: number; cellId: string; cellSeed: number }> = []\n let cellIndex = 0\n for (const scenario of scenarios) {\n for (let rep = 0; rep < reps; rep++) {\n const cellId = `${scenario.id}:${rep}`\n const cellSeed = seed + cellIndex\n schedule.push({ scenario, rep, cellId, cellSeed })\n cellIndex += 1\n }\n }\n return schedule\n}\n\nfunction dispatchRefFor<TScenario extends Scenario, TArtifact>(\n dispatch: DispatchFn<TScenario, TArtifact> | undefined,\n override: string | undefined,\n): string {\n const ref = override ?? dispatch?.name ?? 'anonymous'\n if (typeof ref !== 'string' || ref.trim().length === 0) {\n throw new Error('runCampaign: dispatchRef must be a non-empty string when provided')\n }\n return ref\n}\n\ntype CacheRead<TArtifact> =\n | { status: 'hit'; cell: CampaignCellResult<TArtifact> }\n | { status: 'miss'; reason: 'missing' | 'manifest-mismatch' | 'cell-mismatch' | 'corrupt' }\n\nfunction readCachedCell<TArtifact>(args: {\n storage: CampaignStorage\n cachePath: string\n cellId: string\n manifestHash: string\n}): CacheRead<TArtifact> {\n const raw = args.storage.read(args.cachePath)\n if (raw === undefined) return { status: 'miss', reason: 'missing' }\n\n try {\n const cached = JSON.parse(raw) as CampaignCellResult<TArtifact>\n if (cached.cellId !== args.cellId) return { status: 'miss', reason: 'cell-mismatch' }\n if (cached.manifestHash !== args.manifestHash) {\n return { status: 'miss', reason: 'manifest-mismatch' }\n }\n return { status: 'hit', cell: cached }\n } catch {\n return { status: 'miss', reason: 'corrupt' }\n }\n}\n\ninterface CaptureArgs<TScenario extends Scenario, TArtifact> {\n store: LabeledScenarioStore\n cell: CampaignCellResult<TArtifact>\n scenario: TScenario\n opts: RunCampaignOptions<TScenario, TArtifact>\n now: () => Date\n}\n\nasync function captureToStore<TScenario extends Scenario, TArtifact>(\n args: CaptureArgs<TScenario, TArtifact>,\n): Promise<void> {\n await args.store.observe({\n scenario: args.scenario,\n artifact: args.cell.artifact,\n judgeScores: args.cell.judgeScores,\n source: args.opts.captureSource ?? 'eval-run',\n sourceVersionHash: args.opts.captureSourceVersionHash ?? 'unknown',\n capturedAt: args.now().toISOString(),\n redactionStatus: 'raw',\n })\n}\n\n// ── Aggregates + manifest hash ────────────────────────────────────────\n\nfunction computeManifestHash(input: {\n scenarios: Scenario[]\n judges: JudgeConfig<unknown>[]\n dispatchRef: string\n seed: number\n reps: number\n}): string {\n return contentHash({\n scenarios: input.scenarios,\n judges: input.judges.map((j) => ({ name: j.name, dims: j.dimensions })),\n dispatch: input.dispatchRef,\n seed: input.seed,\n reps: input.reps,\n })\n}\n\nfunction computeAggregates<TArtifact>(\n cells: CampaignCellResult<TArtifact>[],\n judges: JudgeConfig<TArtifact>[],\n seed: number,\n): CampaignAggregates {\n const byJudge: Record<string, JudgeAggregate> = {}\n for (const judge of judges) {\n const scores: number[] = []\n for (const cell of cells) {\n const s = cell.judgeScores[judge.name]\n if (s !== undefined) scores.push(s.composite)\n }\n byJudge[judge.name] = aggregate(scores, seed)\n }\n const byScenario: Record<string, ScenarioAggregate> = {}\n const scenarioGroups = new Map<string, number[]>()\n for (const cell of cells) {\n const composites = Object.values(cell.judgeScores).map((s) => s.composite)\n if (composites.length === 0) continue\n const mean = composites.reduce((a, b) => a + b, 0) / composites.length\n const arr = scenarioGroups.get(cell.scenarioId) ?? []\n arr.push(mean)\n scenarioGroups.set(cell.scenarioId, arr)\n }\n for (const [scenarioId, samples] of scenarioGroups) {\n const ag = aggregate(samples, seed)\n byScenario[scenarioId] = { meanComposite: ag.mean, ci95: ag.ci95, n: ag.n }\n }\n return {\n byJudge,\n byScenario,\n totalCostUsd: cells.reduce((a, c) => a + c.costUsd, 0),\n cellsExecuted: cells.filter((c) => !c.error).length,\n cellsSkipped: cells.filter((c) => c.error?.startsWith('skipped:')).length,\n cellsCached: cells.filter((c) => c.cached).length,\n cellsFailed: cells.filter((c) => c.error && !c.error.startsWith('skipped:')).length,\n }\n}\n\n// Percentile bootstrap CI95 via seeded resampling. Deterministic for a given\n// seed — same campaign re-run produces identical CI bands. Falls back to\n// degenerate intervals at n<=1 (the bootstrap is undefined there).\nfunction aggregate(samples: number[], seed: number): JudgeAggregate {\n const n = samples.length\n if (n === 0) return { mean: 0, stdev: 0, ci95: [0, 0], n: 0 }\n const mean = samples.reduce((a, b) => a + b, 0) / n\n const variance = samples.reduce((a, b) => a + (b - mean) ** 2, 0) / Math.max(1, n - 1)\n const stdev = Math.sqrt(variance)\n const ci = confidenceInterval(samples, 0.95, { seed, resamples: 1000 })\n return { mean, stdev, ci95: [ci.lower, ci.upper], n }\n}\n","/**\n * Backend-integrity guard: distinguish \"agent failed\" from \"eval ran against\n * a stub / unconfigured backend.\" Without this guard a canonical eval can\n * silently report `0/N passed` and look like an agent-quality problem when\n * the LLM was never actually called — the failure mode we just hit running\n * the 4-vertical parallel eval (legal-sandbox-stub returned hard-coded 33-104\n * char strings; gtm/creative defaulted to a cli-bridge that wasn't running).\n *\n * The shape:\n *\n * const report = summarizeBackendIntegrity(records)\n * assertRealBackend(records) // throws BackendIntegrityError if 100% stub\n *\n * A record is \"stub-mode\" if its `tokenUsage.input === 0 && tokenUsage.output === 0`.\n * (`costUsd` alone is unreliable — some backends successfully call LLMs but\n * don't propagate pricing, producing real tokens with $0 cost.)\n *\n * Verdicts:\n * - `real` — at least one record has nonzero token usage\n * - `stub` — every record is stub-mode (eval ran blind)\n * - `mixed` — some records real, some stub (partial backend failure;\n * often the 429-cascade or auth-half-failed case)\n */\n\nimport { AgentEvalError } from '../errors'\nimport type { RunRecord } from '../run-record'\n\nexport interface BackendIntegrityReport {\n /** Total records inspected. */\n totalRecords: number\n /** Records with input=0 AND output=0 (a stub fingerprint). */\n stubRecords: number\n /** Records with nonzero token usage (real LLM activity). */\n realRecords: number\n /** Records where output>0 but costUsd=0 (real LLM, broken cost ledger). */\n uncostedRecords: number\n /** Sum of input tokens across all records. */\n totalInputTokens: number\n /** Sum of output tokens across all records. */\n totalOutputTokens: number\n /** Sum of costUsd across all records. */\n totalCostUsd: number\n /** Worst-case integrity verdict. */\n verdict: 'real' | 'mixed' | 'stub'\n /** Human-readable diagnosis suitable for terminal output. */\n diagnosis: string\n}\n\n/**\n * Error thrown when an integrity assertion fails. Caller can pattern-match\n * by `code === 'AGENT_EVAL_BACKEND_STUB'` to differentiate from other\n * errors.\n */\nexport class BackendIntegrityError extends AgentEvalError {\n constructor(\n message: string,\n public readonly report: BackendIntegrityReport,\n ) {\n super('backend_integrity', message)\n }\n}\n\nfunction isStubRecord(rec: RunRecord): boolean {\n return rec.tokenUsage.input === 0 && rec.tokenUsage.output === 0\n}\n\nfunction isUncostedRecord(rec: RunRecord): boolean {\n return rec.tokenUsage.output > 0 && rec.costUsd === 0\n}\n\n/**\n * Inspect a batch of RunRecords and return an integrity report. Pure\n * function — no I/O, no logging. The caller decides what to do with the\n * verdict (print warning, throw, gate CI, etc.).\n */\nexport function summarizeBackendIntegrity(\n records: ReadonlyArray<RunRecord>,\n): BackendIntegrityReport {\n const totalRecords = records.length\n let stubRecords = 0\n let realRecords = 0\n let uncostedRecords = 0\n let totalInputTokens = 0\n let totalOutputTokens = 0\n let totalCostUsd = 0\n for (const rec of records) {\n totalInputTokens += rec.tokenUsage.input\n totalOutputTokens += rec.tokenUsage.output\n totalCostUsd += rec.costUsd\n if (isStubRecord(rec)) stubRecords++\n else realRecords++\n if (isUncostedRecord(rec)) uncostedRecords++\n }\n const verdict: BackendIntegrityReport['verdict'] =\n totalRecords === 0\n ? 'stub'\n : stubRecords === totalRecords\n ? 'stub'\n : stubRecords === 0\n ? 'real'\n : 'mixed'\n const diagnosis = buildDiagnosis({\n totalRecords,\n stubRecords,\n realRecords,\n uncostedRecords,\n totalInputTokens,\n totalOutputTokens,\n totalCostUsd,\n verdict,\n })\n return {\n totalRecords,\n stubRecords,\n realRecords,\n uncostedRecords,\n totalInputTokens,\n totalOutputTokens,\n totalCostUsd,\n verdict,\n diagnosis,\n }\n}\n\nfunction buildDiagnosis(r: Omit<BackendIntegrityReport, 'diagnosis'>): string {\n if (r.totalRecords === 0) {\n return 'no records — eval produced zero runs; backend likely failed before first turn'\n }\n if (r.verdict === 'stub') {\n return [\n `all ${r.totalRecords} records have zero token usage — the LLM backend was never called.`,\n 'common causes: --backend sandbox without a sandbox bridge running; stub model returning hard-coded strings;',\n 'auth misconfigured so requests were silently dropped before the LLM. Re-run with --backend tcloud and TANGLE_API_KEY set,',\n 'or boot the cli-bridge / sandbox before invoking the eval.',\n ].join(' ')\n }\n if (r.verdict === 'mixed') {\n const pct = ((r.stubRecords / r.totalRecords) * 100).toFixed(0)\n return [\n `${r.stubRecords}/${r.totalRecords} records (${pct}%) have zero token usage — the backend partially failed.`,\n 'common causes: rate-limit cascade (429s after the first N personas);',\n 'transient auth expiry mid-run; provider outage. Treat the affected records as missing data, not agent failures.',\n ].join(' ')\n }\n // verdict === 'real'\n if (r.uncostedRecords > 0) {\n const pct = ((r.uncostedRecords / r.totalRecords) * 100).toFixed(0)\n return [\n `${r.totalRecords} records with real LLM activity (in=${r.totalInputTokens}, out=${r.totalOutputTokens} tokens).`,\n `${r.uncostedRecords} (${pct}%) have output tokens but costUsd=0. Two distinct roots:`,\n '(a) cost ledger mis-wired — no usage propagation from the runtime stream into RunRecord; or',\n '(b) the model is unpriced at the source (sandbox/router returned $0 despite real tokens).',\n 'For (b), price the measured tokens against the substrate table (estimateCost) instead of leaving $0.',\n ].join(' ')\n }\n return `${r.totalRecords} records with real LLM activity (in=${r.totalInputTokens}, out=${r.totalOutputTokens} tokens, $${r.totalCostUsd.toFixed(4)}).`\n}\n\n/**\n * Throw BackendIntegrityError if the verdict is 'stub' — i.e. every record\n * shows zero LLM activity. Non-strict callers can pass `{ allowMixed: false }`\n * to also reject mixed verdicts (recommended for CI gates).\n *\n * Real backends pass through silently.\n */\nexport function assertRealBackend(\n records: ReadonlyArray<RunRecord>,\n opts: { allowMixed?: boolean } = {},\n): BackendIntegrityReport {\n const report = summarizeBackendIntegrity(records)\n const allowMixed = opts.allowMixed ?? true\n if (report.verdict === 'stub') {\n throw new BackendIntegrityError(\n `backend-integrity: ran against a stub or unconfigured backend — ${report.diagnosis}`,\n report,\n )\n }\n if (!allowMixed && report.verdict === 'mixed') {\n throw new BackendIntegrityError(\n `backend-integrity: partial backend failure rejected — ${report.diagnosis}`,\n report,\n )\n }\n return report\n}\n","/**\n * Content-addressed judge-verdict caching.\n *\n * LAW: cache JUDGE VERDICTS only — judging the same artifact with the same\n * judge+rubric is pure. NEVER cache agent rollouts. (A router that cached\n * identical fanout prompts silently destroyed best-of-N diversity; rollout\n * caching reintroduces that failure class. Judging has no diversity to\n * destroy — same artifact + same rubric ⇒ same verdict is the desired\n * property, not a bug.)\n *\n * The cache key is a sha-256 over the canonical JSON of everything that can\n * change a verdict: the artifact content, the scenario id, the judge name,\n * the full dimension list (key + description — the description IS the rubric\n * text shown to the judge), and a caller-supplied `judgeVersion`.\n * `judgeVersion` is REQUIRED: a judge whose prompt/model/ensemble changes\n * without a version bump would otherwise silently serve stale verdicts.\n *\n * Strict canonicalization (`canonicalJson`) throws on undefined / function /\n * symbol / non-finite numbers — an artifact that cannot be unambiguously\n * serialized cannot be content-addressed, and coercing it would let two\n * different artifacts collide on one key.\n */\n\nimport { createHash } from 'node:crypto'\nimport { appendFileSync, existsSync, readFileSync } from 'node:fs'\nimport type { JudgeConfig, JudgeScore, Scenario } from './campaign/types'\n\n// ── canonical JSON + content hash ─────────────────────────────────────────\n\nfunction canonicalizeAt(value: unknown, path: string): string {\n if (value === null) return 'null'\n switch (typeof value) {\n case 'boolean':\n return value ? 'true' : 'false'\n case 'number':\n if (!Number.isFinite(value)) {\n throw new Error(\n `canonicalJson: non-finite number (${value}) at ${path} — ambiguity is an error, not a coercion`,\n )\n }\n return JSON.stringify(value)\n case 'string':\n return JSON.stringify(value)\n case 'undefined':\n case 'function':\n case 'symbol':\n throw new Error(\n `canonicalJson: ${typeof value} at ${path} — ambiguity is an error, not a coercion`,\n )\n case 'bigint':\n throw new Error(`canonicalJson: bigint at ${path} — not representable in JSON`)\n case 'object':\n break\n }\n const obj = value as Record<string, unknown>\n // Honor toJSON (Date → ISO string) before structural checks — without it a\n // Date would canonicalize to '{}' and every timestamp would collide.\n if (typeof obj['toJSON'] === 'function') {\n return canonicalizeAt((obj as { toJSON(): unknown }).toJSON(), path)\n }\n if (Array.isArray(obj)) {\n return `[${obj.map((item, i) => canonicalizeAt(item, `${path}[${i}]`)).join(',')}]`\n }\n if (obj instanceof Map || obj instanceof Set) {\n throw new Error(\n `canonicalJson: ${obj instanceof Map ? 'Map' : 'Set'} at ${path} — would serialize as '{}'; convert to a plain object/array first`,\n )\n }\n const keys = Object.keys(obj).sort()\n const parts = keys.map((k) => `${JSON.stringify(k)}:${canonicalizeAt(obj[k], `${path}.${k}`)}`)\n return `{${parts.join(',')}}`\n}\n\n/**\n * Stable JSON stringify: object keys sorted recursively, so two semantically\n * equal values produce byte-identical output regardless of key insertion\n * order. Throws on undefined / function / symbol / NaN / ±Infinity / bigint /\n * Map / Set — anything JSON.stringify would coerce or drop silently.\n *\n * Distinct from `pre-registration.ts`'s `canonicalize`/`hashJson`, which are\n * permissive (coercion allowed) and async (web-crypto). Use THIS pair when a\n * hash collision or silent coercion would corrupt a cache key or attestation.\n */\nexport function canonicalJson(value: unknown): string {\n return canonicalizeAt(value, '$')\n}\n\n/** Hex sha-256 over `canonicalJson(value)`. The content address used by the\n * verdict cache and report attestation. */\nexport function contentHash(value: unknown): string {\n return createHash('sha256').update(canonicalJson(value)).digest('hex')\n}\n\n// ── store contract ─────────────────────────────────────────────────────────\n\n/** Pluggable verdict store. Sync or async on both legs — `cachedJudge`\n * awaits the results either way. */\nexport interface VerdictCacheStore {\n get(key: string): Promise<JudgeScore | undefined> | JudgeScore | undefined\n set(key: string, score: JudgeScore): Promise<void> | void\n}\n\n/** Process-local Map-backed store. */\nexport function inMemoryVerdictCache(): VerdictCacheStore {\n const entries = new Map<string, JudgeScore>()\n return {\n get: (key) => entries.get(key),\n set: (key, score) => {\n entries.set(key, score)\n },\n }\n}\n\ninterface VerdictCacheLine {\n key: string\n score: JudgeScore\n}\n\nfunction parseCacheLine(line: string, path: string, lineNo: number): VerdictCacheLine {\n let parsed: unknown\n try {\n parsed = JSON.parse(line)\n } catch (err) {\n throw new Error(\n `fileVerdictCache: corrupt JSONL at ${path}:${lineNo} — ${err instanceof Error ? err.message : String(err)}`,\n )\n }\n const rec = parsed as Partial<VerdictCacheLine>\n if (\n typeof rec !== 'object' ||\n rec === null ||\n typeof rec.key !== 'string' ||\n typeof rec.score !== 'object' ||\n rec.score === null ||\n typeof rec.score.composite !== 'number' ||\n typeof rec.score.dimensions !== 'object'\n ) {\n throw new Error(\n `fileVerdictCache: invalid record shape at ${path}:${lineNo} — expected {key, score:{dimensions, composite, notes}}`,\n )\n }\n return rec as VerdictCacheLine\n}\n\n/**\n * JSONL-file-backed store: the full file is loaded into an in-memory index at\n * construction; every `set` appends one line synchronously (durable before\n * the verdict is returned). A corrupt or malformed line throws at load with\n * file:line — a skipped line would silently re-judge (cost) or, worse, mask\n * a half-written file that needs operator attention.\n */\nexport function fileVerdictCache(path: string): VerdictCacheStore {\n const entries = new Map<string, JudgeScore>()\n if (existsSync(path)) {\n const lines = readFileSync(path, 'utf8').split('\\n')\n for (let i = 0; i < lines.length; i++) {\n const line = lines[i]\n if (line === undefined || line.trim() === '') continue\n const rec = parseCacheLine(line, path, i + 1)\n entries.set(rec.key, rec.score)\n }\n }\n return {\n get: (key) => entries.get(key),\n set: (key, score) => {\n appendFileSync(path, `${JSON.stringify({ key, score })}\\n`, 'utf8')\n entries.set(key, score)\n },\n }\n}\n\n// ── cached judge wrapper ───────────────────────────────────────────────────\n\nexport interface VerdictCacheStats {\n hits: number\n misses: number\n}\n\nexport interface CachedJudgeOptions {\n /** REQUIRED — part of the cache key. Bump on any change to the judge's\n * prompt, model, ensemble, or scoring logic; silent judge upgrades must\n * never serve stale verdicts. */\n judgeVersion: string\n}\n\n/** The wrapped judge: same `JudgeConfig` seam, plus hit/miss observability. */\nexport type CachedJudge<TArtifact, TScenario extends Scenario = Scenario> = JudgeConfig<\n TArtifact,\n TScenario\n> & {\n stats(): VerdictCacheStats\n}\n\n/**\n * Wrap a `JudgeConfig` so repeat judgments of the same artifact are served\n * from the store instead of re-invoking `score()`. The wrapper is generic\n * over the judge's own type parameters and preserves `appliesTo` — it is a\n * drop-in replacement anywhere a `JudgeConfig` is accepted.\n *\n * A judge that throws is NOT cached: the error propagates and the next\n * attempt re-judges (caching a failure would pin a transient outage forever).\n */\nexport function cachedJudge<TArtifact, TScenario extends Scenario = Scenario>(\n judge: JudgeConfig<TArtifact, TScenario>,\n store: VerdictCacheStore,\n options: CachedJudgeOptions,\n): CachedJudge<TArtifact, TScenario> {\n if (typeof options.judgeVersion !== 'string' || options.judgeVersion.trim() === '') {\n throw new Error('cachedJudge: judgeVersion is required and must be a non-empty string')\n }\n const stats: VerdictCacheStats = { hits: 0, misses: 0 }\n const wrapped: CachedJudge<TArtifact, TScenario> = {\n name: judge.name,\n dimensions: judge.dimensions,\n async score(input) {\n const key = contentHash({\n artifact: canonicalJson(input.artifact),\n scenarioId: input.scenario.id,\n judgeName: judge.name,\n dimensions: judge.dimensions,\n judgeVersion: options.judgeVersion,\n })\n const cached = await store.get(key)\n if (cached !== undefined) {\n stats.hits += 1\n return cached\n }\n const score = await judge.score(input)\n await store.set(key, score)\n stats.misses += 1\n return score\n },\n stats: () => ({ ...stats }),\n }\n if (judge.appliesTo) wrapped.appliesTo = judge.appliesTo\n return wrapped\n}\n","import { homedir } from 'node:os'\nimport { basename, isAbsolute, join } from 'node:path'\n\n/** The shared, out-of-repo root for campaign/benchmark run bundles. Keeping run\n * outputs here means they never land in a repo working tree (no per-repo\n * gitignore, no clutter, no accidental commits). Layout:\n * ~/.tangle/traces/<repo>/runs/<runName>/\n * where <repo> disambiguates runs across repos in one place. */\nexport function tangleTracesRoot(): string {\n return join(homedir(), '.tangle', 'traces')\n}\n\n/** Resolve a campaign `runDir`. An absolute path is honored as-is (the caller\n * chose an explicit location). A bare name is placed under the shared home root\n * so bundles never pollute a repo working tree — the default the harness should\n * compute so callers pass a *name*, not a path. */\nexport function resolveRunDir(runDir: string, repo?: string): string {\n if (isAbsolute(runDir)) return runDir\n const r = repo && repo.trim().length > 0 ? repo : basename(process.cwd())\n return join(tangleTracesRoot(), r, 'runs', runDir)\n}\n","import { createRequire } from 'node:module'\n\n/**\n * `CampaignStorage` — the filesystem seam `runCampaign` writes through\n * (run/cell dirs, the resumability cache, per-cell artifacts, trace spans).\n *\n * The default (`fsCampaignStorage`) is the Node filesystem — identical\n * behavior to the inline `node:fs` calls it replaces, so existing CLI\n * consumers are unaffected. `inMemoryCampaignStorage` keeps everything in a\n * `Map`, so the substrate runs in environments WITHOUT a filesystem\n * (Cloudflare Workers, Deno Deploy, other edge runtimes) — the campaign\n * still produces its `CampaignResult` (cells + aggregates) in memory;\n * artifacts/traces simply aren't persisted to disk.\n *\n * Paths are opaque keys to the in-memory adapter — it does not parse them,\n * so the same `join(...)`-built paths work unchanged across both adapters.\n */\nexport interface CampaignStorage {\n /** Ensure a directory exists (recursive). No-op for in-memory. */\n ensureDir(dir: string): void\n /** Does this path exist (as a written file or an ensured dir)? */\n exists(path: string): boolean\n /** Read a UTF-8 file; `undefined` when missing or unreadable. */\n read(path: string): string | undefined\n /** Write a file (string or bytes). Parent dir is assumed ensured. */\n write(path: string, content: string | Uint8Array): void\n}\n\n/** Node-filesystem storage — the default. Lazily requires `node:fs` so the\n * module imports cleanly in non-Node runtimes (where the caller passes\n * `inMemoryCampaignStorage` instead and never constructs this).\n *\n * `createRequire(import.meta.url)` is the ESM-native lazy require — a bare\n * `require` is a ReferenceError under `\"type\": \"module\"`, which is exactly\n * the shape this package publishes. */\nexport function fsCampaignStorage(): CampaignStorage {\n const nodeRequire = createRequire(import.meta.url)\n const { existsSync, mkdirSync, readFileSync, writeFileSync } = nodeRequire(\n 'node:fs',\n ) as typeof import('node:fs')\n return {\n ensureDir(dir) {\n if (!existsSync(dir)) mkdirSync(dir, { recursive: true })\n },\n exists(path) {\n return existsSync(path)\n },\n read(path) {\n try {\n return readFileSync(path, 'utf8')\n } catch {\n return undefined\n }\n },\n write(path, content) {\n writeFileSync(path, content as Uint8Array)\n },\n }\n}\n\n/** In-memory storage for filesystem-less runtimes. Artifacts + trace spans\n * live in a `Map` for the duration of the run; the `CampaignResult` is\n * fully populated, but nothing is persisted to disk. */\nexport function inMemoryCampaignStorage(): CampaignStorage {\n const files = new Map<string, string | Uint8Array>()\n const dirs = new Set<string>()\n return {\n ensureDir(dir) {\n dirs.add(dir)\n },\n exists(path) {\n return files.has(path) || dirs.has(path)\n },\n read(path) {\n const value = files.get(path)\n if (value === undefined) return undefined\n return typeof value === 'string' ? value : new TextDecoder().decode(value)\n },\n write(path, content) {\n files.set(path, content)\n },\n }\n}\n"],"mappings":";;;;;;;;AAWA,SAAS,QAAAA,aAAY;;;AC0Cd,IAAM,wBAAN,cAAoC,eAAe;AAAA,EACxD,YACE,SACgB,QAChB;AACA,UAAM,qBAAqB,OAAO;AAFlB;AAAA,EAGlB;AAAA,EAHkB;AAIpB;AAEA,SAAS,aAAa,KAAyB;AAC7C,SAAO,IAAI,WAAW,UAAU,KAAK,IAAI,WAAW,WAAW;AACjE;AAEA,SAAS,iBAAiB,KAAyB;AACjD,SAAO,IAAI,WAAW,SAAS,KAAK,IAAI,YAAY;AACtD;AAOO,SAAS,0BACd,SACwB;AACxB,QAAM,eAAe,QAAQ;AAC7B,MAAI,cAAc;AAClB,MAAI,cAAc;AAClB,MAAI,kBAAkB;AACtB,MAAI,mBAAmB;AACvB,MAAI,oBAAoB;AACxB,MAAI,eAAe;AACnB,aAAW,OAAO,SAAS;AACzB,wBAAoB,IAAI,WAAW;AACnC,yBAAqB,IAAI,WAAW;AACpC,oBAAgB,IAAI;AACpB,QAAI,aAAa,GAAG,EAAG;AAAA,QAClB;AACL,QAAI,iBAAiB,GAAG,EAAG;AAAA,EAC7B;AACA,QAAM,UACJ,iBAAiB,IACb,SACA,gBAAgB,eACd,SACA,gBAAgB,IACd,SACA;AACV,QAAM,YAAY,eAAe;AAAA,IAC/B;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,CAAC;AACD,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAEA,SAAS,eAAe,GAAsD;AAC5E,MAAI,EAAE,iBAAiB,GAAG;AACxB,WAAO;AAAA,EACT;AACA,MAAI,EAAE,YAAY,QAAQ;AACxB,WAAO;AAAA,MACL,OAAO,EAAE,YAAY;AAAA,MACrB;AAAA,MACA;AAAA,MACA;AAAA,IACF,EAAE,KAAK,GAAG;AAAA,EACZ;AACA,MAAI,EAAE,YAAY,SAAS;AACzB,UAAM,OAAQ,EAAE,cAAc,EAAE,eAAgB,KAAK,QAAQ,CAAC;AAC9D,WAAO;AAAA,MACL,GAAG,EAAE,WAAW,IAAI,EAAE,YAAY,aAAa,GAAG;AAAA,MAClD;AAAA,MACA;AAAA,IACF,EAAE,KAAK,GAAG;AAAA,EACZ;AAEA,MAAI,EAAE,kBAAkB,GAAG;AACzB,UAAM,OAAQ,EAAE,kBAAkB,EAAE,eAAgB,KAAK,QAAQ,CAAC;AAClE,WAAO;AAAA,MACL,GAAG,EAAE,YAAY,uCAAuC,EAAE,gBAAgB,SAAS,EAAE,iBAAiB;AAAA,MACtG,GAAG,EAAE,eAAe,KAAK,GAAG;AAAA,MAC5B;AAAA,MACA;AAAA,MACA;AAAA,IACF,EAAE,KAAK,GAAG;AAAA,EACZ;AACA,SAAO,GAAG,EAAE,YAAY,uCAAuC,EAAE,gBAAgB,SAAS,EAAE,iBAAiB,aAAa,EAAE,aAAa,QAAQ,CAAC,CAAC;AACrJ;AASO,SAAS,kBACd,SACA,OAAiC,CAAC,GACV;AACxB,QAAM,SAAS,0BAA0B,OAAO;AAChD,QAAM,aAAa,KAAK,cAAc;AACtC,MAAI,OAAO,YAAY,QAAQ;AAC7B,UAAM,IAAI;AAAA,MACR,wEAAmE,OAAO,SAAS;AAAA,MACnF;AAAA,IACF;AAAA,EACF;AACA,MAAI,CAAC,cAAc,OAAO,YAAY,SAAS;AAC7C,UAAM,IAAI;AAAA,MACR,8DAAyD,OAAO,SAAS;AAAA,MACzE;AAAA,IACF;AAAA,EACF;AACA,SAAO;AACT;;;ACjKA,SAAS,kBAAkB;AAC3B,SAAS,gBAAgB,YAAY,oBAAoB;AAKzD,SAAS,eAAe,OAAgB,MAAsB;AAC5D,MAAI,UAAU,KAAM,QAAO;AAC3B,UAAQ,OAAO,OAAO;AAAA,IACpB,KAAK;AACH,aAAO,QAAQ,SAAS;AAAA,IAC1B,KAAK;AACH,UAAI,CAAC,OAAO,SAAS,KAAK,GAAG;AAC3B,cAAM,IAAI;AAAA,UACR,qCAAqC,KAAK,QAAQ,IAAI;AAAA,QACxD;AAAA,MACF;AACA,aAAO,KAAK,UAAU,KAAK;AAAA,IAC7B,KAAK;AACH,aAAO,KAAK,UAAU,KAAK;AAAA,IAC7B,KAAK;AAAA,IACL,KAAK;AAAA,IACL,KAAK;AACH,YAAM,IAAI;AAAA,QACR,kBAAkB,OAAO,KAAK,OAAO,IAAI;AAAA,MAC3C;AAAA,IACF,KAAK;AACH,YAAM,IAAI,MAAM,4BAA4B,IAAI,mCAA8B;AAAA,IAChF,KAAK;AACH;AAAA,EACJ;AACA,QAAM,MAAM;AAGZ,MAAI,OAAO,IAAI,QAAQ,MAAM,YAAY;AACvC,WAAO,eAAgB,IAA8B,OAAO,GAAG,IAAI;AAAA,EACrE;AACA,MAAI,MAAM,QAAQ,GAAG,GAAG;AACtB,WAAO,IAAI,IAAI,IAAI,CAAC,MAAM,MAAM,eAAe,MAAM,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,KAAK,GAAG,CAAC;AAAA,EAClF;AACA,MAAI,eAAe,OAAO,eAAe,KAAK;AAC5C,UAAM,IAAI;AAAA,MACR,kBAAkB,eAAe,MAAM,QAAQ,KAAK,OAAO,IAAI;AAAA,IACjE;AAAA,EACF;AACA,QAAM,OAAO,OAAO,KAAK,GAAG,EAAE,KAAK;AACnC,QAAM,QAAQ,KAAK,IAAI,CAAC,MAAM,GAAG,KAAK,UAAU,CAAC,CAAC,IAAI,eAAe,IAAI,CAAC,GAAG,GAAG,IAAI,IAAI,CAAC,EAAE,CAAC,EAAE;AAC9F,SAAO,IAAI,MAAM,KAAK,GAAG,CAAC;AAC5B;AAYO,SAAS,cAAc,OAAwB;AACpD,SAAO,eAAe,OAAO,GAAG;AAClC;AAIO,SAAS,YAAY,OAAwB;AAClD,SAAO,WAAW,QAAQ,EAAE,OAAO,cAAc,KAAK,CAAC,EAAE,OAAO,KAAK;AACvE;AAYO,SAAS,uBAA0C;AACxD,QAAM,UAAU,oBAAI,IAAwB;AAC5C,SAAO;AAAA,IACL,KAAK,CAAC,QAAQ,QAAQ,IAAI,GAAG;AAAA,IAC7B,KAAK,CAAC,KAAK,UAAU;AACnB,cAAQ,IAAI,KAAK,KAAK;AAAA,IACxB;AAAA,EACF;AACF;AAOA,SAAS,eAAe,MAAc,MAAc,QAAkC;AACpF,MAAI;AACJ,MAAI;AACF,aAAS,KAAK,MAAM,IAAI;AAAA,EAC1B,SAAS,KAAK;AACZ,UAAM,IAAI;AAAA,MACR,sCAAsC,IAAI,IAAI,MAAM,WAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,IAC5G;AAAA,EACF;AACA,QAAM,MAAM;AACZ,MACE,OAAO,QAAQ,YACf,QAAQ,QACR,OAAO,IAAI,QAAQ,YACnB,OAAO,IAAI,UAAU,YACrB,IAAI,UAAU,QACd,OAAO,IAAI,MAAM,cAAc,YAC/B,OAAO,IAAI,MAAM,eAAe,UAChC;AACA,UAAM,IAAI;AAAA,MACR,6CAA6C,IAAI,IAAI,MAAM;AAAA,IAC7D;AAAA,EACF;AACA,SAAO;AACT;AASO,SAAS,iBAAiB,MAAiC;AAChE,QAAM,UAAU,oBAAI,IAAwB;AAC5C,MAAI,WAAW,IAAI,GAAG;AACpB,UAAM,QAAQ,aAAa,MAAM,MAAM,EAAE,MAAM,IAAI;AACnD,aAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;AACrC,YAAM,OAAO,MAAM,CAAC;AACpB,UAAI,SAAS,UAAa,KAAK,KAAK,MAAM,GAAI;AAC9C,YAAM,MAAM,eAAe,MAAM,MAAM,IAAI,CAAC;AAC5C,cAAQ,IAAI,IAAI,KAAK,IAAI,KAAK;AAAA,IAChC;AAAA,EACF;AACA,SAAO;AAAA,IACL,KAAK,CAAC,QAAQ,QAAQ,IAAI,GAAG;AAAA,IAC7B,KAAK,CAAC,KAAK,UAAU;AACnB,qBAAe,MAAM,GAAG,KAAK,UAAU,EAAE,KAAK,MAAM,CAAC,CAAC;AAAA,GAAM,MAAM;AAClE,cAAQ,IAAI,KAAK,KAAK;AAAA,IACxB;AAAA,EACF;AACF;AAiCO,SAAS,YACd,OACA,OACA,SACmC;AACnC,MAAI,OAAO,QAAQ,iBAAiB,YAAY,QAAQ,aAAa,KAAK,MAAM,IAAI;AAClF,UAAM,IAAI,MAAM,sEAAsE;AAAA,EACxF;AACA,QAAM,QAA2B,EAAE,MAAM,GAAG,QAAQ,EAAE;AACtD,QAAM,UAA6C;AAAA,IACjD,MAAM,MAAM;AAAA,IACZ,YAAY,MAAM;AAAA,IAClB,MAAM,MAAM,OAAO;AACjB,YAAM,MAAM,YAAY;AAAA,QACtB,UAAU,cAAc,MAAM,QAAQ;AAAA,QACtC,YAAY,MAAM,SAAS;AAAA,QAC3B,WAAW,MAAM;AAAA,QACjB,YAAY,MAAM;AAAA,QAClB,cAAc,QAAQ;AAAA,MACxB,CAAC;AACD,YAAM,SAAS,MAAM,MAAM,IAAI,GAAG;AAClC,UAAI,WAAW,QAAW;AACxB,cAAM,QAAQ;AACd,eAAO;AAAA,MACT;AACA,YAAM,QAAQ,MAAM,MAAM,MAAM,KAAK;AACrC,YAAM,MAAM,IAAI,KAAK,KAAK;AAC1B,YAAM,UAAU;AAChB,aAAO;AAAA,IACT;AAAA,IACA,OAAO,OAAO,EAAE,GAAG,MAAM;AAAA,EAC3B;AACA,MAAI,MAAM,UAAW,SAAQ,YAAY,MAAM;AAC/C,SAAO;AACT;;;AC5OA,SAAS,eAAe;AACxB,SAAS,UAAU,YAAY,YAAY;AAOpC,SAAS,mBAA2B;AACzC,SAAO,KAAK,QAAQ,GAAG,WAAW,QAAQ;AAC5C;AAMO,SAAS,cAAc,QAAgB,MAAuB;AACnE,MAAI,WAAW,MAAM,EAAG,QAAO;AAC/B,QAAM,IAAI,QAAQ,KAAK,KAAK,EAAE,SAAS,IAAI,OAAO,SAAS,QAAQ,IAAI,CAAC;AACxE,SAAO,KAAK,iBAAiB,GAAG,GAAG,QAAQ,MAAM;AACnD;;;ACpBA,SAAS,qBAAqB;AAmCvB,SAAS,oBAAqC;AACnD,QAAM,cAAc,cAAc,YAAY,GAAG;AACjD,QAAM,EAAE,YAAAC,aAAY,WAAW,cAAAC,eAAc,cAAc,IAAI;AAAA,IAC7D;AAAA,EACF;AACA,SAAO;AAAA,IACL,UAAU,KAAK;AACb,UAAI,CAACD,YAAW,GAAG,EAAG,WAAU,KAAK,EAAE,WAAW,KAAK,CAAC;AAAA,IAC1D;AAAA,IACA,OAAO,MAAM;AACX,aAAOA,YAAW,IAAI;AAAA,IACxB;AAAA,IACA,KAAK,MAAM;AACT,UAAI;AACF,eAAOC,cAAa,MAAM,MAAM;AAAA,MAClC,QAAQ;AACN,eAAO;AAAA,MACT;AAAA,IACF;AAAA,IACA,MAAM,MAAM,SAAS;AACnB,oBAAc,MAAM,OAAqB;AAAA,IAC3C;AAAA,EACF;AACF;AAKO,SAAS,0BAA2C;AACzD,QAAM,QAAQ,oBAAI,IAAiC;AACnD,QAAM,OAAO,oBAAI,IAAY;AAC7B,SAAO;AAAA,IACL,UAAU,KAAK;AACb,WAAK,IAAI,GAAG;AAAA,IACd;AAAA,IACA,OAAO,MAAM;AACX,aAAO,MAAM,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI;AAAA,IACzC;AAAA,IACA,KAAK,MAAM;AACT,YAAM,QAAQ,MAAM,IAAI,IAAI;AAC5B,UAAI,UAAU,OAAW,QAAO;AAChC,aAAO,OAAO,UAAU,WAAW,QAAQ,IAAI,YAAY,EAAE,OAAO,KAAK;AAAA,IAC3E;AAAA,IACA,MAAM,MAAM,SAAS;AACnB,YAAM,IAAI,MAAM,OAAO;AAAA,IACzB;AAAA,EACF;AACF;;;AJ0CA,eAAsB,YACpB,MAC+C;AAC/C,QAAM,OAAO,KAAK,QAAQ;AAC1B,QAAM,OAAO,KAAK,QAAQ;AAC1B,QAAM,YAAY,KAAK,aAAa;AACpC,QAAM,iBAAiB,KAAK,kBAAkB;AAC9C,QAAM,MAAM,KAAK,QAAQ,MAAM,oBAAI,KAAK;AACxC,QAAM,SAAS,KAAK,UAAU,CAAC;AAC/B,QAAM,UAAU,KAAK,WAAW,kBAAkB;AAElD,MAAI,OAAO,KAAK,WAAW,YAAY,KAAK,OAAO,KAAK,EAAE,WAAW,GAAG;AACtE,UAAM,IAAI,MAAM,gEAAgE;AAAA,EAClF;AACA,OAAK,SAAS,cAAc,KAAK,QAAQ,KAAK,IAAI;AAClD,UAAQ,UAAU,KAAK,MAAM;AAE7B,QAAM,eAAe,oBAAoB;AAAA,IACvC,WAAW,KAAK;AAAA,IAChB;AAAA,IACA,aAAa,eAAe,KAAK,UAAU,KAAK,WAAW;AAAA,IAC3D;AAAA,IACA;AAAA,EACF,CAAC;AAED,QAAM,YAAY,IAAI;AACtB,QAAM,QAAyC,CAAC;AAChD,QAAM,kBAA0C,CAAC;AAGjD,QAAM,WAAW,kBAAkB,KAAK,WAAW,MAAM,IAAI;AAG7D,MAAI,eAAe;AACnB,MAAI,qBAAqB;AACzB,QAAM,kBAAkB,IAAI,gBAAgB;AAI5C,QAAM,QAAyB,CAAC;AAChC,MAAI,UAAU;AACd,QAAM,WAAW;AAEjB,WAAS,IAAI,GAAG,IAAI,gBAAgB,KAAK;AACvC,UAAM;AAAA,OACH,YAAY;AACX,eAAO,MAAM;AACX,gBAAM,QAAQ;AACd,cAAI,SAAS,SAAS,OAAQ;AAC9B,gBAAM,OAAO,SAAS,KAAK;AAC3B,cAAI,oBAAoB;AACtB,qBAAS,KAAK,YAAY,MAAM,sBAAsB,CAAC;AACvD;AAAA,UACF;AACA,gBAAM,SAAS,MAAM,YAAY;AAAA,YAC/B;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA,kBAAkB,KAAK,oBAAoB,wBAAwB,OAAO;AAAA,YAC1E,QAAQ,gBAAgB;AAAA,YACxB,mBAAmB,KAAK;AAAA,UAC1B,CAAC;AACD,mBAAS,KAAK,OAAO,IAAI;AACzB,2BAAiB,OAAO,MAAM,KAAK,eAAe,MAAM;AACxD,0BAAgB,OAAO,KAAK;AAC5B,iBAAO,OAAO,iBAAiB,OAAO,eAAe;AACrD,cAAI,KAAK,gBAAgB,UAAa,gBAAgB,KAAK,aAAa;AACtE,iCAAqB;AAAA,UACvB;AAEA,cAAI,KAAK,gBAAgB,KAAK,iBAAiB,SAAS,CAAC,OAAO,KAAK,OAAO;AAC1E,kBAAM,eAAe;AAAA,cACnB,OAAO,KAAK;AAAA,cACZ,MAAM,OAAO;AAAA,cACb,UAAU,KAAK;AAAA,cACf;AAAA,cACA;AAAA,YACF,CAAC,EAAE,MAAM,CAAC,QAAQ;AAGhB,sBAAQ;AAAA,gBACN,oCAAoC,OAAO,KAAK,MAAM,KAAK,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,cAC7G;AAAA,YACF,CAAC;AAAA,UACH;AAAA,QACF;AAAA,MACF,GAAG;AAAA,IACL;AAAA,EACF;AACA,QAAM,QAAQ,IAAI,KAAK;AAEvB,QAAM,UAAU,IAAI;AACpB,WAAS,KAAK,CAAC,GAAG,MAAM,EAAE,OAAO,cAAc,EAAE,MAAM,CAAC;AAExD,QAAM,aAAa;AAAA,IACjB;AAAA,IACA;AAAA,IACA;AAAA,EACF;AAEA,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,WAAW,UAAU,YAAY;AAAA,IACjC,SAAS,QAAQ,YAAY;AAAA,IAC7B,YAAY,QAAQ,QAAQ,IAAI,UAAU,QAAQ;AAAA,IAClD,OAAO;AAAA,IACP;AAAA,IACA,QAAQ,KAAK;AAAA,IACb;AAAA,IACA,WAAW,KAAK,UAAU,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,MAAM,EAAE,KAAK,EAAE;AAAA,EACnE;AACF;AAgBA,eAAe,YACb,MAC2F;AAC3F,QAAM,UAAU,KAAK;AACrB,QAAM,UAAUC,MAAK,KAAK,KAAK,QAAQ,KAAK,KAAK,OAAO,QAAQ,mBAAmB,GAAG,CAAC;AACvF,UAAQ,UAAU,OAAO;AAGzB,QAAM,YAAYA,MAAK,SAAS,oBAAoB;AACpD,MAAI,KAAK,WAAW;AAClB,UAAM,SAAS,eAA0B;AAAA,MACvC;AAAA,MACA;AAAA,MACA,QAAQ,KAAK,KAAK;AAAA,MAClB,cAAc,KAAK;AAAA,IACrB,CAAC;AACD,QAAI,OAAO,WAAW,OAAO;AAC3B,aAAO,EAAE,MAAM,EAAE,GAAG,OAAO,MAAM,QAAQ,KAAK,GAAG,iBAAiB,CAAC,EAAE;AAAA,IACvE;AAAA,EACF;AAEA,QAAM,UAAU,KAAK,IAAI;AACzB,QAAM,QAAQ,KAAK,iBAAiB,KAAK,KAAK,QAAQ,OAAO;AAC7D,QAAM,kBAA0C,CAAC;AACjD,QAAM,YAAoC;AAAA,IACxC,MAAM,MAAM,MAAM,SAAS;AACzB,YAAM,WAAWA,MAAK,SAAS,IAAI;AACnC,cAAQ,UAAUA,MAAK,UAAU,IAAI,CAAC;AACtC,cAAQ,MAAM,UAAU,OAAO;AAC/B,sBAAgB,GAAG,KAAK,KAAK,MAAM,IAAI,IAAI,EAAE,IAAI;AACjD,aAAO;AAAA,IACT;AAAA,IACA,MAAM,UAAU,MAAM,OAAO;AAC3B,aAAO,UAAU,MAAM,MAAM,KAAK,UAAU,OAAO,MAAM,CAAC,CAAC;AAAA,IAC7D;AAAA,EACF;AACA,MAAI,YAAY;AAChB,QAAM,cAAkC,EAAE,OAAO,GAAG,QAAQ,EAAE;AAC9D,MAAI;AACJ,QAAM,OAA0B;AAAA,IAC9B,QAAQ,QAAQ,QAAQ;AACtB,mBAAa;AACb,YAAM,KAAK,QAAQ,MAAM,IAAI,EAAE,WAAW,OAAO,CAAC,EAAE,IAAI;AAAA,IAC1D;AAAA,IACA,cAAc,OAAO;AACnB,kBAAY,SAAS,MAAM;AAC3B,kBAAY,UAAU,MAAM;AAC5B,UAAI,MAAM,OAAQ,aAAY,UAAU,YAAY,UAAU,KAAK,MAAM;AAAA,IAC3E;AAAA,IACA,aAAa,OAAO;AAClB,YAAM,UAAU,OAAO,KAAK;AAC5B,UAAI,QAAS,iBAAgB;AAAA,IAC/B;AAAA,IACA,UAAU;AACR,aAAO;AAAA,IACT;AAAA,IACA,SAAS;AACP,aAAO,EAAE,GAAG,YAAY;AAAA,IAC1B;AAAA,IACA,gBAAgB;AACd,aAAO;AAAA,IACT;AAAA,EACF;AAEA,QAAM,YAAY,KAAK,KAAK,gBAAgB;AAAA,IAC1C,UAAU,KAAK,KAAK;AAAA,IACpB,KAAK,KAAK,KAAK;AAAA,EACjB,CAAC;AAMD,QAAM,YAAY,IAAI,gBAAgB;AACtC,QAAM,kBAAkB,MAAM,UAAU,MAAO,KAAK,OAAgC,MAAM;AAC1F,MAAI,KAAK,OAAO,QAAS,WAAU,MAAO,KAAK,OAAgC,MAAM;AAAA,MAChF,MAAK,OAAO,iBAAiB,SAAS,iBAAiB,EAAE,MAAM,KAAK,CAAC;AAE1E,QAAM,MAAuB;AAAA,IAC3B,QAAQ,KAAK,KAAK;AAAA,IAClB,KAAK,KAAK,KAAK;AAAA,IACf,MAAM,KAAK,KAAK;AAAA,IAChB,QAAQ,UAAU;AAAA,IAClB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AAEA,MAAI;AACJ,MAAI;AACJ,QAAM,YAAY,KAAK;AACvB,MAAI;AACJ,MAAI;AACF,UAAM,aAAa,KAAK,KAAK,SAAS,KAAK,KAAK,UAAU,GAAG;AAC7D,QAAI,cAAc,UAAa,YAAY,GAAG;AAK5C,iBAAW,MAAM,QAAQ,KAAK;AAAA,QAC5B;AAAA,QACA,IAAI,QAAe,CAAC,GAAG,WAAW;AAChC,yBAAe,WAAW,MAAM;AAC9B,sBAAU,MAAM,IAAI,MAAM,kBAAkB,CAAC;AAC7C;AAAA,cACE,IAAI;AAAA,gBACF,qBAAqB,SAAS,gBAAgB,KAAK,KAAK,MAAM;AAAA,cAChE;AAAA,YACF;AAAA,UACF,GAAG,SAAS;AACZ,cAAI,OAAQ,aAAwC,UAAU;AAC5D,YAAC,aAAuC,MAAM;AAAA,QAClD,CAAC;AAAA,MACH,CAAC;AAAA,IACH,OAAO;AACL,iBAAW,MAAM;AAAA,IACnB;AAAA,EACF,SAAS,KAAK;AACZ,mBAAe,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAAA,EAChE,UAAE;AACA,QAAI,aAAc,cAAa,YAAY;AAC3C,SAAK,OAAO,oBAAoB,SAAS,eAAe;AAAA,EAC1D;AAKA,QAAM,cAA0C,CAAC;AACjD,MAAI,aAAa,QAAW;AAC1B,eAAW,SAAS,KAAK,KAAK,UAAU,CAAC,GAAG;AAC1C,UAAI,MAAM,aAAa,CAAC,MAAM,UAAU,KAAK,KAAK,QAAQ,EAAG;AAC7D,UAAI;AACF,oBAAY,MAAM,IAAI,IAAI,MAAM,aAAa,OAAO;AAAA,UAClD;AAAA,UACA,UAAU,KAAK,KAAK;AAAA,UACpB,QAAQ,KAAK;AAAA,QACf,CAAC;AAAA,MACH,SAAS,KAAK;AACZ,uBAAe,UAAU,MAAM,IAAI,aAAa,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAChG;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,QAAM,MAAM,MAAM;AAElB,QAAM,OAAsC;AAAA,IAC1C,cAAc,KAAK;AAAA,IACnB,QAAQ,KAAK,KAAK;AAAA,IAClB,YAAY,KAAK,KAAK,SAAS;AAAA,IAC/B,KAAK,KAAK,KAAK;AAAA,IACf,UAAW,YAAY;AAAA,IACvB;AAAA,IACA,SAAS;AAAA,IACT,YAAY,EAAE,GAAG,YAAY;AAAA,IAC7B,GAAI,gBAAgB,EAAE,cAAc,IAAI,CAAC;AAAA,IACzC,YAAY,KAAK,IAAI,IAAI;AAAA,IACzB,MAAM,KAAK,KAAK;AAAA,IAChB,QAAQ;AAAA,IACR,OAAO;AAAA,EACT;AAEA,MAAI,CAAC,gBAAgB,KAAK,WAAW;AACnC,YAAQ,MAAM,WAAW,KAAK,UAAU,IAAI,CAAC;AAAA,EAC/C;AAEA,SAAO,EAAE,MAAM,gBAAgB;AACjC;AAkCO,SAAS,gBACd,MACiB;AACjB,QAAM,OAAO,KAAK,QAAQ;AAC1B,QAAM,OAAO,KAAK,QAAQ;AAC1B,QAAM,YAAY,KAAK,aAAa;AACpC,QAAM,UAAU,KAAK,WAAW,kBAAkB;AAElD,MAAI,OAAO,KAAK,WAAW,YAAY,KAAK,OAAO,KAAK,EAAE,WAAW,GAAG;AACtE,UAAM,IAAI,MAAM,oEAAoE;AAAA,EACtF;AACA,OAAK,SAAS,cAAc,KAAK,QAAQ,KAAK,IAAI;AAElD,QAAM,eAAe,oBAAoB;AAAA,IACvC,WAAW,KAAK;AAAA,IAChB,QAAS,KAAK,UAAU,CAAC;AAAA,IACzB,aAAa,eAAe,KAAK,UAAU,KAAK,WAAW;AAAA,IAC3D;AAAA,IACA;AAAA,EACF,CAAC;AAED,QAAM,QAAQ,kBAAkB,KAAK,WAAW,MAAM,IAAI,EAAE,IAAI,CAAC,SAA8B;AAC7F,UAAM,YAAYA;AAAA,MAChB,KAAK;AAAA,MACL,KAAK,OAAO,QAAQ,mBAAmB,GAAG;AAAA,MAC1C;AAAA,IACF;AACA,QAAI,CAAC,WAAW;AACd,aAAO;AAAA,QACL,QAAQ,KAAK;AAAA,QACb,YAAY,KAAK,SAAS;AAAA,QAC1B,KAAK,KAAK;AAAA,QACV,MAAM,KAAK;AAAA,QACX;AAAA,QACA,QAAQ;AAAA,QACR,QAAQ;AAAA,MACV;AAAA,IACF;AAEA,UAAM,SAAS,eAAwB;AAAA,MACrC;AAAA,MACA;AAAA,MACA,QAAQ,KAAK;AAAA,MACb;AAAA,IACF,CAAC;AACD,QAAI,OAAO,WAAW,OAAO;AAC3B,aAAO;AAAA,QACL,QAAQ,KAAK;AAAA,QACb,YAAY,KAAK,SAAS;AAAA,QAC1B,KAAK,KAAK;AAAA,QACV,MAAM,KAAK;AAAA,QACX;AAAA,QACA,QAAQ;AAAA,MACV;AAAA,IACF;AAEA,WAAO;AAAA,MACL,QAAQ,KAAK;AAAA,MACb,YAAY,KAAK,SAAS;AAAA,MAC1B,KAAK,KAAK;AAAA,MACV,MAAM,KAAK;AAAA,MACX;AAAA,MACA,QAAQ;AAAA,MACR,QAAQ,OAAO;AAAA,IACjB;AAAA,EACF,CAAC;AAED,QAAM,cAAc,MAAM,OAAO,CAAC,SAAS,KAAK,WAAW,QAAQ,EAAE;AACrE,SAAO;AAAA,IACL;AAAA,IACA,YAAY,MAAM;AAAA,IAClB;AAAA,IACA,YAAY,MAAM,SAAS;AAAA,IAC3B;AAAA,EACF;AACF;AASA,SAAS,iBACP,MACA,MACM;AACN,MAAI,SAAS,SAAS,KAAK,MAAO;AAClC,MAAI,KAAK,aAAa,QAAQ,KAAK,aAAa,OAAW;AAC3D,QAAM,aAAa,KAAK,WAAW,UAAU,KAAK,KAAK,WAAW,WAAW;AAC7E,MAAI,KAAK,YAAY,KAAK,CAAC,WAAY;AACvC,QAAM,MAAM,SAAS,KAAK,MAAM;AAChC,MAAI,SAAS,UAAU;AACrB,UAAM,SAAiC;AAAA,MACrC,cAAc;AAAA,MACd,aAAa;AAAA,MACb,aAAa;AAAA,MACb,iBAAiB;AAAA,MACjB,kBAAkB;AAAA,MAClB,mBAAmB;AAAA,MACnB,cAAc;AAAA,MACd,SAAS;AAAA,MACT,WAAW;AAAA,IACb;AACA,UAAM,IAAI,sBAAsB,gBAAgB,GAAG,IAAI,MAAM;AAAA,EAC/D;AAEA,UAAQ,KAAK,8BAA8B,GAAG,EAAE;AAClD;AAEA,eAAe,aACb,OACA,OACqB;AACrB,SAAO,MAAM,MAAM,KAAK;AAC1B;AAEA,SAAS,wBACP,SACsD;AACtD,SAAO,CAAC,QAAQ,QAAQ;AACtB,UAAM,QAAwC,CAAC;AAC/C,WAAO;AAAA,MACL,KAAK,MAAM,YAAY;AACrB,cAAM,UAAU,KAAK,IAAI;AACzB,cAAM,SAAkC,EAAE,MAAM,QAAQ,SAAS,GAAI,cAAc,CAAC,EAAG;AACvF,cAAM,SAAoB;AAAA,UACxB,IAAI,UAAU;AACZ,mBAAO,aAAa,KAAK,IAAI,IAAI;AACjC,gBAAI,SAAU,QAAO,OAAO,QAAQ,QAAQ;AAC5C,kBAAM,KAAK,MAAM;AAAA,UACnB;AAAA,UACA,aAAa,KAAK,OAAO;AACvB,mBAAO,GAAG,IAAI;AAAA,UAChB;AAAA,QACF;AACA,eAAO;AAAA,MACT;AAAA,MACA,MAAM,QAAQ;AACZ,gBAAQ,MAAMA,MAAK,KAAK,aAAa,GAAG,MAAM,IAAI,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC;AAAA,MACxF;AAAA,IACF;AAAA,EACF;AACF;AAEA,SAAS,YACP,MACA,QAC+B;AAC/B,SAAO;AAAA,IACL,QAAQ,KAAK;AAAA,IACb,YAAY,KAAK,SAAS;AAAA,IAC1B,KAAK,KAAK;AAAA,IACV,UAAU;AAAA,IACV,aAAa,CAAC;AAAA,IACd,SAAS;AAAA,IACT,YAAY,EAAE,OAAO,GAAG,QAAQ,EAAE;AAAA,IAClC,YAAY;AAAA,IACZ,MAAM,KAAK;AAAA,IACX,QAAQ;AAAA,IACR,OAAO,YAAY,MAAM;AAAA,EAC3B;AACF;AAEA,SAAS,kBACP,WACA,MACA,MAC+E;AAC/E,QAAM,WAA0F,CAAC;AACjG,MAAI,YAAY;AAChB,aAAW,YAAY,WAAW;AAChC,aAAS,MAAM,GAAG,MAAM,MAAM,OAAO;AACnC,YAAM,SAAS,GAAG,SAAS,EAAE,IAAI,GAAG;AACpC,YAAM,WAAW,OAAO;AACxB,eAAS,KAAK,EAAE,UAAU,KAAK,QAAQ,SAAS,CAAC;AACjD,mBAAa;AAAA,IACf;AAAA,EACF;AACA,SAAO;AACT;AAEA,SAAS,eACP,UACA,UACQ;AACR,QAAM,MAAM,YAAY,UAAU,QAAQ;AAC1C,MAAI,OAAO,QAAQ,YAAY,IAAI,KAAK,EAAE,WAAW,GAAG;AACtD,UAAM,IAAI,MAAM,mEAAmE;AAAA,EACrF;AACA,SAAO;AACT;AAMA,SAAS,eAA0B,MAKV;AACvB,QAAM,MAAM,KAAK,QAAQ,KAAK,KAAK,SAAS;AAC5C,MAAI,QAAQ,OAAW,QAAO,EAAE,QAAQ,QAAQ,QAAQ,UAAU;AAElE,MAAI;AACF,UAAM,SAAS,KAAK,MAAM,GAAG;AAC7B,QAAI,OAAO,WAAW,KAAK,OAAQ,QAAO,EAAE,QAAQ,QAAQ,QAAQ,gBAAgB;AACpF,QAAI,OAAO,iBAAiB,KAAK,cAAc;AAC7C,aAAO,EAAE,QAAQ,QAAQ,QAAQ,oBAAoB;AAAA,IACvD;AACA,WAAO,EAAE,QAAQ,OAAO,MAAM,OAAO;AAAA,EACvC,QAAQ;AACN,WAAO,EAAE,QAAQ,QAAQ,QAAQ,UAAU;AAAA,EAC7C;AACF;AAUA,eAAe,eACb,MACe;AACf,QAAM,KAAK,MAAM,QAAQ;AAAA,IACvB,UAAU,KAAK;AAAA,IACf,UAAU,KAAK,KAAK;AAAA,IACpB,aAAa,KAAK,KAAK;AAAA,IACvB,QAAQ,KAAK,KAAK,iBAAiB;AAAA,IACnC,mBAAmB,KAAK,KAAK,4BAA4B;AAAA,IACzD,YAAY,KAAK,IAAI,EAAE,YAAY;AAAA,IACnC,iBAAiB;AAAA,EACnB,CAAC;AACH;AAIA,SAAS,oBAAoB,OAMlB;AACT,SAAO,YAAY;AAAA,IACjB,WAAW,MAAM;AAAA,IACjB,QAAQ,MAAM,OAAO,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,MAAM,EAAE,WAAW,EAAE;AAAA,IACtE,UAAU,MAAM;AAAA,IAChB,MAAM,MAAM;AAAA,IACZ,MAAM,MAAM;AAAA,EACd,CAAC;AACH;AAEA,SAAS,kBACP,OACA,QACA,MACoB;AACpB,QAAM,UAA0C,CAAC;AACjD,aAAW,SAAS,QAAQ;AAC1B,UAAM,SAAmB,CAAC;AAC1B,eAAW,QAAQ,OAAO;AACxB,YAAM,IAAI,KAAK,YAAY,MAAM,IAAI;AACrC,UAAI,MAAM,OAAW,QAAO,KAAK,EAAE,SAAS;AAAA,IAC9C;AACA,YAAQ,MAAM,IAAI,IAAI,UAAU,QAAQ,IAAI;AAAA,EAC9C;AACA,QAAM,aAAgD,CAAC;AACvD,QAAM,iBAAiB,oBAAI,IAAsB;AACjD,aAAW,QAAQ,OAAO;AACxB,UAAM,aAAa,OAAO,OAAO,KAAK,WAAW,EAAE,IAAI,CAAC,MAAM,EAAE,SAAS;AACzE,QAAI,WAAW,WAAW,EAAG;AAC7B,UAAM,OAAO,WAAW,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,WAAW;AAChE,UAAM,MAAM,eAAe,IAAI,KAAK,UAAU,KAAK,CAAC;AACpD,QAAI,KAAK,IAAI;AACb,mBAAe,IAAI,KAAK,YAAY,GAAG;AAAA,EACzC;AACA,aAAW,CAAC,YAAY,OAAO,KAAK,gBAAgB;AAClD,UAAM,KAAK,UAAU,SAAS,IAAI;AAClC,eAAW,UAAU,IAAI,EAAE,eAAe,GAAG,MAAM,MAAM,GAAG,MAAM,GAAG,GAAG,EAAE;AAAA,EAC5E;AACA,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,cAAc,MAAM,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,SAAS,CAAC;AAAA,IACrD,eAAe,MAAM,OAAO,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE;AAAA,IAC7C,cAAc,MAAM,OAAO,CAAC,MAAM,EAAE,OAAO,WAAW,UAAU,CAAC,EAAE;AAAA,IACnE,aAAa,MAAM,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE;AAAA,IAC3C,aAAa,MAAM,OAAO,CAAC,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,WAAW,UAAU,CAAC,EAAE;AAAA,EAC/E;AACF;AAKA,SAAS,UAAU,SAAmB,MAA8B;AAClE,QAAM,IAAI,QAAQ;AAClB,MAAI,MAAM,EAAG,QAAO,EAAE,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,GAAG,EAAE;AAC5D,QAAM,OAAO,QAAQ,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI;AAClD,QAAM,WAAW,QAAQ,OAAO,CAAC,GAAG,MAAM,KAAK,IAAI,SAAS,GAAG,CAAC,IAAI,KAAK,IAAI,GAAG,IAAI,CAAC;AACrF,QAAM,QAAQ,KAAK,KAAK,QAAQ;AAChC,QAAM,KAAK,mBAAmB,SAAS,MAAM,EAAE,MAAM,WAAW,IAAK,CAAC;AACtE,SAAO,EAAE,MAAM,OAAO,MAAM,CAAC,GAAG,OAAO,GAAG,KAAK,GAAG,EAAE;AACtD;","names":["join","existsSync","readFileSync","join"]}
@@ -1 +0,0 @@
1
- {"version":3,"sources":["../src/trace/query.ts"],"sourcesContent":["/**\n * Typed query helpers over TraceStore.\n *\n * Not a full SQL engine — a minimal, composable set of operators that\n * cover the canned-pipeline use cases. For ad-hoc analytics, persist to\n * NDJSON and point DuckDB at it; the schema is stable so external SQL\n * tooling works out of the box.\n */\n\nimport type { FailureClass, JudgeSpan, LlmSpan, Run, ToolSpan } from './schema'\nimport { isJudgeSpan, isLlmSpan, isToolSpan } from './schema'\nimport type { TraceStore } from './store'\n\nexport async function runsForScenario(store: TraceStore, scenarioId: string): Promise<Run[]> {\n return store.listRuns({ scenarioId })\n}\n\nexport async function llmSpans(store: TraceStore, runId?: string): Promise<LlmSpan[]> {\n const spans = await store.spans({ runId, kind: 'llm' })\n return spans.filter(isLlmSpan)\n}\n\nexport async function toolSpans(\n store: TraceStore,\n runId?: string,\n toolName?: string,\n): Promise<ToolSpan[]> {\n const spans = await store.spans({ runId, kind: 'tool', toolName })\n return spans.filter(isToolSpan)\n}\n\nexport async function judgeSpans(store: TraceStore, runId?: string): Promise<JudgeSpan[]> {\n const spans = await store.spans({ runId, kind: 'judge' })\n return spans.filter(isJudgeSpan)\n}\n\n/** Group spans by any key selector. */\nexport function groupBy<T, K extends string | number>(items: T[], key: (t: T) => K): Map<K, T[]> {\n const map = new Map<K, T[]>()\n for (const item of items) {\n const k = key(item)\n let bucket = map.get(k)\n if (!bucket) {\n bucket = []\n map.set(k, bucket)\n }\n bucket.push(item)\n }\n return map\n}\n\n/** Hash tool arguments to an orderless-key-stable string for de-duplication. */\nexport function argHash(args: unknown): string {\n return stableStringify(args)\n}\n\nfunction stableStringify(value: unknown): string {\n // Must ALWAYS return a string: JSON.stringify(undefined) — and stringify of\n // functions/symbols — yields the JS value `undefined`, which would make\n // argHash return a non-string and silently break the de-dup keys behind\n // stuck-loop and failure-cluster.\n if (value === undefined) return 'undefined'\n if (value === null || typeof value !== 'object') return JSON.stringify(value) ?? 'undefined'\n if (Array.isArray(value)) return `[${value.map(stableStringify).join(',')}]`\n // Drop undefined-valued keys to match JSON.stringify semantics, so\n // `{a:1}` and `{a:1,b:undefined}` hash identically.\n const obj = value as Record<string, unknown>\n const keys = Object.keys(obj)\n .filter((k) => obj[k] !== undefined)\n .sort()\n const parts = keys.map((k) => `${JSON.stringify(k)}:${stableStringify(obj[k])}`)\n return `{${parts.join(',')}}`\n}\n\n/** Sum an LLM-span array into aggregate token + cost. */\nexport function aggregateLlm(spans: LlmSpan[]): {\n inputTokens: number\n outputTokens: number\n cachedTokens: number\n reasoningTokens: number\n costUsd: number\n} {\n return spans.reduce(\n (acc, s) => ({\n inputTokens: acc.inputTokens + (s.inputTokens ?? 0),\n outputTokens: acc.outputTokens + (s.outputTokens ?? 0),\n cachedTokens: acc.cachedTokens + (s.cachedTokens ?? 0),\n // reasoningTokens is on LlmSpan but was omitted here — reasoning usage was\n // invisible to any cost/perf analysis reading this aggregate.\n reasoningTokens: acc.reasoningTokens + (s.reasoningTokens ?? 0),\n costUsd: acc.costUsd + (s.costUsd ?? 0),\n }),\n { inputTokens: 0, outputTokens: 0, cachedTokens: 0, reasoningTokens: 0, costUsd: 0 },\n )\n}\n\n/** Pick the outcome's failure class when present, else derive 'success' from run status. */\nexport function runFailureClass(run: Run): FailureClass {\n if (run.outcome?.failureClass) return run.outcome.failureClass\n if (run.status === 'completed' && run.outcome?.pass !== false) return 'success'\n if (run.status === 'aborted') return 'budget_exceeded'\n return 'unknown'\n}\n"],"mappings":";;;;;;;AAaA,eAAsB,gBAAgB,OAAmB,YAAoC;AAC3F,SAAO,MAAM,SAAS,EAAE,WAAW,CAAC;AACtC;AAEA,eAAsB,SAAS,OAAmB,OAAoC;AACpF,QAAM,QAAQ,MAAM,MAAM,MAAM,EAAE,OAAO,MAAM,MAAM,CAAC;AACtD,SAAO,MAAM,OAAO,SAAS;AAC/B;AAEA,eAAsB,UACpB,OACA,OACA,UACqB;AACrB,QAAM,QAAQ,MAAM,MAAM,MAAM,EAAE,OAAO,MAAM,QAAQ,SAAS,CAAC;AACjE,SAAO,MAAM,OAAO,UAAU;AAChC;AAEA,eAAsB,WAAW,OAAmB,OAAsC;AACxF,QAAM,QAAQ,MAAM,MAAM,MAAM,EAAE,OAAO,MAAM,QAAQ,CAAC;AACxD,SAAO,MAAM,OAAO,WAAW;AACjC;AAGO,SAAS,QAAsC,OAAY,KAA+B;AAC/F,QAAM,MAAM,oBAAI,IAAY;AAC5B,aAAW,QAAQ,OAAO;AACxB,UAAM,IAAI,IAAI,IAAI;AAClB,QAAI,SAAS,IAAI,IAAI,CAAC;AACtB,QAAI,CAAC,QAAQ;AACX,eAAS,CAAC;AACV,UAAI,IAAI,GAAG,MAAM;AAAA,IACnB;AACA,WAAO,KAAK,IAAI;AAAA,EAClB;AACA,SAAO;AACT;AAGO,SAAS,QAAQ,MAAuB;AAC7C,SAAO,gBAAgB,IAAI;AAC7B;AAEA,SAAS,gBAAgB,OAAwB;AAK/C,MAAI,UAAU,OAAW,QAAO;AAChC,MAAI,UAAU,QAAQ,OAAO,UAAU,SAAU,QAAO,KAAK,UAAU,KAAK,KAAK;AACjF,MAAI,MAAM,QAAQ,KAAK,EAAG,QAAO,IAAI,MAAM,IAAI,eAAe,EAAE,KAAK,GAAG,CAAC;AAGzE,QAAM,MAAM;AACZ,QAAM,OAAO,OAAO,KAAK,GAAG,EACzB,OAAO,CAAC,MAAM,IAAI,CAAC,MAAM,MAAS,EAClC,KAAK;AACR,QAAM,QAAQ,KAAK,IAAI,CAAC,MAAM,GAAG,KAAK,UAAU,CAAC,CAAC,IAAI,gBAAgB,IAAI,CAAC,CAAC,CAAC,EAAE;AAC/E,SAAO,IAAI,MAAM,KAAK,GAAG,CAAC;AAC5B;AAGO,SAAS,aAAa,OAM3B;AACA,SAAO,MAAM;AAAA,IACX,CAAC,KAAK,OAAO;AAAA,MACX,aAAa,IAAI,eAAe,EAAE,eAAe;AAAA,MACjD,cAAc,IAAI,gBAAgB,EAAE,gBAAgB;AAAA,MACpD,cAAc,IAAI,gBAAgB,EAAE,gBAAgB;AAAA;AAAA;AAAA,MAGpD,iBAAiB,IAAI,mBAAmB,EAAE,mBAAmB;AAAA,MAC7D,SAAS,IAAI,WAAW,EAAE,WAAW;AAAA,IACvC;AAAA,IACA,EAAE,aAAa,GAAG,cAAc,GAAG,cAAc,GAAG,iBAAiB,GAAG,SAAS,EAAE;AAAA,EACrF;AACF;AAGO,SAAS,gBAAgB,KAAwB;AACtD,MAAI,IAAI,SAAS,aAAc,QAAO,IAAI,QAAQ;AAClD,MAAI,IAAI,WAAW,eAAe,IAAI,SAAS,SAAS,MAAO,QAAO;AACtE,MAAI,IAAI,WAAW,UAAW,QAAO;AACrC,SAAO;AACT;","names":[]}