@tangle-network/agent-eval 0.126.7 → 0.127.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (129) hide show
  1. package/CHANGELOG.md +44 -0
  2. package/README.md +2 -2
  3. package/dist/analyst/index.d.ts +51 -39
  4. package/dist/analyst/index.js +4 -4
  5. package/dist/belief-state/index.d.ts +34 -36
  6. package/dist/belief-state/index.js +2 -2
  7. package/dist/benchmarks/index.d.ts +30 -12
  8. package/dist/benchmarks/index.js +12 -10
  9. package/dist/builder-eval/index.js +1 -1
  10. package/dist/campaign/index.d.ts +432 -146
  11. package/dist/campaign/index.js +11 -9
  12. package/dist/{chunk-ARU2PZFM.js → chunk-424XHVKV.js} +43 -23
  13. package/dist/chunk-424XHVKV.js.map +1 -0
  14. package/dist/{chunk-NUKSVU3W.js → chunk-5DTSBUL2.js} +2 -2
  15. package/dist/{chunk-BTACVO2W.js → chunk-65QETEKG.js} +542 -258
  16. package/dist/chunk-65QETEKG.js.map +1 -0
  17. package/dist/{chunk-CGG5SLH3.js → chunk-ARCFIY7U.js} +2 -2
  18. package/dist/{chunk-P22LJ3Y2.js → chunk-CLELFRQG.js} +6 -4
  19. package/dist/chunk-CLELFRQG.js.map +1 -0
  20. package/dist/{chunk-LUNF2SEL.js → chunk-DJKY2TSY.js} +3 -3
  21. package/dist/chunk-DPUHNQLN.js +232 -0
  22. package/dist/chunk-DPUHNQLN.js.map +1 -0
  23. package/dist/{chunk-IPYXE555.js → chunk-EJGRPCO3.js} +36 -13
  24. package/dist/chunk-EJGRPCO3.js.map +1 -0
  25. package/dist/{chunk-K6IAZZ6L.js → chunk-FA3MFYNA.js} +132 -110
  26. package/dist/chunk-FA3MFYNA.js.map +1 -0
  27. package/dist/chunk-IG35YDN3.js +614 -0
  28. package/dist/chunk-IG35YDN3.js.map +1 -0
  29. package/dist/{chunk-DPZAEKA6.js → chunk-IMGJZBFG.js} +84 -56
  30. package/dist/chunk-IMGJZBFG.js.map +1 -0
  31. package/dist/{chunk-H5UD2323.js → chunk-IN455XKS.js} +53 -1
  32. package/dist/chunk-IN455XKS.js.map +1 -0
  33. package/dist/{chunk-PJQFMIOX.js → chunk-MHELPNRP.js} +31 -1
  34. package/dist/chunk-MHELPNRP.js.map +1 -0
  35. package/dist/{chunk-IILEIWGW.js → chunk-NL33OXEP.js} +41 -36
  36. package/dist/chunk-NL33OXEP.js.map +1 -0
  37. package/dist/{chunk-474LBSOX.js → chunk-OD5NQ2CS.js} +41 -16
  38. package/dist/chunk-OD5NQ2CS.js.map +1 -0
  39. package/dist/{chunk-QOTFXW5L.js → chunk-P5W7RQKK.js} +2 -2
  40. package/dist/{chunk-IR3KBHOY.js → chunk-P6FYH6K4.js} +39 -3
  41. package/dist/chunk-P6FYH6K4.js.map +1 -0
  42. package/dist/{chunk-J5SQWP6Y.js → chunk-PBE2LOSS.js} +18 -2
  43. package/dist/chunk-PBE2LOSS.js.map +1 -0
  44. package/dist/{chunk-E3IADPLG.js → chunk-PRB53E3J.js} +50 -224
  45. package/dist/chunk-PRB53E3J.js.map +1 -0
  46. package/dist/{chunk-X4UCIOTZ.js → chunk-S5YLIBFX.js} +2 -2
  47. package/dist/{chunk-7VYEGHWF.js → chunk-SZLVEKMJ.js} +41 -20
  48. package/dist/chunk-SZLVEKMJ.js.map +1 -0
  49. package/dist/{chunk-LKKT3IVV.js → chunk-TSN7JT6D.js} +6 -4
  50. package/dist/{chunk-LKKT3IVV.js.map → chunk-TSN7JT6D.js.map} +1 -1
  51. package/dist/{chunk-M7AH34KV.js → chunk-UTA2TGC4.js} +32 -16
  52. package/dist/chunk-UTA2TGC4.js.map +1 -0
  53. package/dist/{chunk-MAX3TN3C.js → chunk-UWZZKKU7.js} +7 -19
  54. package/dist/chunk-UWZZKKU7.js.map +1 -0
  55. package/dist/{chunk-ZVCHKKOP.js → chunk-WED5KLZQ.js} +39 -15
  56. package/dist/chunk-WED5KLZQ.js.map +1 -0
  57. package/dist/{chunk-OCFJACJU.js → chunk-X6VOPMLD.js} +74 -14
  58. package/dist/chunk-X6VOPMLD.js.map +1 -0
  59. package/dist/{chunk-MOXWMGPC.js → chunk-Y5CYOL42.js} +159 -70
  60. package/dist/chunk-Y5CYOL42.js.map +1 -0
  61. package/dist/{chunk-FO7HEH76.js → chunk-YJBNWCAA.js} +2 -2
  62. package/dist/chunk-ZPUEEM5U.js +179 -0
  63. package/dist/chunk-ZPUEEM5U.js.map +1 -0
  64. package/dist/{chunk-KE2VWPZX.js → chunk-ZUUWPZCV.js} +3 -3
  65. package/dist/cli.js +4 -4
  66. package/dist/contract/index.d.ts +421 -124
  67. package/dist/contract/index.js +242 -115
  68. package/dist/contract/index.js.map +1 -1
  69. package/dist/control.d.ts +34 -32
  70. package/dist/control.js +2 -2
  71. package/dist/hosted/index.d.ts +142 -41
  72. package/dist/hosted/index.js +23 -1
  73. package/dist/index.d.ts +364 -255
  74. package/dist/index.js +315 -226
  75. package/dist/index.js.map +1 -1
  76. package/dist/meta-eval/index.d.ts +32 -32
  77. package/dist/meta-eval/index.js +2 -2
  78. package/dist/multishot/index.d.ts +1 -0
  79. package/dist/openapi.json +1 -1
  80. package/dist/pipelines/index.js +2 -2
  81. package/dist/reporting.d.ts +97 -74
  82. package/dist/reporting.js +5 -4
  83. package/dist/rl.d.ts +177 -131
  84. package/dist/rl.js +232 -126
  85. package/dist/rl.js.map +1 -1
  86. package/dist/rollout/index.d.ts +53 -63
  87. package/dist/rollout/index.js +7 -5
  88. package/dist/{run-campaign-FFRM3RH5.js → run-campaign-3A2EBYBE.js} +4 -3
  89. package/dist/supervisor-run/index.d.ts +7 -15
  90. package/dist/supervisor-run/index.js +2 -2
  91. package/dist/traces.d.ts +66 -41
  92. package/dist/traces.js +9 -5
  93. package/dist/wire/index.d.ts +1 -0
  94. package/dist/wire/index.js +2 -2
  95. package/docs/concepts.md +18 -1
  96. package/docs/hosted-ingest-spec.md +37 -40
  97. package/docs/insight-report.md +57 -7
  98. package/docs/research-report-methodology.md +20 -21
  99. package/docs/rollout.md +13 -8
  100. package/package.json +4 -3
  101. package/dist/chunk-474LBSOX.js.map +0 -1
  102. package/dist/chunk-7VYEGHWF.js.map +0 -1
  103. package/dist/chunk-ARU2PZFM.js.map +0 -1
  104. package/dist/chunk-BTACVO2W.js.map +0 -1
  105. package/dist/chunk-DPZAEKA6.js.map +0 -1
  106. package/dist/chunk-E3IADPLG.js.map +0 -1
  107. package/dist/chunk-H5UD2323.js.map +0 -1
  108. package/dist/chunk-IILEIWGW.js.map +0 -1
  109. package/dist/chunk-IPYXE555.js.map +0 -1
  110. package/dist/chunk-IR3KBHOY.js.map +0 -1
  111. package/dist/chunk-J5SQWP6Y.js.map +0 -1
  112. package/dist/chunk-K6IAZZ6L.js.map +0 -1
  113. package/dist/chunk-M7AH34KV.js.map +0 -1
  114. package/dist/chunk-MAX3TN3C.js.map +0 -1
  115. package/dist/chunk-MOXWMGPC.js.map +0 -1
  116. package/dist/chunk-OCFJACJU.js.map +0 -1
  117. package/dist/chunk-P22LJ3Y2.js.map +0 -1
  118. package/dist/chunk-PJQFMIOX.js.map +0 -1
  119. package/dist/chunk-ZVCHKKOP.js.map +0 -1
  120. package/dist/chunk-ZZUXHH3R.js +0 -99
  121. package/dist/chunk-ZZUXHH3R.js.map +0 -1
  122. /package/dist/{chunk-NUKSVU3W.js.map → chunk-5DTSBUL2.js.map} +0 -0
  123. /package/dist/{chunk-CGG5SLH3.js.map → chunk-ARCFIY7U.js.map} +0 -0
  124. /package/dist/{chunk-LUNF2SEL.js.map → chunk-DJKY2TSY.js.map} +0 -0
  125. /package/dist/{chunk-QOTFXW5L.js.map → chunk-P5W7RQKK.js.map} +0 -0
  126. /package/dist/{chunk-X4UCIOTZ.js.map → chunk-S5YLIBFX.js.map} +0 -0
  127. /package/dist/{chunk-FO7HEH76.js.map → chunk-YJBNWCAA.js.map} +0 -0
  128. /package/dist/{chunk-KE2VWPZX.js.map → chunk-ZUUWPZCV.js.map} +0 -0
  129. /package/dist/{run-campaign-FFRM3RH5.js.map → run-campaign-3A2EBYBE.js.map} +0 -0
@@ -7,14 +7,14 @@ import {
7
7
  fromPigraphSession,
8
8
  observeCodeAgentSession,
9
9
  parseCodeAgentJsonl
10
- } from "../chunk-7VYEGHWF.js";
10
+ } from "../chunk-SZLVEKMJ.js";
11
11
  import {
12
12
  createHostedClient
13
- } from "../chunk-ZZUXHH3R.js";
13
+ } from "../chunk-IG35YDN3.js";
14
14
  import {
15
15
  analyzeRuns,
16
16
  summarizeExecution
17
- } from "../chunk-K6IAZZ6L.js";
17
+ } from "../chunk-FA3MFYNA.js";
18
18
  import {
19
19
  REFERENCE_EQUIVALENCE_INPUT_LIMITS,
20
20
  REFERENCE_EQUIVALENCE_JUDGE_VERSION,
@@ -40,7 +40,7 @@ import {
40
40
  skillOptOptimizationMethod,
41
41
  surfaceContentHash,
42
42
  surfaceHash
43
- } from "../chunk-BTACVO2W.js";
43
+ } from "../chunk-65QETEKG.js";
44
44
  import {
45
45
  campaignSplitDigest,
46
46
  createRunCostLedger,
@@ -48,47 +48,60 @@ import {
48
48
  inMemoryCampaignStorage,
49
49
  resolveRunDir,
50
50
  runCampaign
51
- } from "../chunk-ZVCHKKOP.js";
51
+ } from "../chunk-WED5KLZQ.js";
52
52
  import {
53
53
  buildDefaultAnalystRegistry,
54
54
  createChatClient
55
- } from "../chunk-LUNF2SEL.js";
55
+ } from "../chunk-DJKY2TSY.js";
56
56
  import "../chunk-HHWE3POT.js";
57
57
  import "../chunk-WGXIEX7P.js";
58
58
  import {
59
59
  FileSystemOutcomeStore,
60
60
  InMemoryOutcomeStore
61
61
  } from "../chunk-3RF76KTD.js";
62
- import "../chunk-ARU2PZFM.js";
63
- import "../chunk-J5SQWP6Y.js";
64
- import "../chunk-DPZAEKA6.js";
62
+ import "../chunk-424XHVKV.js";
63
+ import {
64
+ campaignCellExecutionEvidence,
65
+ campaignCellJudgeDimensions,
66
+ campaignCellTaskScore,
67
+ campaignCellToRunRecord
68
+ } from "../chunk-ZPUEEM5U.js";
69
+ import "../chunk-PBE2LOSS.js";
70
+ import "../chunk-IMGJZBFG.js";
71
+ import "../chunk-DPUHNQLN.js";
65
72
  import {
66
73
  pairedBootstrap
67
- } from "../chunk-PJQFMIOX.js";
74
+ } from "../chunk-MHELPNRP.js";
68
75
  import "../chunk-WS3NZZQQ.js";
69
76
  import "../chunk-VI2UW6B6.js";
70
77
  import {
71
78
  recordAggregateMeasurements,
72
- summarizeExecutionMeasurements
73
- } from "../chunk-H5UD2323.js";
79
+ summarizeExecutionMeasurements,
80
+ summarizeTraceErrors
81
+ } from "../chunk-IN455XKS.js";
74
82
  import "../chunk-PXE2VKMX.js";
75
83
  import "../chunk-ZET2UAYW.js";
76
84
  import "../chunk-GGE4NNQT.js";
77
- import "../chunk-IR3KBHOY.js";
85
+ import {
86
+ classifyOtlpSpanRole,
87
+ isOtlpModelCall
88
+ } from "../chunk-P6FYH6K4.js";
78
89
  import "../chunk-PC4UYEBM.js";
79
90
  import {
80
91
  parseRunRecordSafe
81
- } from "../chunk-IILEIWGW.js";
92
+ } from "../chunk-NL33OXEP.js";
82
93
  import "../chunk-MA6HLL3S.js";
83
94
  import "../chunk-VSMTAMNK.js";
84
- import "../chunk-ONWEPEDO.js";
85
95
  import {
86
- LLM_MODEL_ATTR_KEYS
96
+ ValidationError
97
+ } from "../chunk-ONWEPEDO.js";
98
+ import {
99
+ LLM_MODEL_ATTR_KEYS,
100
+ SPAN_KIND_ATTR_KEYS
87
101
  } from "../chunk-K4DBDHLK.js";
88
102
  import "../chunk-PZ5AY32C.js";
89
103
 
90
104
  // src/contract/self-improve.ts
91
- import { createHash } from "crypto";
92
105
  var SelfImproveRunError = class extends Error {
93
106
  cost;
94
107
  receipts;
@@ -343,11 +356,11 @@ async function runSelfImprove(opts, costLedger, startedAt, runDir, storage) {
343
356
  findings: opts.findings,
344
357
  selectionRankKey: opts.selectionRankKey
345
358
  });
346
- const winnerSearch = holdoutDeferred ? winnerSearchCampaign(result) : void 0;
347
- const baseline = meanComposite(
348
- (holdoutDeferred ? result.baselineCampaign : result.baselineOnHoldout).aggregates.byScenario
349
- );
350
- const winnerStats = meanComposite((winnerSearch ?? result.winnerOnHoldout).aggregates.byScenario);
359
+ const reportSplit = holdoutDeferred ? "search" : "holdout";
360
+ const reportBaselineCampaign = holdoutDeferred ? result.baselineCampaign : result.baselineOnHoldout;
361
+ const reportWinnerCampaign = holdoutDeferred ? winnerSearchCampaign(result) : result.winnerOnHoldout;
362
+ const baseline = meanComposite(reportBaselineCampaign.aggregates.byScenario);
363
+ const winnerStats = meanComposite(reportWinnerCampaign.aggregates.byScenario);
351
364
  let power;
352
365
  const baselineHoldoutComposites = result.baselineOnHoldout.cells.filter((cell) => !cell.error).map((cell) => {
353
366
  const scores = Object.values(cell.judgeScores);
@@ -390,12 +403,19 @@ async function runSelfImprove(opts, costLedger, startedAt, runDir, storage) {
390
403
  const totalCost = cost.totalCostUsd;
391
404
  const insight = await analyzeRuns({
392
405
  runs: [
393
- ...cellsToRunRecords(result.baselineCampaign.cells, "baseline", runDir, opts.baselineSurface),
394
406
  ...cellsToRunRecords(
395
- (winnerSearch ?? result.winnerOnHoldout).cells,
407
+ reportBaselineCampaign.cells,
408
+ "baseline",
409
+ runDir,
410
+ opts.baselineSurface,
411
+ reportSplit
412
+ ),
413
+ ...cellsToRunRecords(
414
+ reportWinnerCampaign.cells,
396
415
  "winner",
397
416
  runDir,
398
- result.winnerSurface
417
+ result.winnerSurface,
418
+ reportSplit
399
419
  )
400
420
  ],
401
421
  baselineCandidateId: "baseline",
@@ -468,19 +488,21 @@ async function shipEvalRunToHosted(tenant, opts, summary, raw, runDir) {
468
488
  const client = createHostedClient(tenant);
469
489
  function snapshotFromCampaign(index, surface, campaign, durationMs) {
470
490
  const cells = campaign.cells.map((cell) => {
471
- const judgeScores = Object.values(cell.judgeScores);
472
- const composite = judgeScores.length === 0 ? 0 : judgeScores.reduce((s, j) => s + j.composite, 0) / judgeScores.length;
491
+ const execution = campaignCellExecutionEvidence(cell);
473
492
  return {
474
493
  scenarioId: cell.scenarioId,
475
494
  rep: cell.rep,
476
- compositeMean: composite,
477
- dimensions: Object.fromEntries(
478
- Object.entries(cell.judgeScores).map(([name, score]) => [name, score.dimensions])
479
- ),
495
+ compositeMean: campaignCellTaskScore(cell) ?? null,
496
+ dimensions: campaignCellJudgeDimensions(cell),
497
+ terminalOutcome: execution.terminalOutcome,
498
+ executionErrorCount: execution.executionErrorCount ?? null,
480
499
  errorMessage: cell.error ?? void 0
481
500
  };
482
501
  });
483
- const compositeMean = cells.length === 0 ? 0 : cells.reduce((s, c) => s + c.compositeMean, 0) / cells.length;
502
+ const scoredCells = cells.flatMap(
503
+ (cell) => cell.compositeMean === null ? [] : [cell.compositeMean]
504
+ );
505
+ const compositeMean = scoredCells.length === 0 ? null : scoredCells.reduce((sum, score) => sum + score, 0) / scoredCells.length;
484
506
  return {
485
507
  index,
486
508
  surfaceHash: surfaceHash(surface),
@@ -531,67 +553,23 @@ function hashString(s) {
531
553
  }
532
554
  return h.toString(16).padStart(8, "0");
533
555
  }
534
- function cellsToRunRecords(cells, candidateId, runId, surface) {
556
+ function cellsToRunRecords(cells, candidateId, runId, surface, splitTag) {
535
557
  const promptHash = surfaceContentHash(surface);
536
- const configHash = `sha256:${createHash("sha256").update(candidateId).digest("hex")}`;
537
- return cells.map((cell) => {
538
- const perJudge = {};
539
- const perDimMeanAccum = {};
540
- let compositeSum = 0;
541
- let compositeCount = 0;
542
- for (const [judgeId, score] of Object.entries(cell.judgeScores)) {
543
- perJudge[judgeId] = { ...score.dimensions };
544
- for (const [dim, value] of Object.entries(score.dimensions)) {
545
- if (!Number.isFinite(value)) continue;
546
- const accum = perDimMeanAccum[dim] ?? { sum: 0, n: 0 };
547
- accum.sum += value;
548
- accum.n += 1;
549
- perDimMeanAccum[dim] = accum;
550
- }
551
- if (Number.isFinite(score.composite)) {
552
- compositeSum += score.composite;
553
- compositeCount += 1;
554
- }
555
- }
556
- const perDimMean = {};
557
- for (const [dim, { sum, n }] of Object.entries(perDimMeanAccum)) {
558
- perDimMean[dim] = n === 0 ? 0 : sum / n;
559
- }
560
- const composite = compositeCount === 0 ? 0 : compositeSum / compositeCount;
561
- const judgeScores = {
562
- perJudge,
563
- perDimMean,
564
- composite
565
- };
566
- return {
558
+ const configHash = surfaceContentHash(candidateId);
559
+ return cells.map(
560
+ (cell) => campaignCellToRunRecord(cell, {
567
561
  runId: `${runId}::${candidateId}::${cell.cellId}`,
568
562
  experimentId: runId,
569
563
  candidateId,
570
- // Pair on (scenarioId, rep) analyzeRuns pairs on (experimentId, seed).
571
- // Synthesize a stable seed for that pairing.
564
+ // scenarioId is explicit; seed keeps repeated runs distinct.
572
565
  seed: cell.rep * 1e6 + hashString(cell.scenarioId).slice(0, 6).split("").reduce((a, c) => a * 31 + c.charCodeAt(0) >>> 0, 0),
573
566
  model: "campaign-cell",
574
567
  promptHash,
575
568
  configHash,
576
569
  commitSha: "cell",
577
- wallMs: cell.durationMs,
578
- costUsd: cell.costUsd,
579
- tokenUsage: {
580
- input: cell.tokenUsage.input,
581
- output: cell.tokenUsage.output,
582
- ...cell.tokenUsage.reasoning === void 0 ? {} : { reasoning: cell.tokenUsage.reasoning },
583
- ...cell.tokenUsage.cached === void 0 ? {} : { cached: cell.tokenUsage.cached },
584
- ...cell.tokenUsage.cacheWrite === void 0 ? {} : { cacheWrite: cell.tokenUsage.cacheWrite }
585
- },
586
- outcome: {
587
- holdoutScore: composite,
588
- raw: {},
589
- judgeScores
590
- },
591
- splitTag: "holdout",
592
- ...cell.error ? { failureMode: cell.error } : {}
593
- };
594
- });
570
+ splitTag
571
+ })
572
+ );
595
573
  }
596
574
 
597
575
  // src/contract/define-agent-eval.ts
@@ -1565,7 +1543,7 @@ function diffGenerations(before, after) {
1565
1543
  rep: beforeCell.rep,
1566
1544
  compositeBefore: beforeCell.compositeMean,
1567
1545
  compositeAfter: afterCell.compositeMean,
1568
- compositeDelta: afterCell.compositeMean - beforeCell.compositeMean,
1546
+ compositeDelta: beforeCell.compositeMean === null || afterCell.compositeMean === null ? null : afterCell.compositeMean - beforeCell.compositeMean,
1569
1547
  dimensions: diffDimensions(beforeCell.dimensions, afterCell.dimensions)
1570
1548
  });
1571
1549
  }
@@ -1583,7 +1561,7 @@ function diffGenerations(before, after) {
1583
1561
  added,
1584
1562
  compositeBefore: before.compositeMean,
1585
1563
  compositeAfter: after.compositeMean,
1586
- compositeDelta: after.compositeMean - before.compositeMean,
1564
+ compositeDelta: before.compositeMean === null || after.compositeMean === null ? null : after.compositeMean - before.compositeMean,
1587
1565
  costUsdBefore: before.costUsd,
1588
1566
  costUsdAfter: after.costUsd,
1589
1567
  costUsdDelta: after.costUsd - before.costUsd,
@@ -1736,14 +1714,13 @@ function fromFeedbackTable(opts) {
1736
1714
  perDimMean: { rating: meanScore },
1737
1715
  composite: meanScore
1738
1716
  };
1717
+ const splitTag = runMeta.splitTag ?? "holdout";
1739
1718
  const outcome = {
1740
- // Feedback corpora ARE the holdout signal score lands on
1741
- // `holdoutScore` so downstream substrate primitives (`paretoChart`,
1742
- // promotion gates) read it correctly by default.
1743
- holdoutScore: meanScore,
1719
+ ...splitTag === "holdout" ? { holdoutScore: meanScore } : { searchScore: meanScore },
1744
1720
  raw: Object.fromEntries(normalised.map((r) => [`rater:${r.rater}`, r.score])),
1745
1721
  judgeScores
1746
1722
  };
1723
+ const costUsd = runMeta.costUsd ?? null;
1747
1724
  runs.push({
1748
1725
  runId,
1749
1726
  experimentId: runMeta.experimentId ?? "feedback-corpus",
@@ -1754,10 +1731,13 @@ function fromFeedbackTable(opts) {
1754
1731
  configHash: runMeta.configHash ?? "sha256:unknown",
1755
1732
  commitSha: runMeta.commitSha ?? "unknown",
1756
1733
  wallMs: runMeta.wallMs ?? 0,
1757
- costUsd: runMeta.costUsd ?? 0,
1734
+ costUsd,
1735
+ costProvenance: costUsd === null ? { kind: "uncaptured", usd: null } : { kind: "observed", usd: costUsd },
1758
1736
  tokenUsage: { input: 0, output: 0 },
1737
+ terminalOutcome: "unknown",
1759
1738
  outcome,
1760
- splitTag: runMeta.splitTag ?? "holdout"
1739
+ splitTag,
1740
+ scenarioId: runMeta.scenarioId ?? runId
1761
1741
  });
1762
1742
  if (emitRaterScores) {
1763
1743
  for (const r of normalised) raterScores.push({ runId, rater: r.rater, score: r.score });
@@ -1767,7 +1747,12 @@ function fromFeedbackTable(opts) {
1767
1747
  }
1768
1748
 
1769
1749
  // src/contract/intake/otel-spans.ts
1770
- var SCORE_KEYS = ["tangle.score", "eval.score", "score"];
1750
+ var TASK_SCORE_ATTR_KEYS = [
1751
+ "gen_ai.evaluation.score.value",
1752
+ "tangle.task.score",
1753
+ "eval.score",
1754
+ "tangle.score"
1755
+ ];
1771
1756
  var MODEL_KEYS = ["tangle.model", ...LLM_MODEL_ATTR_KEYS, "model"];
1772
1757
  var PROMPT_HASH_KEYS = ["tangle.prompt_hash", "prompt.hash"];
1773
1758
  var CONFIG_HASH_KEYS = ["tangle.config_hash", "config.hash"];
@@ -1789,15 +1774,31 @@ function fromOtelSpans(opts) {
1789
1774
  );
1790
1775
  const callSpanIds = new Set(measurements.callSpanIds);
1791
1776
  const callSpans = groupSpans2.filter((span) => callSpanIds.has(span.spanId));
1792
- const wallMs = Math.max(0, (root.endTimeUnixNano - root.startTimeUnixNano) / 1e6);
1777
+ const wallMs = unixNanoDurationMs(root.startTimeUnixNano, root.endTimeUnixNano);
1793
1778
  const model = readAttrString(callSpans, MODEL_KEYS) ?? readAttrString(groupSpans2, MODEL_KEYS) ?? "unknown@unknown";
1794
1779
  const capturedCost = (measurements.cost.complete ? measurements.cost.value : void 0) ?? measurements.aggregate?.costUsd;
1795
- const costUsd = capturedCost ?? 0;
1780
+ const costUsd = capturedCost ?? null;
1781
+ const scenarioId = readConsistentScenarioId(groupKey, groupSpans2) ?? groupKey;
1796
1782
  const promptHash = readAttrString(groupSpans2, PROMPT_HASH_KEYS) ?? "sha256:unknown";
1797
1783
  const configHash = readAttrString(groupSpans2, CONFIG_HASH_KEYS) ?? "sha256:unknown";
1798
- const score = readAttrNumber(groupSpans2, SCORE_KEYS);
1784
+ const score = resolveTaskScore(groupKey, groupSpans2, opts.scoreForRun);
1799
1785
  const rawNumeric = collectNumericAttrs(groupSpans2);
1800
- rawNumeric.error_span_count = groupSpans2.filter((span) => span.status?.code === "ERROR").length;
1786
+ const errorSummary = summarizeTraceErrors(
1787
+ groupSpans2.map((span) => ({
1788
+ id: spanIdentity(span),
1789
+ ...span.parentSpanId ? { parentId: parentIdentity(span) } : {},
1790
+ role: errorRoleForSpan(span),
1791
+ error: span.status?.code === "ERROR",
1792
+ processRoot: !span.parentSpanId && isTerminalRootCandidate(span)
1793
+ }))
1794
+ );
1795
+ rawNumeric.error_span_count = errorSummary.total;
1796
+ rawNumeric.execution_error_count = errorSummary.execution;
1797
+ rawNumeric.process_error_count = errorSummary.process;
1798
+ rawNumeric.guardrail_error_count = errorSummary.guardrail;
1799
+ rawNumeric.judge_error_count = errorSummary.evaluation;
1800
+ rawNumeric.propagated_error_count = errorSummary.propagated;
1801
+ rawNumeric.unclassified_error_count = errorSummary.unclassified;
1801
1802
  rawNumeric.llm_span_count = measurements.modelCallCount;
1802
1803
  if (measurements.cost.value !== void 0 && !measurements.cost.complete) {
1803
1804
  rawNumeric.partial_observed_cost_usd = measurements.cost.value;
@@ -1808,12 +1809,18 @@ function fromOtelSpans(opts) {
1808
1809
  perDimMean: { score },
1809
1810
  composite: score
1810
1811
  } : void 0;
1811
- const errorSpan = groupSpans2.find((s) => s.status?.code === "ERROR");
1812
+ const terminalOutcome = terminalOutcomeFromRoots(groupSpans2);
1813
+ const failedRoot = terminalOutcome === "failed" ? groupSpans2.find(
1814
+ (span) => !span.parentSpanId && isTerminalRootCandidate(span) && span.status?.code === "ERROR"
1815
+ ) : void 0;
1812
1816
  const outcome = {
1813
- ...opts.defaultSplit === "search" ? { searchScore: score } : { holdoutScore: score },
1814
1817
  raw: rawNumeric,
1815
1818
  ...judgeScores ? { judgeScores } : {}
1816
1819
  };
1820
+ if (score !== void 0) {
1821
+ if (defaultSplit === "holdout") outcome.holdoutScore = score;
1822
+ else outcome.searchScore = score;
1823
+ }
1817
1824
  runs.push({
1818
1825
  runId: groupKey,
1819
1826
  experimentId,
@@ -1827,22 +1834,55 @@ function fromOtelSpans(opts) {
1827
1834
  costUsd,
1828
1835
  costProvenance: capturedCost === void 0 ? { kind: "uncaptured", usd: null } : { kind: "observed", usd: capturedCost },
1829
1836
  tokenUsage: measurements.tokenUsage,
1837
+ terminalOutcome,
1838
+ ...failedRoot ? { terminalFailureReason: failedRoot.status?.message ?? failedRoot.name } : {},
1830
1839
  outcome,
1831
1840
  splitTag: defaultSplit,
1832
- ...errorSpan ? { failureMode: errorSpan.name } : {}
1841
+ scenarioId
1833
1842
  });
1834
1843
  }
1835
1844
  return runs;
1836
1845
  }
1846
+ function terminalOutcomeFromRoots(spans) {
1847
+ const roots = spans.filter((span) => !span.parentSpanId && isTerminalRootCandidate(span));
1848
+ if (roots.length !== 1) return "unknown";
1849
+ if (roots[0].status?.code === "OK") return "succeeded";
1850
+ if (roots[0].status?.code === "ERROR") return "failed";
1851
+ return "unknown";
1852
+ }
1853
+ function isTerminalRootCandidate(span) {
1854
+ const role = errorRoleForSpan(span);
1855
+ if (role === "LLM" || role === "TOOL" || role === "GUARDRAIL" || role === "EVALUATOR") {
1856
+ return false;
1857
+ }
1858
+ return true;
1859
+ }
1860
+ function readSpanKind(span) {
1861
+ return readAttrString([span], [...SPAN_KIND_ATTR_KEYS, "span.kind"])?.toUpperCase();
1862
+ }
1863
+ function errorRoleForSpan(span) {
1864
+ return classifyOtlpSpanRole({
1865
+ kind: readSpanKind(span),
1866
+ name: span.name,
1867
+ attributes: span.attributes
1868
+ });
1869
+ }
1870
+ function spanIdentity(span) {
1871
+ return `${span.traceId}:${span.spanId}`;
1872
+ }
1873
+ function parentIdentity(span) {
1874
+ return `${span.traceId}:${span.parentSpanId}`;
1875
+ }
1837
1876
  function isExplicitModelCall(span) {
1838
- const kind = span.attributes["openinference.span.kind"];
1839
- if (typeof kind === "string") return kind.toUpperCase() === "LLM";
1840
- const spanType = span.attributes["span.type"];
1841
- return typeof spanType === "string" && spanType.toLowerCase() === "llm_request" || span.name.toLowerCase().includes("llm") || readAttrString([span], MODEL_KEYS) !== void 0 || typeof span.attributes["gen_ai.operation.name"] === "string";
1877
+ return isOtlpModelCall({
1878
+ kind: readSpanKind(span),
1879
+ name: span.name,
1880
+ attributes: span.attributes
1881
+ });
1842
1882
  }
1843
1883
  function isExplicitAggregate(span) {
1844
- const kind = span.attributes["openinference.span.kind"];
1845
- return typeof kind === "string" && kind.toUpperCase() !== "LLM";
1884
+ const kind = readSpanKind(span);
1885
+ return kind !== void 0 && kind !== "LLM";
1846
1886
  }
1847
1887
  function groupSpans(spans) {
1848
1888
  const m = /* @__PURE__ */ new Map();
@@ -1855,7 +1895,10 @@ function groupSpans(spans) {
1855
1895
  return m;
1856
1896
  }
1857
1897
  function findRoot(group) {
1858
- return group.find((s) => !s.parentSpanId) ?? group[0];
1898
+ const structuralRoots = group.filter((span) => !span.parentSpanId);
1899
+ const terminalRoots = structuralRoots.filter(isTerminalRootCandidate);
1900
+ const pool = terminalRoots.length > 0 ? terminalRoots : structuralRoots.length > 0 ? structuralRoots : group;
1901
+ return orderSpans(pool)[0];
1859
1902
  }
1860
1903
  function readAttrString(spans, keys) {
1861
1904
  for (const span of spans) {
@@ -1866,18 +1909,102 @@ function readAttrString(spans, keys) {
1866
1909
  }
1867
1910
  return void 0;
1868
1911
  }
1869
- function readAttrNumber(spans, keys) {
1912
+ function readConsistentScenarioId(runId, spans) {
1913
+ const values = /* @__PURE__ */ new Set();
1870
1914
  for (const span of spans) {
1871
- for (const key of keys) {
1872
- const v = span.attributes[key];
1873
- if (typeof v === "number" && Number.isFinite(v)) return v;
1874
- if (typeof v === "string") {
1875
- const parsed = Number(v);
1876
- if (Number.isFinite(parsed)) return parsed;
1915
+ const topLevel = span["tangle.scenarioId"];
1916
+ if (typeof topLevel === "string" && topLevel.length > 0) values.add(topLevel);
1917
+ const attribute = span.attributes["tangle.scenarioId"];
1918
+ if (typeof attribute === "string" && attribute.length > 0) values.add(attribute);
1919
+ }
1920
+ if (values.size > 1) {
1921
+ throw new ValidationError(
1922
+ `fromOtelSpans: conflicting scenario ids for run '${runId}': ${[...values].sort().join(", ")}`
1923
+ );
1924
+ }
1925
+ return values.values().next().value;
1926
+ }
1927
+ function resolveTaskScore(runId, spans, scoreForRun) {
1928
+ const orderedSpans = orderSpans(spans);
1929
+ const sources = [];
1930
+ if (scoreForRun) {
1931
+ const supplied = scoreForRun(runId, orderedSpans);
1932
+ if (supplied !== void 0) {
1933
+ if (typeof supplied !== "number" || !Number.isFinite(supplied)) {
1934
+ throw new ValidationError(
1935
+ `fromOtelSpans: scoreForRun returned a non-finite number for run '${runId}'`
1936
+ );
1877
1937
  }
1938
+ sources.push({ label: "scoreForRun", value: supplied });
1878
1939
  }
1879
1940
  }
1880
- return void 0;
1941
+ for (const span of orderedSpans) {
1942
+ const role = errorRoleForSpan(span);
1943
+ if (span.parentSpanId && role !== "EVALUATOR") continue;
1944
+ if (role === "EVALUATOR" && span.status?.code === "ERROR") continue;
1945
+ for (const key of TASK_SCORE_ATTR_KEYS) {
1946
+ if (!Object.hasOwn(span.attributes, key)) continue;
1947
+ sources.push({
1948
+ label: `span '${span.spanId}' attribute '${key}'`,
1949
+ value: parseTaskScoreAttribute(runId, span.spanId, key, span.attributes[key])
1950
+ });
1951
+ }
1952
+ }
1953
+ if (sources.length === 0) return void 0;
1954
+ sources.sort((left, right) => left.label.localeCompare(right.label));
1955
+ const score = sources[0].value;
1956
+ if (sources.some((source) => source.value !== score)) {
1957
+ const details = sources.map((source) => `${source.label}=${source.value}`).join(", ");
1958
+ throw new ValidationError(
1959
+ `fromOtelSpans: conflicting task-quality scores for run '${runId}': ${details}`
1960
+ );
1961
+ }
1962
+ return score;
1963
+ }
1964
+ function parseTaskScoreAttribute(runId, spanId, key, value) {
1965
+ const source = `span '${spanId}' attribute '${key}'`;
1966
+ if (typeof value === "string") {
1967
+ if (value.trim().length === 0) {
1968
+ throw new ValidationError(
1969
+ `fromOtelSpans: ${source} is blank for run '${runId}'; task quality must be finite`
1970
+ );
1971
+ }
1972
+ const parsed = Number(value);
1973
+ if (Number.isFinite(parsed)) return parsed;
1974
+ } else if (typeof value === "number" && Number.isFinite(value)) {
1975
+ return value;
1976
+ }
1977
+ throw new ValidationError(
1978
+ `fromOtelSpans: ${source} is not a finite task-quality score for run '${runId}'`
1979
+ );
1980
+ }
1981
+ function orderSpans(spans) {
1982
+ return [...spans].sort(
1983
+ (left, right) => compareUnixNano(left.startTimeUnixNano, right.startTimeUnixNano) || left.spanId.localeCompare(right.spanId)
1984
+ );
1985
+ }
1986
+ function parseUnixNano(value) {
1987
+ try {
1988
+ return BigInt(value);
1989
+ } catch {
1990
+ throw new ValidationError(`fromOtelSpans: invalid Unix nanosecond timestamp '${value}'`);
1991
+ }
1992
+ }
1993
+ function compareUnixNano(left, right) {
1994
+ const leftValue = parseUnixNano(left);
1995
+ const rightValue = parseUnixNano(right);
1996
+ return leftValue < rightValue ? -1 : leftValue > rightValue ? 1 : 0;
1997
+ }
1998
+ function unixNanoDurationMs(start, end) {
1999
+ const delta = parseUnixNano(end) - parseUnixNano(start);
2000
+ if (delta <= 0n) return 0;
2001
+ const wholeMs = delta / 1000000n;
2002
+ const fractionalMs = delta % 1000000n;
2003
+ const value = Number(wholeMs) + Number(fractionalMs) / 1e6;
2004
+ if (!Number.isSafeInteger(Number(wholeMs))) {
2005
+ throw new ValidationError("fromOtelSpans: span duration exceeds the safe millisecond range");
2006
+ }
2007
+ return value;
1881
2008
  }
1882
2009
  function collectNumericAttrs(spans) {
1883
2010
  const raw = {};