@tangle-network/agent-eval 0.126.7 → 0.128.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. package/CHANGELOG.md +59 -0
  2. package/README.md +6 -2
  3. package/dist/analyst/index.d.ts +52 -43
  4. package/dist/analyst/index.js +4 -4
  5. package/dist/belief-state/index.d.ts +35 -40
  6. package/dist/belief-state/index.js +2 -2
  7. package/dist/benchmarks/index.d.ts +30 -12
  8. package/dist/benchmarks/index.js +12 -11
  9. package/dist/builder-eval/index.js +1 -1
  10. package/dist/campaign/index.d.ts +441 -160
  11. package/dist/campaign/index.js +11 -10
  12. package/dist/{chunk-IILEIWGW.js → chunk-2JX3CFMB.js} +100 -40
  13. package/dist/chunk-2JX3CFMB.js.map +1 -0
  14. package/dist/chunk-2MKQIFS4.js +183 -0
  15. package/dist/chunk-2MKQIFS4.js.map +1 -0
  16. package/dist/{chunk-NUKSVU3W.js → chunk-5DTSBUL2.js} +2 -2
  17. package/dist/{chunk-H5UD2323.js → chunk-7ZZMD7UK.js} +102 -2
  18. package/dist/chunk-7ZZMD7UK.js.map +1 -0
  19. package/dist/{chunk-474LBSOX.js → chunk-BYT7ELPS.js} +45 -18
  20. package/dist/chunk-BYT7ELPS.js.map +1 -0
  21. package/dist/{chunk-LUNF2SEL.js → chunk-DJKY2TSY.js} +3 -3
  22. package/dist/chunk-DPUHNQLN.js +232 -0
  23. package/dist/chunk-DPUHNQLN.js.map +1 -0
  24. package/dist/chunk-DRYIUNWY.js +622 -0
  25. package/dist/chunk-DRYIUNWY.js.map +1 -0
  26. package/dist/{chunk-IPYXE555.js → chunk-EJGRPCO3.js} +36 -13
  27. package/dist/chunk-EJGRPCO3.js.map +1 -0
  28. package/dist/{chunk-OCFJACJU.js → chunk-EOSZT7PL.js} +82 -18
  29. package/dist/chunk-EOSZT7PL.js.map +1 -0
  30. package/dist/{chunk-ZVCHKKOP.js → chunk-EZJEIH2R.js} +39 -15
  31. package/dist/chunk-EZJEIH2R.js.map +1 -0
  32. package/dist/{chunk-M7AH34KV.js → chunk-IHQDPH7D.js} +32 -16
  33. package/dist/chunk-IHQDPH7D.js.map +1 -0
  34. package/dist/{chunk-PJQFMIOX.js → chunk-MHELPNRP.js} +31 -1
  35. package/dist/chunk-MHELPNRP.js.map +1 -0
  36. package/dist/{chunk-K6IAZZ6L.js → chunk-NACAGYSY.js} +153 -122
  37. package/dist/chunk-NACAGYSY.js.map +1 -0
  38. package/dist/{chunk-BTACVO2W.js → chunk-NKAGIDE2.js} +542 -258
  39. package/dist/chunk-NKAGIDE2.js.map +1 -0
  40. package/dist/{chunk-ARU2PZFM.js → chunk-NYLOYM6N.js} +43 -23
  41. package/dist/chunk-NYLOYM6N.js.map +1 -0
  42. package/dist/{chunk-QOTFXW5L.js → chunk-P5W7RQKK.js} +2 -2
  43. package/dist/{chunk-IR3KBHOY.js → chunk-P6FYH6K4.js} +39 -3
  44. package/dist/chunk-P6FYH6K4.js.map +1 -0
  45. package/dist/{chunk-J5SQWP6Y.js → chunk-PBE2LOSS.js} +18 -2
  46. package/dist/chunk-PBE2LOSS.js.map +1 -0
  47. package/dist/{chunk-X4UCIOTZ.js → chunk-S5YLIBFX.js} +2 -2
  48. package/dist/{chunk-7VYEGHWF.js → chunk-SZLVEKMJ.js} +41 -20
  49. package/dist/chunk-SZLVEKMJ.js.map +1 -0
  50. package/dist/{chunk-P22LJ3Y2.js → chunk-TBL77AUT.js} +8 -8
  51. package/dist/chunk-TBL77AUT.js.map +1 -0
  52. package/dist/{chunk-LKKT3IVV.js → chunk-TSN7JT6D.js} +6 -4
  53. package/dist/{chunk-LKKT3IVV.js.map → chunk-TSN7JT6D.js.map} +1 -1
  54. package/dist/{chunk-E3IADPLG.js → chunk-UB2LOJ6Q.js} +51 -227
  55. package/dist/chunk-UB2LOJ6Q.js.map +1 -0
  56. package/dist/{chunk-MAX3TN3C.js → chunk-UWZZKKU7.js} +7 -19
  57. package/dist/chunk-UWZZKKU7.js.map +1 -0
  58. package/dist/{chunk-DPZAEKA6.js → chunk-VLOATJQ2.js} +85 -57
  59. package/dist/chunk-VLOATJQ2.js.map +1 -0
  60. package/dist/{chunk-MOXWMGPC.js → chunk-XDWDC2MP.js} +197 -79
  61. package/dist/chunk-XDWDC2MP.js.map +1 -0
  62. package/dist/{chunk-CGG5SLH3.js → chunk-XPRT64IE.js} +2 -2
  63. package/dist/{chunk-FO7HEH76.js → chunk-YJBNWCAA.js} +2 -2
  64. package/dist/{chunk-KE2VWPZX.js → chunk-ZUUWPZCV.js} +3 -3
  65. package/dist/cli.js +4 -4
  66. package/dist/contract/index.d.ts +438 -138
  67. package/dist/contract/index.js +259 -113
  68. package/dist/contract/index.js.map +1 -1
  69. package/dist/control.d.ts +56 -39
  70. package/dist/control.js +2 -3
  71. package/dist/hosted/index.d.ts +152 -51
  72. package/dist/hosted/index.js +24 -1
  73. package/dist/index.d.ts +399 -277
  74. package/dist/index.js +363 -244
  75. package/dist/index.js.map +1 -1
  76. package/dist/meta-eval/index.d.ts +33 -36
  77. package/dist/meta-eval/index.js +2 -2
  78. package/dist/multishot/index.d.ts +1 -0
  79. package/dist/openapi.json +1 -1
  80. package/dist/pipelines/index.js +2 -2
  81. package/dist/reporting.d.ts +101 -80
  82. package/dist/reporting.js +7 -5
  83. package/dist/rl.d.ts +198 -142
  84. package/dist/rl.js +235 -127
  85. package/dist/rl.js.map +1 -1
  86. package/dist/rollout/index.d.ts +54 -67
  87. package/dist/rollout/index.js +6 -5
  88. package/dist/{run-campaign-FFRM3RH5.js → run-campaign-ISHFZ7FJ.js} +6 -3
  89. package/dist/supervisor-run/index.d.ts +7 -15
  90. package/dist/supervisor-run/index.js +2 -2
  91. package/dist/traces.d.ts +72 -50
  92. package/dist/traces.js +9 -6
  93. package/dist/wire/index.d.ts +1 -0
  94. package/dist/wire/index.js +2 -2
  95. package/docs/concepts.md +18 -1
  96. package/docs/customer-journeys.md +7 -6
  97. package/docs/hosted-ingest-spec.md +37 -40
  98. package/docs/insight-report.md +78 -7
  99. package/docs/research-report-methodology.md +20 -21
  100. package/docs/rollout.md +13 -8
  101. package/package.json +4 -3
  102. package/dist/chunk-474LBSOX.js.map +0 -1
  103. package/dist/chunk-7VYEGHWF.js.map +0 -1
  104. package/dist/chunk-ARU2PZFM.js.map +0 -1
  105. package/dist/chunk-BTACVO2W.js.map +0 -1
  106. package/dist/chunk-DPZAEKA6.js.map +0 -1
  107. package/dist/chunk-E3IADPLG.js.map +0 -1
  108. package/dist/chunk-H5UD2323.js.map +0 -1
  109. package/dist/chunk-IILEIWGW.js.map +0 -1
  110. package/dist/chunk-IPYXE555.js.map +0 -1
  111. package/dist/chunk-IR3KBHOY.js.map +0 -1
  112. package/dist/chunk-J5SQWP6Y.js.map +0 -1
  113. package/dist/chunk-K6IAZZ6L.js.map +0 -1
  114. package/dist/chunk-M7AH34KV.js.map +0 -1
  115. package/dist/chunk-MAX3TN3C.js.map +0 -1
  116. package/dist/chunk-MOXWMGPC.js.map +0 -1
  117. package/dist/chunk-OCFJACJU.js.map +0 -1
  118. package/dist/chunk-P22LJ3Y2.js.map +0 -1
  119. package/dist/chunk-PJQFMIOX.js.map +0 -1
  120. package/dist/chunk-VSMTAMNK.js +0 -53
  121. package/dist/chunk-VSMTAMNK.js.map +0 -1
  122. package/dist/chunk-ZVCHKKOP.js.map +0 -1
  123. package/dist/chunk-ZZUXHH3R.js +0 -99
  124. package/dist/chunk-ZZUXHH3R.js.map +0 -1
  125. /package/dist/{chunk-NUKSVU3W.js.map → chunk-5DTSBUL2.js.map} +0 -0
  126. /package/dist/{chunk-LUNF2SEL.js.map → chunk-DJKY2TSY.js.map} +0 -0
  127. /package/dist/{chunk-QOTFXW5L.js.map → chunk-P5W7RQKK.js.map} +0 -0
  128. /package/dist/{chunk-X4UCIOTZ.js.map → chunk-S5YLIBFX.js.map} +0 -0
  129. /package/dist/{chunk-CGG5SLH3.js.map → chunk-XPRT64IE.js.map} +0 -0
  130. /package/dist/{chunk-FO7HEH76.js.map → chunk-YJBNWCAA.js.map} +0 -0
  131. /package/dist/{chunk-KE2VWPZX.js.map → chunk-ZUUWPZCV.js.map} +0 -0
  132. /package/dist/{run-campaign-FFRM3RH5.js.map → run-campaign-ISHFZ7FJ.js.map} +0 -0
@@ -7,14 +7,14 @@ import {
7
7
  fromPigraphSession,
8
8
  observeCodeAgentSession,
9
9
  parseCodeAgentJsonl
10
- } from "../chunk-7VYEGHWF.js";
10
+ } from "../chunk-SZLVEKMJ.js";
11
11
  import {
12
12
  createHostedClient
13
- } from "../chunk-ZZUXHH3R.js";
13
+ } from "../chunk-DRYIUNWY.js";
14
14
  import {
15
15
  analyzeRuns,
16
16
  summarizeExecution
17
- } from "../chunk-K6IAZZ6L.js";
17
+ } from "../chunk-NACAGYSY.js";
18
18
  import {
19
19
  REFERENCE_EQUIVALENCE_INPUT_LIMITS,
20
20
  REFERENCE_EQUIVALENCE_JUDGE_VERSION,
@@ -40,7 +40,7 @@ import {
40
40
  skillOptOptimizationMethod,
41
41
  surfaceContentHash,
42
42
  surfaceHash
43
- } from "../chunk-BTACVO2W.js";
43
+ } from "../chunk-NKAGIDE2.js";
44
44
  import {
45
45
  campaignSplitDigest,
46
46
  createRunCostLedger,
@@ -48,47 +48,61 @@ import {
48
48
  inMemoryCampaignStorage,
49
49
  resolveRunDir,
50
50
  runCampaign
51
- } from "../chunk-ZVCHKKOP.js";
51
+ } from "../chunk-EZJEIH2R.js";
52
52
  import {
53
53
  buildDefaultAnalystRegistry,
54
54
  createChatClient
55
- } from "../chunk-LUNF2SEL.js";
55
+ } from "../chunk-DJKY2TSY.js";
56
56
  import "../chunk-HHWE3POT.js";
57
57
  import "../chunk-WGXIEX7P.js";
58
58
  import {
59
59
  FileSystemOutcomeStore,
60
60
  InMemoryOutcomeStore
61
61
  } from "../chunk-3RF76KTD.js";
62
- import "../chunk-ARU2PZFM.js";
63
- import "../chunk-J5SQWP6Y.js";
64
- import "../chunk-DPZAEKA6.js";
62
+ import "../chunk-NYLOYM6N.js";
63
+ import {
64
+ campaignCellExecutionEvidence,
65
+ campaignCellJudgeDimensions,
66
+ campaignCellTaskScore,
67
+ campaignCellToRunRecord
68
+ } from "../chunk-2MKQIFS4.js";
69
+ import "../chunk-PBE2LOSS.js";
70
+ import "../chunk-VLOATJQ2.js";
71
+ import "../chunk-DPUHNQLN.js";
65
72
  import {
66
73
  pairedBootstrap
67
- } from "../chunk-PJQFMIOX.js";
74
+ } from "../chunk-MHELPNRP.js";
68
75
  import "../chunk-WS3NZZQQ.js";
69
76
  import "../chunk-VI2UW6B6.js";
70
77
  import {
78
+ readTaskFailureLabels,
71
79
  recordAggregateMeasurements,
72
- summarizeExecutionMeasurements
73
- } from "../chunk-H5UD2323.js";
80
+ summarizeExecutionMeasurements,
81
+ summarizeTraceErrors
82
+ } from "../chunk-7ZZMD7UK.js";
74
83
  import "../chunk-PXE2VKMX.js";
75
84
  import "../chunk-ZET2UAYW.js";
76
85
  import "../chunk-GGE4NNQT.js";
77
- import "../chunk-IR3KBHOY.js";
86
+ import {
87
+ classifyOtlpSpanRole,
88
+ isOtlpModelCall
89
+ } from "../chunk-P6FYH6K4.js";
78
90
  import "../chunk-PC4UYEBM.js";
79
91
  import {
92
+ modelHasSnapshot,
80
93
  parseRunRecordSafe
81
- } from "../chunk-IILEIWGW.js";
94
+ } from "../chunk-2JX3CFMB.js";
82
95
  import "../chunk-MA6HLL3S.js";
83
- import "../chunk-VSMTAMNK.js";
84
- import "../chunk-ONWEPEDO.js";
85
96
  import {
86
- LLM_MODEL_ATTR_KEYS
97
+ ValidationError
98
+ } from "../chunk-ONWEPEDO.js";
99
+ import {
100
+ LLM_MODEL_ATTR_KEYS,
101
+ SPAN_KIND_ATTR_KEYS
87
102
  } from "../chunk-K4DBDHLK.js";
88
103
  import "../chunk-PZ5AY32C.js";
89
104
 
90
105
  // src/contract/self-improve.ts
91
- import { createHash } from "crypto";
92
106
  var SelfImproveRunError = class extends Error {
93
107
  cost;
94
108
  receipts;
@@ -343,11 +357,11 @@ async function runSelfImprove(opts, costLedger, startedAt, runDir, storage) {
343
357
  findings: opts.findings,
344
358
  selectionRankKey: opts.selectionRankKey
345
359
  });
346
- const winnerSearch = holdoutDeferred ? winnerSearchCampaign(result) : void 0;
347
- const baseline = meanComposite(
348
- (holdoutDeferred ? result.baselineCampaign : result.baselineOnHoldout).aggregates.byScenario
349
- );
350
- const winnerStats = meanComposite((winnerSearch ?? result.winnerOnHoldout).aggregates.byScenario);
360
+ const reportSplit = holdoutDeferred ? "search" : "holdout";
361
+ const reportBaselineCampaign = holdoutDeferred ? result.baselineCampaign : result.baselineOnHoldout;
362
+ const reportWinnerCampaign = holdoutDeferred ? winnerSearchCampaign(result) : result.winnerOnHoldout;
363
+ const baseline = meanComposite(reportBaselineCampaign.aggregates.byScenario);
364
+ const winnerStats = meanComposite(reportWinnerCampaign.aggregates.byScenario);
351
365
  let power;
352
366
  const baselineHoldoutComposites = result.baselineOnHoldout.cells.filter((cell) => !cell.error).map((cell) => {
353
367
  const scores = Object.values(cell.judgeScores);
@@ -390,12 +404,21 @@ async function runSelfImprove(opts, costLedger, startedAt, runDir, storage) {
390
404
  const totalCost = cost.totalCostUsd;
391
405
  const insight = await analyzeRuns({
392
406
  runs: [
393
- ...cellsToRunRecords(result.baselineCampaign.cells, "baseline", runDir, opts.baselineSurface),
394
407
  ...cellsToRunRecords(
395
- (winnerSearch ?? result.winnerOnHoldout).cells,
408
+ reportBaselineCampaign.cells,
409
+ "baseline",
410
+ runDir,
411
+ opts.baselineSurface,
412
+ reportSplit,
413
+ opts.model
414
+ ),
415
+ ...cellsToRunRecords(
416
+ reportWinnerCampaign.cells,
396
417
  "winner",
397
418
  runDir,
398
- result.winnerSurface
419
+ result.winnerSurface,
420
+ reportSplit,
421
+ opts.model
399
422
  )
400
423
  ],
401
424
  baselineCandidateId: "baseline",
@@ -468,19 +491,21 @@ async function shipEvalRunToHosted(tenant, opts, summary, raw, runDir) {
468
491
  const client = createHostedClient(tenant);
469
492
  function snapshotFromCampaign(index, surface, campaign, durationMs) {
470
493
  const cells = campaign.cells.map((cell) => {
471
- const judgeScores = Object.values(cell.judgeScores);
472
- const composite = judgeScores.length === 0 ? 0 : judgeScores.reduce((s, j) => s + j.composite, 0) / judgeScores.length;
494
+ const execution = campaignCellExecutionEvidence(cell);
473
495
  return {
474
496
  scenarioId: cell.scenarioId,
475
497
  rep: cell.rep,
476
- compositeMean: composite,
477
- dimensions: Object.fromEntries(
478
- Object.entries(cell.judgeScores).map(([name, score]) => [name, score.dimensions])
479
- ),
498
+ compositeMean: campaignCellTaskScore(cell) ?? null,
499
+ dimensions: campaignCellJudgeDimensions(cell),
500
+ terminalOutcome: execution.terminalOutcome,
501
+ executionErrorCount: execution.executionErrorCount ?? null,
480
502
  errorMessage: cell.error ?? void 0
481
503
  };
482
504
  });
483
- const compositeMean = cells.length === 0 ? 0 : cells.reduce((s, c) => s + c.compositeMean, 0) / cells.length;
505
+ const scoredCells = cells.flatMap(
506
+ (cell) => cell.compositeMean === null ? [] : [cell.compositeMean]
507
+ );
508
+ const compositeMean = scoredCells.length === 0 ? null : scoredCells.reduce((sum, score) => sum + score, 0) / scoredCells.length;
484
509
  return {
485
510
  index,
486
511
  surfaceHash: surfaceHash(surface),
@@ -531,66 +556,33 @@ function hashString(s) {
531
556
  }
532
557
  return h.toString(16).padStart(8, "0");
533
558
  }
534
- function cellsToRunRecords(cells, candidateId, runId, surface) {
559
+ function cellsToRunRecords(cells, candidateId, runId, surface, splitTag, fallbackModel) {
535
560
  const promptHash = surfaceContentHash(surface);
536
- const configHash = `sha256:${createHash("sha256").update(candidateId).digest("hex")}`;
561
+ const configHash = surfaceContentHash(candidateId);
537
562
  return cells.map((cell) => {
538
- const perJudge = {};
539
- const perDimMeanAccum = {};
540
- let compositeSum = 0;
541
- let compositeCount = 0;
542
- for (const [judgeId, score] of Object.entries(cell.judgeScores)) {
543
- perJudge[judgeId] = { ...score.dimensions };
544
- for (const [dim, value] of Object.entries(score.dimensions)) {
545
- if (!Number.isFinite(value)) continue;
546
- const accum = perDimMeanAccum[dim] ?? { sum: 0, n: 0 };
547
- accum.sum += value;
548
- accum.n += 1;
549
- perDimMeanAccum[dim] = accum;
550
- }
551
- if (Number.isFinite(score.composite)) {
552
- compositeSum += score.composite;
553
- compositeCount += 1;
554
- }
563
+ const model = cell.resolvedModel ?? fallbackModel;
564
+ if (!model) {
565
+ throw new ValidationError(
566
+ `selfImprove.model is required when cell ${cell.cellId} has no paid-call model receipt`
567
+ );
555
568
  }
556
- const perDimMean = {};
557
- for (const [dim, { sum, n }] of Object.entries(perDimMeanAccum)) {
558
- perDimMean[dim] = n === 0 ? 0 : sum / n;
569
+ if (!modelHasSnapshot(model)) {
570
+ throw new ValidationError(
571
+ `selfImprove model "${model}" lacks a snapshot version for cell ${cell.cellId}`
572
+ );
559
573
  }
560
- const composite = compositeCount === 0 ? 0 : compositeSum / compositeCount;
561
- const judgeScores = {
562
- perJudge,
563
- perDimMean,
564
- composite
565
- };
566
- return {
574
+ return campaignCellToRunRecord(cell, {
567
575
  runId: `${runId}::${candidateId}::${cell.cellId}`,
568
576
  experimentId: runId,
569
577
  candidateId,
570
- // Pair on (scenarioId, rep) analyzeRuns pairs on (experimentId, seed).
571
- // Synthesize a stable seed for that pairing.
578
+ // scenarioId is explicit; seed keeps repeated runs distinct.
572
579
  seed: cell.rep * 1e6 + hashString(cell.scenarioId).slice(0, 6).split("").reduce((a, c) => a * 31 + c.charCodeAt(0) >>> 0, 0),
573
- model: "campaign-cell",
580
+ model,
574
581
  promptHash,
575
582
  configHash,
576
583
  commitSha: "cell",
577
- wallMs: cell.durationMs,
578
- costUsd: cell.costUsd,
579
- tokenUsage: {
580
- input: cell.tokenUsage.input,
581
- output: cell.tokenUsage.output,
582
- ...cell.tokenUsage.reasoning === void 0 ? {} : { reasoning: cell.tokenUsage.reasoning },
583
- ...cell.tokenUsage.cached === void 0 ? {} : { cached: cell.tokenUsage.cached },
584
- ...cell.tokenUsage.cacheWrite === void 0 ? {} : { cacheWrite: cell.tokenUsage.cacheWrite }
585
- },
586
- outcome: {
587
- holdoutScore: composite,
588
- raw: {},
589
- judgeScores
590
- },
591
- splitTag: "holdout",
592
- ...cell.error ? { failureMode: cell.error } : {}
593
- };
584
+ splitTag
585
+ });
594
586
  });
595
587
  }
596
588
 
@@ -1565,7 +1557,7 @@ function diffGenerations(before, after) {
1565
1557
  rep: beforeCell.rep,
1566
1558
  compositeBefore: beforeCell.compositeMean,
1567
1559
  compositeAfter: afterCell.compositeMean,
1568
- compositeDelta: afterCell.compositeMean - beforeCell.compositeMean,
1560
+ compositeDelta: beforeCell.compositeMean === null || afterCell.compositeMean === null ? null : afterCell.compositeMean - beforeCell.compositeMean,
1569
1561
  dimensions: diffDimensions(beforeCell.dimensions, afterCell.dimensions)
1570
1562
  });
1571
1563
  }
@@ -1583,7 +1575,7 @@ function diffGenerations(before, after) {
1583
1575
  added,
1584
1576
  compositeBefore: before.compositeMean,
1585
1577
  compositeAfter: after.compositeMean,
1586
- compositeDelta: after.compositeMean - before.compositeMean,
1578
+ compositeDelta: before.compositeMean === null || after.compositeMean === null ? null : after.compositeMean - before.compositeMean,
1587
1579
  costUsdBefore: before.costUsd,
1588
1580
  costUsdAfter: after.costUsd,
1589
1581
  costUsdDelta: after.costUsd - before.costUsd,
@@ -1736,14 +1728,13 @@ function fromFeedbackTable(opts) {
1736
1728
  perDimMean: { rating: meanScore },
1737
1729
  composite: meanScore
1738
1730
  };
1731
+ const splitTag = runMeta.splitTag ?? "holdout";
1739
1732
  const outcome = {
1740
- // Feedback corpora ARE the holdout signal score lands on
1741
- // `holdoutScore` so downstream substrate primitives (`paretoChart`,
1742
- // promotion gates) read it correctly by default.
1743
- holdoutScore: meanScore,
1733
+ ...splitTag === "holdout" ? { holdoutScore: meanScore } : { searchScore: meanScore },
1744
1734
  raw: Object.fromEntries(normalised.map((r) => [`rater:${r.rater}`, r.score])),
1745
1735
  judgeScores
1746
1736
  };
1737
+ const costUsd = runMeta.costUsd ?? null;
1747
1738
  runs.push({
1748
1739
  runId,
1749
1740
  experimentId: runMeta.experimentId ?? "feedback-corpus",
@@ -1754,10 +1745,13 @@ function fromFeedbackTable(opts) {
1754
1745
  configHash: runMeta.configHash ?? "sha256:unknown",
1755
1746
  commitSha: runMeta.commitSha ?? "unknown",
1756
1747
  wallMs: runMeta.wallMs ?? 0,
1757
- costUsd: runMeta.costUsd ?? 0,
1748
+ costUsd,
1749
+ costProvenance: costUsd === null ? { kind: "uncaptured", usd: null } : { kind: "observed", usd: costUsd },
1758
1750
  tokenUsage: { input: 0, output: 0 },
1751
+ terminalOutcome: "unknown",
1759
1752
  outcome,
1760
- splitTag: runMeta.splitTag ?? "holdout"
1753
+ splitTag,
1754
+ scenarioId: runMeta.scenarioId ?? runId
1761
1755
  });
1762
1756
  if (emitRaterScores) {
1763
1757
  for (const r of normalised) raterScores.push({ runId, rater: r.rater, score: r.score });
@@ -1767,7 +1761,12 @@ function fromFeedbackTable(opts) {
1767
1761
  }
1768
1762
 
1769
1763
  // src/contract/intake/otel-spans.ts
1770
- var SCORE_KEYS = ["tangle.score", "eval.score", "score"];
1764
+ var TASK_SCORE_ATTR_KEYS = [
1765
+ "gen_ai.evaluation.score.value",
1766
+ "tangle.task.score",
1767
+ "eval.score",
1768
+ "tangle.score"
1769
+ ];
1771
1770
  var MODEL_KEYS = ["tangle.model", ...LLM_MODEL_ATTR_KEYS, "model"];
1772
1771
  var PROMPT_HASH_KEYS = ["tangle.prompt_hash", "prompt.hash"];
1773
1772
  var CONFIG_HASH_KEYS = ["tangle.config_hash", "config.hash"];
@@ -1789,15 +1788,35 @@ function fromOtelSpans(opts) {
1789
1788
  );
1790
1789
  const callSpanIds = new Set(measurements.callSpanIds);
1791
1790
  const callSpans = groupSpans2.filter((span) => callSpanIds.has(span.spanId));
1792
- const wallMs = Math.max(0, (root.endTimeUnixNano - root.startTimeUnixNano) / 1e6);
1791
+ const wallMs = unixNanoDurationMs(root.startTimeUnixNano, root.endTimeUnixNano);
1793
1792
  const model = readAttrString(callSpans, MODEL_KEYS) ?? readAttrString(groupSpans2, MODEL_KEYS) ?? "unknown@unknown";
1794
1793
  const capturedCost = (measurements.cost.complete ? measurements.cost.value : void 0) ?? measurements.aggregate?.costUsd;
1795
- const costUsd = capturedCost ?? 0;
1794
+ const costUsd = capturedCost ?? null;
1795
+ const scenarioId = readConsistentScenarioId(groupKey, groupSpans2) ?? groupKey;
1796
1796
  const promptHash = readAttrString(groupSpans2, PROMPT_HASH_KEYS) ?? "sha256:unknown";
1797
1797
  const configHash = readAttrString(groupSpans2, CONFIG_HASH_KEYS) ?? "sha256:unknown";
1798
- const score = readAttrNumber(groupSpans2, SCORE_KEYS);
1798
+ const score = resolveTaskScore(groupKey, groupSpans2, opts.scoreForRun);
1799
+ const taskFailure = readTaskFailureLabels(
1800
+ groupSpans2.filter((span) => !span.parentSpanId && isTerminalRootCandidate(span)),
1801
+ `fromOtelSpans: run '${groupKey}'`
1802
+ );
1799
1803
  const rawNumeric = collectNumericAttrs(groupSpans2);
1800
- rawNumeric.error_span_count = groupSpans2.filter((span) => span.status?.code === "ERROR").length;
1804
+ const errorSummary = summarizeTraceErrors(
1805
+ groupSpans2.map((span) => ({
1806
+ id: spanIdentity(span),
1807
+ ...span.parentSpanId ? { parentId: parentIdentity(span) } : {},
1808
+ role: errorRoleForSpan(span),
1809
+ error: span.status?.code === "ERROR",
1810
+ processRoot: !span.parentSpanId && isTerminalRootCandidate(span)
1811
+ }))
1812
+ );
1813
+ rawNumeric.error_span_count = errorSummary.total;
1814
+ rawNumeric.execution_error_count = errorSummary.execution;
1815
+ rawNumeric.process_error_count = errorSummary.process;
1816
+ rawNumeric.guardrail_error_count = errorSummary.guardrail;
1817
+ rawNumeric.judge_error_count = errorSummary.evaluation;
1818
+ rawNumeric.propagated_error_count = errorSummary.propagated;
1819
+ rawNumeric.unclassified_error_count = errorSummary.unclassified;
1801
1820
  rawNumeric.llm_span_count = measurements.modelCallCount;
1802
1821
  if (measurements.cost.value !== void 0 && !measurements.cost.complete) {
1803
1822
  rawNumeric.partial_observed_cost_usd = measurements.cost.value;
@@ -1808,12 +1827,18 @@ function fromOtelSpans(opts) {
1808
1827
  perDimMean: { score },
1809
1828
  composite: score
1810
1829
  } : void 0;
1811
- const errorSpan = groupSpans2.find((s) => s.status?.code === "ERROR");
1830
+ const terminalOutcome = terminalOutcomeFromRoots(groupSpans2);
1831
+ const failedRoot = terminalOutcome === "failed" ? groupSpans2.find(
1832
+ (span) => !span.parentSpanId && isTerminalRootCandidate(span) && span.status?.code === "ERROR"
1833
+ ) : void 0;
1812
1834
  const outcome = {
1813
- ...opts.defaultSplit === "search" ? { searchScore: score } : { holdoutScore: score },
1814
1835
  raw: rawNumeric,
1815
1836
  ...judgeScores ? { judgeScores } : {}
1816
1837
  };
1838
+ if (score !== void 0) {
1839
+ if (defaultSplit === "holdout") outcome.holdoutScore = score;
1840
+ else outcome.searchScore = score;
1841
+ }
1817
1842
  runs.push({
1818
1843
  runId: groupKey,
1819
1844
  experimentId,
@@ -1827,22 +1852,56 @@ function fromOtelSpans(opts) {
1827
1852
  costUsd,
1828
1853
  costProvenance: capturedCost === void 0 ? { kind: "uncaptured", usd: null } : { kind: "observed", usd: capturedCost },
1829
1854
  tokenUsage: measurements.tokenUsage,
1855
+ terminalOutcome,
1856
+ ...failedRoot ? { terminalFailureReason: failedRoot.status?.message ?? failedRoot.name } : {},
1830
1857
  outcome,
1858
+ ...taskFailure,
1831
1859
  splitTag: defaultSplit,
1832
- ...errorSpan ? { failureMode: errorSpan.name } : {}
1860
+ scenarioId
1833
1861
  });
1834
1862
  }
1835
1863
  return runs;
1836
1864
  }
1865
+ function terminalOutcomeFromRoots(spans) {
1866
+ const roots = spans.filter((span) => !span.parentSpanId && isTerminalRootCandidate(span));
1867
+ if (roots.length !== 1) return "unknown";
1868
+ if (roots[0].status?.code === "OK") return "succeeded";
1869
+ if (roots[0].status?.code === "ERROR") return "failed";
1870
+ return "unknown";
1871
+ }
1872
+ function isTerminalRootCandidate(span) {
1873
+ const role = errorRoleForSpan(span);
1874
+ if (role === "LLM" || role === "TOOL" || role === "GUARDRAIL" || role === "EVALUATOR") {
1875
+ return false;
1876
+ }
1877
+ return true;
1878
+ }
1879
+ function readSpanKind(span) {
1880
+ return readAttrString([span], [...SPAN_KIND_ATTR_KEYS, "span.kind"])?.toUpperCase();
1881
+ }
1882
+ function errorRoleForSpan(span) {
1883
+ return classifyOtlpSpanRole({
1884
+ kind: readSpanKind(span),
1885
+ name: span.name,
1886
+ attributes: span.attributes
1887
+ });
1888
+ }
1889
+ function spanIdentity(span) {
1890
+ return `${span.traceId}:${span.spanId}`;
1891
+ }
1892
+ function parentIdentity(span) {
1893
+ return `${span.traceId}:${span.parentSpanId}`;
1894
+ }
1837
1895
  function isExplicitModelCall(span) {
1838
- const kind = span.attributes["openinference.span.kind"];
1839
- if (typeof kind === "string") return kind.toUpperCase() === "LLM";
1840
- const spanType = span.attributes["span.type"];
1841
- return typeof spanType === "string" && spanType.toLowerCase() === "llm_request" || span.name.toLowerCase().includes("llm") || readAttrString([span], MODEL_KEYS) !== void 0 || typeof span.attributes["gen_ai.operation.name"] === "string";
1896
+ return isOtlpModelCall({
1897
+ kind: readSpanKind(span),
1898
+ name: span.name,
1899
+ attributes: span.attributes
1900
+ });
1842
1901
  }
1843
1902
  function isExplicitAggregate(span) {
1844
- const kind = span.attributes["openinference.span.kind"];
1845
- return typeof kind === "string" && kind.toUpperCase() !== "LLM";
1903
+ const kind = readSpanKind(span);
1904
+ return kind !== void 0 && kind !== "LLM";
1846
1905
  }
1847
1906
  function groupSpans(spans) {
1848
1907
  const m = /* @__PURE__ */ new Map();
@@ -1855,7 +1914,10 @@ function groupSpans(spans) {
1855
1914
  return m;
1856
1915
  }
1857
1916
  function findRoot(group) {
1858
- return group.find((s) => !s.parentSpanId) ?? group[0];
1917
+ const structuralRoots = group.filter((span) => !span.parentSpanId);
1918
+ const terminalRoots = structuralRoots.filter(isTerminalRootCandidate);
1919
+ const pool = terminalRoots.length > 0 ? terminalRoots : structuralRoots.length > 0 ? structuralRoots : group;
1920
+ return orderSpans(pool)[0];
1859
1921
  }
1860
1922
  function readAttrString(spans, keys) {
1861
1923
  for (const span of spans) {
@@ -1866,18 +1928,102 @@ function readAttrString(spans, keys) {
1866
1928
  }
1867
1929
  return void 0;
1868
1930
  }
1869
- function readAttrNumber(spans, keys) {
1931
+ function readConsistentScenarioId(runId, spans) {
1932
+ const values = /* @__PURE__ */ new Set();
1870
1933
  for (const span of spans) {
1871
- for (const key of keys) {
1872
- const v = span.attributes[key];
1873
- if (typeof v === "number" && Number.isFinite(v)) return v;
1874
- if (typeof v === "string") {
1875
- const parsed = Number(v);
1876
- if (Number.isFinite(parsed)) return parsed;
1934
+ const topLevel = span["tangle.scenarioId"];
1935
+ if (typeof topLevel === "string" && topLevel.length > 0) values.add(topLevel);
1936
+ const attribute = span.attributes["tangle.scenarioId"];
1937
+ if (typeof attribute === "string" && attribute.length > 0) values.add(attribute);
1938
+ }
1939
+ if (values.size > 1) {
1940
+ throw new ValidationError(
1941
+ `fromOtelSpans: conflicting scenario ids for run '${runId}': ${[...values].sort().join(", ")}`
1942
+ );
1943
+ }
1944
+ return values.values().next().value;
1945
+ }
1946
+ function resolveTaskScore(runId, spans, scoreForRun) {
1947
+ const orderedSpans = orderSpans(spans);
1948
+ const sources = [];
1949
+ if (scoreForRun) {
1950
+ const supplied = scoreForRun(runId, orderedSpans);
1951
+ if (supplied !== void 0) {
1952
+ if (typeof supplied !== "number" || !Number.isFinite(supplied)) {
1953
+ throw new ValidationError(
1954
+ `fromOtelSpans: scoreForRun returned a non-finite number for run '${runId}'`
1955
+ );
1877
1956
  }
1957
+ sources.push({ label: "scoreForRun", value: supplied });
1878
1958
  }
1879
1959
  }
1880
- return void 0;
1960
+ for (const span of orderedSpans) {
1961
+ const role = errorRoleForSpan(span);
1962
+ if (span.parentSpanId && role !== "EVALUATOR") continue;
1963
+ if (role === "EVALUATOR" && span.status?.code === "ERROR") continue;
1964
+ for (const key of TASK_SCORE_ATTR_KEYS) {
1965
+ if (!Object.hasOwn(span.attributes, key)) continue;
1966
+ sources.push({
1967
+ label: `span '${span.spanId}' attribute '${key}'`,
1968
+ value: parseTaskScoreAttribute(runId, span.spanId, key, span.attributes[key])
1969
+ });
1970
+ }
1971
+ }
1972
+ if (sources.length === 0) return void 0;
1973
+ sources.sort((left, right) => left.label.localeCompare(right.label));
1974
+ const score = sources[0].value;
1975
+ if (sources.some((source) => source.value !== score)) {
1976
+ const details = sources.map((source) => `${source.label}=${source.value}`).join(", ");
1977
+ throw new ValidationError(
1978
+ `fromOtelSpans: conflicting task-quality scores for run '${runId}': ${details}`
1979
+ );
1980
+ }
1981
+ return score;
1982
+ }
1983
+ function parseTaskScoreAttribute(runId, spanId, key, value) {
1984
+ const source = `span '${spanId}' attribute '${key}'`;
1985
+ if (typeof value === "string") {
1986
+ if (value.trim().length === 0) {
1987
+ throw new ValidationError(
1988
+ `fromOtelSpans: ${source} is blank for run '${runId}'; task quality must be finite`
1989
+ );
1990
+ }
1991
+ const parsed = Number(value);
1992
+ if (Number.isFinite(parsed)) return parsed;
1993
+ } else if (typeof value === "number" && Number.isFinite(value)) {
1994
+ return value;
1995
+ }
1996
+ throw new ValidationError(
1997
+ `fromOtelSpans: ${source} is not a finite task-quality score for run '${runId}'`
1998
+ );
1999
+ }
2000
+ function orderSpans(spans) {
2001
+ return [...spans].sort(
2002
+ (left, right) => compareUnixNano(left.startTimeUnixNano, right.startTimeUnixNano) || left.spanId.localeCompare(right.spanId)
2003
+ );
2004
+ }
2005
+ function parseUnixNano(value) {
2006
+ try {
2007
+ return BigInt(value);
2008
+ } catch {
2009
+ throw new ValidationError(`fromOtelSpans: invalid Unix nanosecond timestamp '${value}'`);
2010
+ }
2011
+ }
2012
+ function compareUnixNano(left, right) {
2013
+ const leftValue = parseUnixNano(left);
2014
+ const rightValue = parseUnixNano(right);
2015
+ return leftValue < rightValue ? -1 : leftValue > rightValue ? 1 : 0;
2016
+ }
2017
+ function unixNanoDurationMs(start, end) {
2018
+ const delta = parseUnixNano(end) - parseUnixNano(start);
2019
+ if (delta <= 0n) return 0;
2020
+ const wholeMs = delta / 1000000n;
2021
+ const fractionalMs = delta % 1000000n;
2022
+ const value = Number(wholeMs) + Number(fractionalMs) / 1e6;
2023
+ if (!Number.isSafeInteger(Number(wholeMs))) {
2024
+ throw new ValidationError("fromOtelSpans: span duration exceeds the safe millisecond range");
2025
+ }
2026
+ return value;
1881
2027
  }
1882
2028
  function collectNumericAttrs(spans) {
1883
2029
  const raw = {};