@tangle-network/agent-eval 0.126.7 → 0.127.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +44 -0
- package/README.md +2 -2
- package/dist/analyst/index.d.ts +51 -39
- package/dist/analyst/index.js +4 -4
- package/dist/belief-state/index.d.ts +34 -36
- package/dist/belief-state/index.js +2 -2
- package/dist/benchmarks/index.d.ts +30 -12
- package/dist/benchmarks/index.js +12 -10
- package/dist/builder-eval/index.js +1 -1
- package/dist/campaign/index.d.ts +432 -146
- package/dist/campaign/index.js +11 -9
- package/dist/{chunk-ARU2PZFM.js → chunk-424XHVKV.js} +43 -23
- package/dist/chunk-424XHVKV.js.map +1 -0
- package/dist/{chunk-NUKSVU3W.js → chunk-5DTSBUL2.js} +2 -2
- package/dist/{chunk-BTACVO2W.js → chunk-65QETEKG.js} +542 -258
- package/dist/chunk-65QETEKG.js.map +1 -0
- package/dist/{chunk-CGG5SLH3.js → chunk-ARCFIY7U.js} +2 -2
- package/dist/{chunk-P22LJ3Y2.js → chunk-CLELFRQG.js} +6 -4
- package/dist/chunk-CLELFRQG.js.map +1 -0
- package/dist/{chunk-LUNF2SEL.js → chunk-DJKY2TSY.js} +3 -3
- package/dist/chunk-DPUHNQLN.js +232 -0
- package/dist/chunk-DPUHNQLN.js.map +1 -0
- package/dist/{chunk-IPYXE555.js → chunk-EJGRPCO3.js} +36 -13
- package/dist/chunk-EJGRPCO3.js.map +1 -0
- package/dist/{chunk-K6IAZZ6L.js → chunk-FA3MFYNA.js} +132 -110
- package/dist/chunk-FA3MFYNA.js.map +1 -0
- package/dist/chunk-IG35YDN3.js +614 -0
- package/dist/chunk-IG35YDN3.js.map +1 -0
- package/dist/{chunk-DPZAEKA6.js → chunk-IMGJZBFG.js} +84 -56
- package/dist/chunk-IMGJZBFG.js.map +1 -0
- package/dist/{chunk-H5UD2323.js → chunk-IN455XKS.js} +53 -1
- package/dist/chunk-IN455XKS.js.map +1 -0
- package/dist/{chunk-PJQFMIOX.js → chunk-MHELPNRP.js} +31 -1
- package/dist/chunk-MHELPNRP.js.map +1 -0
- package/dist/{chunk-IILEIWGW.js → chunk-NL33OXEP.js} +41 -36
- package/dist/chunk-NL33OXEP.js.map +1 -0
- package/dist/{chunk-474LBSOX.js → chunk-OD5NQ2CS.js} +41 -16
- package/dist/chunk-OD5NQ2CS.js.map +1 -0
- package/dist/{chunk-QOTFXW5L.js → chunk-P5W7RQKK.js} +2 -2
- package/dist/{chunk-IR3KBHOY.js → chunk-P6FYH6K4.js} +39 -3
- package/dist/chunk-P6FYH6K4.js.map +1 -0
- package/dist/{chunk-J5SQWP6Y.js → chunk-PBE2LOSS.js} +18 -2
- package/dist/chunk-PBE2LOSS.js.map +1 -0
- package/dist/{chunk-E3IADPLG.js → chunk-PRB53E3J.js} +50 -224
- package/dist/chunk-PRB53E3J.js.map +1 -0
- package/dist/{chunk-X4UCIOTZ.js → chunk-S5YLIBFX.js} +2 -2
- package/dist/{chunk-7VYEGHWF.js → chunk-SZLVEKMJ.js} +41 -20
- package/dist/chunk-SZLVEKMJ.js.map +1 -0
- package/dist/{chunk-LKKT3IVV.js → chunk-TSN7JT6D.js} +6 -4
- package/dist/{chunk-LKKT3IVV.js.map → chunk-TSN7JT6D.js.map} +1 -1
- package/dist/{chunk-M7AH34KV.js → chunk-UTA2TGC4.js} +32 -16
- package/dist/chunk-UTA2TGC4.js.map +1 -0
- package/dist/{chunk-MAX3TN3C.js → chunk-UWZZKKU7.js} +7 -19
- package/dist/chunk-UWZZKKU7.js.map +1 -0
- package/dist/{chunk-ZVCHKKOP.js → chunk-WED5KLZQ.js} +39 -15
- package/dist/chunk-WED5KLZQ.js.map +1 -0
- package/dist/{chunk-OCFJACJU.js → chunk-X6VOPMLD.js} +74 -14
- package/dist/chunk-X6VOPMLD.js.map +1 -0
- package/dist/{chunk-MOXWMGPC.js → chunk-Y5CYOL42.js} +159 -70
- package/dist/chunk-Y5CYOL42.js.map +1 -0
- package/dist/{chunk-FO7HEH76.js → chunk-YJBNWCAA.js} +2 -2
- package/dist/chunk-ZPUEEM5U.js +179 -0
- package/dist/chunk-ZPUEEM5U.js.map +1 -0
- package/dist/{chunk-KE2VWPZX.js → chunk-ZUUWPZCV.js} +3 -3
- package/dist/cli.js +4 -4
- package/dist/contract/index.d.ts +421 -124
- package/dist/contract/index.js +242 -115
- package/dist/contract/index.js.map +1 -1
- package/dist/control.d.ts +34 -32
- package/dist/control.js +2 -2
- package/dist/hosted/index.d.ts +142 -41
- package/dist/hosted/index.js +23 -1
- package/dist/index.d.ts +364 -255
- package/dist/index.js +315 -226
- package/dist/index.js.map +1 -1
- package/dist/meta-eval/index.d.ts +32 -32
- package/dist/meta-eval/index.js +2 -2
- package/dist/multishot/index.d.ts +1 -0
- package/dist/openapi.json +1 -1
- package/dist/pipelines/index.js +2 -2
- package/dist/reporting.d.ts +97 -74
- package/dist/reporting.js +5 -4
- package/dist/rl.d.ts +177 -131
- package/dist/rl.js +232 -126
- package/dist/rl.js.map +1 -1
- package/dist/rollout/index.d.ts +53 -63
- package/dist/rollout/index.js +7 -5
- package/dist/{run-campaign-FFRM3RH5.js → run-campaign-3A2EBYBE.js} +4 -3
- package/dist/supervisor-run/index.d.ts +7 -15
- package/dist/supervisor-run/index.js +2 -2
- package/dist/traces.d.ts +66 -41
- package/dist/traces.js +9 -5
- package/dist/wire/index.d.ts +1 -0
- package/dist/wire/index.js +2 -2
- package/docs/concepts.md +18 -1
- package/docs/hosted-ingest-spec.md +37 -40
- package/docs/insight-report.md +57 -7
- package/docs/research-report-methodology.md +20 -21
- package/docs/rollout.md +13 -8
- package/package.json +4 -3
- package/dist/chunk-474LBSOX.js.map +0 -1
- package/dist/chunk-7VYEGHWF.js.map +0 -1
- package/dist/chunk-ARU2PZFM.js.map +0 -1
- package/dist/chunk-BTACVO2W.js.map +0 -1
- package/dist/chunk-DPZAEKA6.js.map +0 -1
- package/dist/chunk-E3IADPLG.js.map +0 -1
- package/dist/chunk-H5UD2323.js.map +0 -1
- package/dist/chunk-IILEIWGW.js.map +0 -1
- package/dist/chunk-IPYXE555.js.map +0 -1
- package/dist/chunk-IR3KBHOY.js.map +0 -1
- package/dist/chunk-J5SQWP6Y.js.map +0 -1
- package/dist/chunk-K6IAZZ6L.js.map +0 -1
- package/dist/chunk-M7AH34KV.js.map +0 -1
- package/dist/chunk-MAX3TN3C.js.map +0 -1
- package/dist/chunk-MOXWMGPC.js.map +0 -1
- package/dist/chunk-OCFJACJU.js.map +0 -1
- package/dist/chunk-P22LJ3Y2.js.map +0 -1
- package/dist/chunk-PJQFMIOX.js.map +0 -1
- package/dist/chunk-ZVCHKKOP.js.map +0 -1
- package/dist/chunk-ZZUXHH3R.js +0 -99
- package/dist/chunk-ZZUXHH3R.js.map +0 -1
- /package/dist/{chunk-NUKSVU3W.js.map → chunk-5DTSBUL2.js.map} +0 -0
- /package/dist/{chunk-CGG5SLH3.js.map → chunk-ARCFIY7U.js.map} +0 -0
- /package/dist/{chunk-LUNF2SEL.js.map → chunk-DJKY2TSY.js.map} +0 -0
- /package/dist/{chunk-QOTFXW5L.js.map → chunk-P5W7RQKK.js.map} +0 -0
- /package/dist/{chunk-X4UCIOTZ.js.map → chunk-S5YLIBFX.js.map} +0 -0
- /package/dist/{chunk-FO7HEH76.js.map → chunk-YJBNWCAA.js.map} +0 -0
- /package/dist/{chunk-KE2VWPZX.js.map → chunk-ZUUWPZCV.js.map} +0 -0
- /package/dist/{run-campaign-FFRM3RH5.js.map → run-campaign-3A2EBYBE.js.map} +0 -0
package/dist/contract/index.js
CHANGED
|
@@ -7,14 +7,14 @@ import {
|
|
|
7
7
|
fromPigraphSession,
|
|
8
8
|
observeCodeAgentSession,
|
|
9
9
|
parseCodeAgentJsonl
|
|
10
|
-
} from "../chunk-
|
|
10
|
+
} from "../chunk-SZLVEKMJ.js";
|
|
11
11
|
import {
|
|
12
12
|
createHostedClient
|
|
13
|
-
} from "../chunk-
|
|
13
|
+
} from "../chunk-IG35YDN3.js";
|
|
14
14
|
import {
|
|
15
15
|
analyzeRuns,
|
|
16
16
|
summarizeExecution
|
|
17
|
-
} from "../chunk-
|
|
17
|
+
} from "../chunk-FA3MFYNA.js";
|
|
18
18
|
import {
|
|
19
19
|
REFERENCE_EQUIVALENCE_INPUT_LIMITS,
|
|
20
20
|
REFERENCE_EQUIVALENCE_JUDGE_VERSION,
|
|
@@ -40,7 +40,7 @@ import {
|
|
|
40
40
|
skillOptOptimizationMethod,
|
|
41
41
|
surfaceContentHash,
|
|
42
42
|
surfaceHash
|
|
43
|
-
} from "../chunk-
|
|
43
|
+
} from "../chunk-65QETEKG.js";
|
|
44
44
|
import {
|
|
45
45
|
campaignSplitDigest,
|
|
46
46
|
createRunCostLedger,
|
|
@@ -48,47 +48,60 @@ import {
|
|
|
48
48
|
inMemoryCampaignStorage,
|
|
49
49
|
resolveRunDir,
|
|
50
50
|
runCampaign
|
|
51
|
-
} from "../chunk-
|
|
51
|
+
} from "../chunk-WED5KLZQ.js";
|
|
52
52
|
import {
|
|
53
53
|
buildDefaultAnalystRegistry,
|
|
54
54
|
createChatClient
|
|
55
|
-
} from "../chunk-
|
|
55
|
+
} from "../chunk-DJKY2TSY.js";
|
|
56
56
|
import "../chunk-HHWE3POT.js";
|
|
57
57
|
import "../chunk-WGXIEX7P.js";
|
|
58
58
|
import {
|
|
59
59
|
FileSystemOutcomeStore,
|
|
60
60
|
InMemoryOutcomeStore
|
|
61
61
|
} from "../chunk-3RF76KTD.js";
|
|
62
|
-
import "../chunk-
|
|
63
|
-
import
|
|
64
|
-
|
|
62
|
+
import "../chunk-424XHVKV.js";
|
|
63
|
+
import {
|
|
64
|
+
campaignCellExecutionEvidence,
|
|
65
|
+
campaignCellJudgeDimensions,
|
|
66
|
+
campaignCellTaskScore,
|
|
67
|
+
campaignCellToRunRecord
|
|
68
|
+
} from "../chunk-ZPUEEM5U.js";
|
|
69
|
+
import "../chunk-PBE2LOSS.js";
|
|
70
|
+
import "../chunk-IMGJZBFG.js";
|
|
71
|
+
import "../chunk-DPUHNQLN.js";
|
|
65
72
|
import {
|
|
66
73
|
pairedBootstrap
|
|
67
|
-
} from "../chunk-
|
|
74
|
+
} from "../chunk-MHELPNRP.js";
|
|
68
75
|
import "../chunk-WS3NZZQQ.js";
|
|
69
76
|
import "../chunk-VI2UW6B6.js";
|
|
70
77
|
import {
|
|
71
78
|
recordAggregateMeasurements,
|
|
72
|
-
summarizeExecutionMeasurements
|
|
73
|
-
|
|
79
|
+
summarizeExecutionMeasurements,
|
|
80
|
+
summarizeTraceErrors
|
|
81
|
+
} from "../chunk-IN455XKS.js";
|
|
74
82
|
import "../chunk-PXE2VKMX.js";
|
|
75
83
|
import "../chunk-ZET2UAYW.js";
|
|
76
84
|
import "../chunk-GGE4NNQT.js";
|
|
77
|
-
import
|
|
85
|
+
import {
|
|
86
|
+
classifyOtlpSpanRole,
|
|
87
|
+
isOtlpModelCall
|
|
88
|
+
} from "../chunk-P6FYH6K4.js";
|
|
78
89
|
import "../chunk-PC4UYEBM.js";
|
|
79
90
|
import {
|
|
80
91
|
parseRunRecordSafe
|
|
81
|
-
} from "../chunk-
|
|
92
|
+
} from "../chunk-NL33OXEP.js";
|
|
82
93
|
import "../chunk-MA6HLL3S.js";
|
|
83
94
|
import "../chunk-VSMTAMNK.js";
|
|
84
|
-
import "../chunk-ONWEPEDO.js";
|
|
85
95
|
import {
|
|
86
|
-
|
|
96
|
+
ValidationError
|
|
97
|
+
} from "../chunk-ONWEPEDO.js";
|
|
98
|
+
import {
|
|
99
|
+
LLM_MODEL_ATTR_KEYS,
|
|
100
|
+
SPAN_KIND_ATTR_KEYS
|
|
87
101
|
} from "../chunk-K4DBDHLK.js";
|
|
88
102
|
import "../chunk-PZ5AY32C.js";
|
|
89
103
|
|
|
90
104
|
// src/contract/self-improve.ts
|
|
91
|
-
import { createHash } from "crypto";
|
|
92
105
|
var SelfImproveRunError = class extends Error {
|
|
93
106
|
cost;
|
|
94
107
|
receipts;
|
|
@@ -343,11 +356,11 @@ async function runSelfImprove(opts, costLedger, startedAt, runDir, storage) {
|
|
|
343
356
|
findings: opts.findings,
|
|
344
357
|
selectionRankKey: opts.selectionRankKey
|
|
345
358
|
});
|
|
346
|
-
const
|
|
347
|
-
const
|
|
348
|
-
|
|
349
|
-
);
|
|
350
|
-
const winnerStats = meanComposite(
|
|
359
|
+
const reportSplit = holdoutDeferred ? "search" : "holdout";
|
|
360
|
+
const reportBaselineCampaign = holdoutDeferred ? result.baselineCampaign : result.baselineOnHoldout;
|
|
361
|
+
const reportWinnerCampaign = holdoutDeferred ? winnerSearchCampaign(result) : result.winnerOnHoldout;
|
|
362
|
+
const baseline = meanComposite(reportBaselineCampaign.aggregates.byScenario);
|
|
363
|
+
const winnerStats = meanComposite(reportWinnerCampaign.aggregates.byScenario);
|
|
351
364
|
let power;
|
|
352
365
|
const baselineHoldoutComposites = result.baselineOnHoldout.cells.filter((cell) => !cell.error).map((cell) => {
|
|
353
366
|
const scores = Object.values(cell.judgeScores);
|
|
@@ -390,12 +403,19 @@ async function runSelfImprove(opts, costLedger, startedAt, runDir, storage) {
|
|
|
390
403
|
const totalCost = cost.totalCostUsd;
|
|
391
404
|
const insight = await analyzeRuns({
|
|
392
405
|
runs: [
|
|
393
|
-
...cellsToRunRecords(result.baselineCampaign.cells, "baseline", runDir, opts.baselineSurface),
|
|
394
406
|
...cellsToRunRecords(
|
|
395
|
-
|
|
407
|
+
reportBaselineCampaign.cells,
|
|
408
|
+
"baseline",
|
|
409
|
+
runDir,
|
|
410
|
+
opts.baselineSurface,
|
|
411
|
+
reportSplit
|
|
412
|
+
),
|
|
413
|
+
...cellsToRunRecords(
|
|
414
|
+
reportWinnerCampaign.cells,
|
|
396
415
|
"winner",
|
|
397
416
|
runDir,
|
|
398
|
-
result.winnerSurface
|
|
417
|
+
result.winnerSurface,
|
|
418
|
+
reportSplit
|
|
399
419
|
)
|
|
400
420
|
],
|
|
401
421
|
baselineCandidateId: "baseline",
|
|
@@ -468,19 +488,21 @@ async function shipEvalRunToHosted(tenant, opts, summary, raw, runDir) {
|
|
|
468
488
|
const client = createHostedClient(tenant);
|
|
469
489
|
function snapshotFromCampaign(index, surface, campaign, durationMs) {
|
|
470
490
|
const cells = campaign.cells.map((cell) => {
|
|
471
|
-
const
|
|
472
|
-
const composite = judgeScores.length === 0 ? 0 : judgeScores.reduce((s, j) => s + j.composite, 0) / judgeScores.length;
|
|
491
|
+
const execution = campaignCellExecutionEvidence(cell);
|
|
473
492
|
return {
|
|
474
493
|
scenarioId: cell.scenarioId,
|
|
475
494
|
rep: cell.rep,
|
|
476
|
-
compositeMean:
|
|
477
|
-
dimensions:
|
|
478
|
-
|
|
479
|
-
|
|
495
|
+
compositeMean: campaignCellTaskScore(cell) ?? null,
|
|
496
|
+
dimensions: campaignCellJudgeDimensions(cell),
|
|
497
|
+
terminalOutcome: execution.terminalOutcome,
|
|
498
|
+
executionErrorCount: execution.executionErrorCount ?? null,
|
|
480
499
|
errorMessage: cell.error ?? void 0
|
|
481
500
|
};
|
|
482
501
|
});
|
|
483
|
-
const
|
|
502
|
+
const scoredCells = cells.flatMap(
|
|
503
|
+
(cell) => cell.compositeMean === null ? [] : [cell.compositeMean]
|
|
504
|
+
);
|
|
505
|
+
const compositeMean = scoredCells.length === 0 ? null : scoredCells.reduce((sum, score) => sum + score, 0) / scoredCells.length;
|
|
484
506
|
return {
|
|
485
507
|
index,
|
|
486
508
|
surfaceHash: surfaceHash(surface),
|
|
@@ -531,67 +553,23 @@ function hashString(s) {
|
|
|
531
553
|
}
|
|
532
554
|
return h.toString(16).padStart(8, "0");
|
|
533
555
|
}
|
|
534
|
-
function cellsToRunRecords(cells, candidateId, runId, surface) {
|
|
556
|
+
function cellsToRunRecords(cells, candidateId, runId, surface, splitTag) {
|
|
535
557
|
const promptHash = surfaceContentHash(surface);
|
|
536
|
-
const configHash =
|
|
537
|
-
return cells.map(
|
|
538
|
-
|
|
539
|
-
const perDimMeanAccum = {};
|
|
540
|
-
let compositeSum = 0;
|
|
541
|
-
let compositeCount = 0;
|
|
542
|
-
for (const [judgeId, score] of Object.entries(cell.judgeScores)) {
|
|
543
|
-
perJudge[judgeId] = { ...score.dimensions };
|
|
544
|
-
for (const [dim, value] of Object.entries(score.dimensions)) {
|
|
545
|
-
if (!Number.isFinite(value)) continue;
|
|
546
|
-
const accum = perDimMeanAccum[dim] ?? { sum: 0, n: 0 };
|
|
547
|
-
accum.sum += value;
|
|
548
|
-
accum.n += 1;
|
|
549
|
-
perDimMeanAccum[dim] = accum;
|
|
550
|
-
}
|
|
551
|
-
if (Number.isFinite(score.composite)) {
|
|
552
|
-
compositeSum += score.composite;
|
|
553
|
-
compositeCount += 1;
|
|
554
|
-
}
|
|
555
|
-
}
|
|
556
|
-
const perDimMean = {};
|
|
557
|
-
for (const [dim, { sum, n }] of Object.entries(perDimMeanAccum)) {
|
|
558
|
-
perDimMean[dim] = n === 0 ? 0 : sum / n;
|
|
559
|
-
}
|
|
560
|
-
const composite = compositeCount === 0 ? 0 : compositeSum / compositeCount;
|
|
561
|
-
const judgeScores = {
|
|
562
|
-
perJudge,
|
|
563
|
-
perDimMean,
|
|
564
|
-
composite
|
|
565
|
-
};
|
|
566
|
-
return {
|
|
558
|
+
const configHash = surfaceContentHash(candidateId);
|
|
559
|
+
return cells.map(
|
|
560
|
+
(cell) => campaignCellToRunRecord(cell, {
|
|
567
561
|
runId: `${runId}::${candidateId}::${cell.cellId}`,
|
|
568
562
|
experimentId: runId,
|
|
569
563
|
candidateId,
|
|
570
|
-
//
|
|
571
|
-
// Synthesize a stable seed for that pairing.
|
|
564
|
+
// scenarioId is explicit; seed keeps repeated runs distinct.
|
|
572
565
|
seed: cell.rep * 1e6 + hashString(cell.scenarioId).slice(0, 6).split("").reduce((a, c) => a * 31 + c.charCodeAt(0) >>> 0, 0),
|
|
573
566
|
model: "campaign-cell",
|
|
574
567
|
promptHash,
|
|
575
568
|
configHash,
|
|
576
569
|
commitSha: "cell",
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
input: cell.tokenUsage.input,
|
|
581
|
-
output: cell.tokenUsage.output,
|
|
582
|
-
...cell.tokenUsage.reasoning === void 0 ? {} : { reasoning: cell.tokenUsage.reasoning },
|
|
583
|
-
...cell.tokenUsage.cached === void 0 ? {} : { cached: cell.tokenUsage.cached },
|
|
584
|
-
...cell.tokenUsage.cacheWrite === void 0 ? {} : { cacheWrite: cell.tokenUsage.cacheWrite }
|
|
585
|
-
},
|
|
586
|
-
outcome: {
|
|
587
|
-
holdoutScore: composite,
|
|
588
|
-
raw: {},
|
|
589
|
-
judgeScores
|
|
590
|
-
},
|
|
591
|
-
splitTag: "holdout",
|
|
592
|
-
...cell.error ? { failureMode: cell.error } : {}
|
|
593
|
-
};
|
|
594
|
-
});
|
|
570
|
+
splitTag
|
|
571
|
+
})
|
|
572
|
+
);
|
|
595
573
|
}
|
|
596
574
|
|
|
597
575
|
// src/contract/define-agent-eval.ts
|
|
@@ -1565,7 +1543,7 @@ function diffGenerations(before, after) {
|
|
|
1565
1543
|
rep: beforeCell.rep,
|
|
1566
1544
|
compositeBefore: beforeCell.compositeMean,
|
|
1567
1545
|
compositeAfter: afterCell.compositeMean,
|
|
1568
|
-
compositeDelta: afterCell.compositeMean - beforeCell.compositeMean,
|
|
1546
|
+
compositeDelta: beforeCell.compositeMean === null || afterCell.compositeMean === null ? null : afterCell.compositeMean - beforeCell.compositeMean,
|
|
1569
1547
|
dimensions: diffDimensions(beforeCell.dimensions, afterCell.dimensions)
|
|
1570
1548
|
});
|
|
1571
1549
|
}
|
|
@@ -1583,7 +1561,7 @@ function diffGenerations(before, after) {
|
|
|
1583
1561
|
added,
|
|
1584
1562
|
compositeBefore: before.compositeMean,
|
|
1585
1563
|
compositeAfter: after.compositeMean,
|
|
1586
|
-
compositeDelta: after.compositeMean - before.compositeMean,
|
|
1564
|
+
compositeDelta: before.compositeMean === null || after.compositeMean === null ? null : after.compositeMean - before.compositeMean,
|
|
1587
1565
|
costUsdBefore: before.costUsd,
|
|
1588
1566
|
costUsdAfter: after.costUsd,
|
|
1589
1567
|
costUsdDelta: after.costUsd - before.costUsd,
|
|
@@ -1736,14 +1714,13 @@ function fromFeedbackTable(opts) {
|
|
|
1736
1714
|
perDimMean: { rating: meanScore },
|
|
1737
1715
|
composite: meanScore
|
|
1738
1716
|
};
|
|
1717
|
+
const splitTag = runMeta.splitTag ?? "holdout";
|
|
1739
1718
|
const outcome = {
|
|
1740
|
-
|
|
1741
|
-
// `holdoutScore` so downstream substrate primitives (`paretoChart`,
|
|
1742
|
-
// promotion gates) read it correctly by default.
|
|
1743
|
-
holdoutScore: meanScore,
|
|
1719
|
+
...splitTag === "holdout" ? { holdoutScore: meanScore } : { searchScore: meanScore },
|
|
1744
1720
|
raw: Object.fromEntries(normalised.map((r) => [`rater:${r.rater}`, r.score])),
|
|
1745
1721
|
judgeScores
|
|
1746
1722
|
};
|
|
1723
|
+
const costUsd = runMeta.costUsd ?? null;
|
|
1747
1724
|
runs.push({
|
|
1748
1725
|
runId,
|
|
1749
1726
|
experimentId: runMeta.experimentId ?? "feedback-corpus",
|
|
@@ -1754,10 +1731,13 @@ function fromFeedbackTable(opts) {
|
|
|
1754
1731
|
configHash: runMeta.configHash ?? "sha256:unknown",
|
|
1755
1732
|
commitSha: runMeta.commitSha ?? "unknown",
|
|
1756
1733
|
wallMs: runMeta.wallMs ?? 0,
|
|
1757
|
-
costUsd
|
|
1734
|
+
costUsd,
|
|
1735
|
+
costProvenance: costUsd === null ? { kind: "uncaptured", usd: null } : { kind: "observed", usd: costUsd },
|
|
1758
1736
|
tokenUsage: { input: 0, output: 0 },
|
|
1737
|
+
terminalOutcome: "unknown",
|
|
1759
1738
|
outcome,
|
|
1760
|
-
splitTag
|
|
1739
|
+
splitTag,
|
|
1740
|
+
scenarioId: runMeta.scenarioId ?? runId
|
|
1761
1741
|
});
|
|
1762
1742
|
if (emitRaterScores) {
|
|
1763
1743
|
for (const r of normalised) raterScores.push({ runId, rater: r.rater, score: r.score });
|
|
@@ -1767,7 +1747,12 @@ function fromFeedbackTable(opts) {
|
|
|
1767
1747
|
}
|
|
1768
1748
|
|
|
1769
1749
|
// src/contract/intake/otel-spans.ts
|
|
1770
|
-
var
|
|
1750
|
+
var TASK_SCORE_ATTR_KEYS = [
|
|
1751
|
+
"gen_ai.evaluation.score.value",
|
|
1752
|
+
"tangle.task.score",
|
|
1753
|
+
"eval.score",
|
|
1754
|
+
"tangle.score"
|
|
1755
|
+
];
|
|
1771
1756
|
var MODEL_KEYS = ["tangle.model", ...LLM_MODEL_ATTR_KEYS, "model"];
|
|
1772
1757
|
var PROMPT_HASH_KEYS = ["tangle.prompt_hash", "prompt.hash"];
|
|
1773
1758
|
var CONFIG_HASH_KEYS = ["tangle.config_hash", "config.hash"];
|
|
@@ -1789,15 +1774,31 @@ function fromOtelSpans(opts) {
|
|
|
1789
1774
|
);
|
|
1790
1775
|
const callSpanIds = new Set(measurements.callSpanIds);
|
|
1791
1776
|
const callSpans = groupSpans2.filter((span) => callSpanIds.has(span.spanId));
|
|
1792
|
-
const wallMs =
|
|
1777
|
+
const wallMs = unixNanoDurationMs(root.startTimeUnixNano, root.endTimeUnixNano);
|
|
1793
1778
|
const model = readAttrString(callSpans, MODEL_KEYS) ?? readAttrString(groupSpans2, MODEL_KEYS) ?? "unknown@unknown";
|
|
1794
1779
|
const capturedCost = (measurements.cost.complete ? measurements.cost.value : void 0) ?? measurements.aggregate?.costUsd;
|
|
1795
|
-
const costUsd = capturedCost ??
|
|
1780
|
+
const costUsd = capturedCost ?? null;
|
|
1781
|
+
const scenarioId = readConsistentScenarioId(groupKey, groupSpans2) ?? groupKey;
|
|
1796
1782
|
const promptHash = readAttrString(groupSpans2, PROMPT_HASH_KEYS) ?? "sha256:unknown";
|
|
1797
1783
|
const configHash = readAttrString(groupSpans2, CONFIG_HASH_KEYS) ?? "sha256:unknown";
|
|
1798
|
-
const score =
|
|
1784
|
+
const score = resolveTaskScore(groupKey, groupSpans2, opts.scoreForRun);
|
|
1799
1785
|
const rawNumeric = collectNumericAttrs(groupSpans2);
|
|
1800
|
-
|
|
1786
|
+
const errorSummary = summarizeTraceErrors(
|
|
1787
|
+
groupSpans2.map((span) => ({
|
|
1788
|
+
id: spanIdentity(span),
|
|
1789
|
+
...span.parentSpanId ? { parentId: parentIdentity(span) } : {},
|
|
1790
|
+
role: errorRoleForSpan(span),
|
|
1791
|
+
error: span.status?.code === "ERROR",
|
|
1792
|
+
processRoot: !span.parentSpanId && isTerminalRootCandidate(span)
|
|
1793
|
+
}))
|
|
1794
|
+
);
|
|
1795
|
+
rawNumeric.error_span_count = errorSummary.total;
|
|
1796
|
+
rawNumeric.execution_error_count = errorSummary.execution;
|
|
1797
|
+
rawNumeric.process_error_count = errorSummary.process;
|
|
1798
|
+
rawNumeric.guardrail_error_count = errorSummary.guardrail;
|
|
1799
|
+
rawNumeric.judge_error_count = errorSummary.evaluation;
|
|
1800
|
+
rawNumeric.propagated_error_count = errorSummary.propagated;
|
|
1801
|
+
rawNumeric.unclassified_error_count = errorSummary.unclassified;
|
|
1801
1802
|
rawNumeric.llm_span_count = measurements.modelCallCount;
|
|
1802
1803
|
if (measurements.cost.value !== void 0 && !measurements.cost.complete) {
|
|
1803
1804
|
rawNumeric.partial_observed_cost_usd = measurements.cost.value;
|
|
@@ -1808,12 +1809,18 @@ function fromOtelSpans(opts) {
|
|
|
1808
1809
|
perDimMean: { score },
|
|
1809
1810
|
composite: score
|
|
1810
1811
|
} : void 0;
|
|
1811
|
-
const
|
|
1812
|
+
const terminalOutcome = terminalOutcomeFromRoots(groupSpans2);
|
|
1813
|
+
const failedRoot = terminalOutcome === "failed" ? groupSpans2.find(
|
|
1814
|
+
(span) => !span.parentSpanId && isTerminalRootCandidate(span) && span.status?.code === "ERROR"
|
|
1815
|
+
) : void 0;
|
|
1812
1816
|
const outcome = {
|
|
1813
|
-
...opts.defaultSplit === "search" ? { searchScore: score } : { holdoutScore: score },
|
|
1814
1817
|
raw: rawNumeric,
|
|
1815
1818
|
...judgeScores ? { judgeScores } : {}
|
|
1816
1819
|
};
|
|
1820
|
+
if (score !== void 0) {
|
|
1821
|
+
if (defaultSplit === "holdout") outcome.holdoutScore = score;
|
|
1822
|
+
else outcome.searchScore = score;
|
|
1823
|
+
}
|
|
1817
1824
|
runs.push({
|
|
1818
1825
|
runId: groupKey,
|
|
1819
1826
|
experimentId,
|
|
@@ -1827,22 +1834,55 @@ function fromOtelSpans(opts) {
|
|
|
1827
1834
|
costUsd,
|
|
1828
1835
|
costProvenance: capturedCost === void 0 ? { kind: "uncaptured", usd: null } : { kind: "observed", usd: capturedCost },
|
|
1829
1836
|
tokenUsage: measurements.tokenUsage,
|
|
1837
|
+
terminalOutcome,
|
|
1838
|
+
...failedRoot ? { terminalFailureReason: failedRoot.status?.message ?? failedRoot.name } : {},
|
|
1830
1839
|
outcome,
|
|
1831
1840
|
splitTag: defaultSplit,
|
|
1832
|
-
|
|
1841
|
+
scenarioId
|
|
1833
1842
|
});
|
|
1834
1843
|
}
|
|
1835
1844
|
return runs;
|
|
1836
1845
|
}
|
|
1846
|
+
function terminalOutcomeFromRoots(spans) {
|
|
1847
|
+
const roots = spans.filter((span) => !span.parentSpanId && isTerminalRootCandidate(span));
|
|
1848
|
+
if (roots.length !== 1) return "unknown";
|
|
1849
|
+
if (roots[0].status?.code === "OK") return "succeeded";
|
|
1850
|
+
if (roots[0].status?.code === "ERROR") return "failed";
|
|
1851
|
+
return "unknown";
|
|
1852
|
+
}
|
|
1853
|
+
function isTerminalRootCandidate(span) {
|
|
1854
|
+
const role = errorRoleForSpan(span);
|
|
1855
|
+
if (role === "LLM" || role === "TOOL" || role === "GUARDRAIL" || role === "EVALUATOR") {
|
|
1856
|
+
return false;
|
|
1857
|
+
}
|
|
1858
|
+
return true;
|
|
1859
|
+
}
|
|
1860
|
+
function readSpanKind(span) {
|
|
1861
|
+
return readAttrString([span], [...SPAN_KIND_ATTR_KEYS, "span.kind"])?.toUpperCase();
|
|
1862
|
+
}
|
|
1863
|
+
function errorRoleForSpan(span) {
|
|
1864
|
+
return classifyOtlpSpanRole({
|
|
1865
|
+
kind: readSpanKind(span),
|
|
1866
|
+
name: span.name,
|
|
1867
|
+
attributes: span.attributes
|
|
1868
|
+
});
|
|
1869
|
+
}
|
|
1870
|
+
function spanIdentity(span) {
|
|
1871
|
+
return `${span.traceId}:${span.spanId}`;
|
|
1872
|
+
}
|
|
1873
|
+
function parentIdentity(span) {
|
|
1874
|
+
return `${span.traceId}:${span.parentSpanId}`;
|
|
1875
|
+
}
|
|
1837
1876
|
function isExplicitModelCall(span) {
|
|
1838
|
-
|
|
1839
|
-
|
|
1840
|
-
|
|
1841
|
-
|
|
1877
|
+
return isOtlpModelCall({
|
|
1878
|
+
kind: readSpanKind(span),
|
|
1879
|
+
name: span.name,
|
|
1880
|
+
attributes: span.attributes
|
|
1881
|
+
});
|
|
1842
1882
|
}
|
|
1843
1883
|
function isExplicitAggregate(span) {
|
|
1844
|
-
const kind = span
|
|
1845
|
-
return
|
|
1884
|
+
const kind = readSpanKind(span);
|
|
1885
|
+
return kind !== void 0 && kind !== "LLM";
|
|
1846
1886
|
}
|
|
1847
1887
|
function groupSpans(spans) {
|
|
1848
1888
|
const m = /* @__PURE__ */ new Map();
|
|
@@ -1855,7 +1895,10 @@ function groupSpans(spans) {
|
|
|
1855
1895
|
return m;
|
|
1856
1896
|
}
|
|
1857
1897
|
function findRoot(group) {
|
|
1858
|
-
|
|
1898
|
+
const structuralRoots = group.filter((span) => !span.parentSpanId);
|
|
1899
|
+
const terminalRoots = structuralRoots.filter(isTerminalRootCandidate);
|
|
1900
|
+
const pool = terminalRoots.length > 0 ? terminalRoots : structuralRoots.length > 0 ? structuralRoots : group;
|
|
1901
|
+
return orderSpans(pool)[0];
|
|
1859
1902
|
}
|
|
1860
1903
|
function readAttrString(spans, keys) {
|
|
1861
1904
|
for (const span of spans) {
|
|
@@ -1866,18 +1909,102 @@ function readAttrString(spans, keys) {
|
|
|
1866
1909
|
}
|
|
1867
1910
|
return void 0;
|
|
1868
1911
|
}
|
|
1869
|
-
function
|
|
1912
|
+
function readConsistentScenarioId(runId, spans) {
|
|
1913
|
+
const values = /* @__PURE__ */ new Set();
|
|
1870
1914
|
for (const span of spans) {
|
|
1871
|
-
|
|
1872
|
-
|
|
1873
|
-
|
|
1874
|
-
|
|
1875
|
-
|
|
1876
|
-
|
|
1915
|
+
const topLevel = span["tangle.scenarioId"];
|
|
1916
|
+
if (typeof topLevel === "string" && topLevel.length > 0) values.add(topLevel);
|
|
1917
|
+
const attribute = span.attributes["tangle.scenarioId"];
|
|
1918
|
+
if (typeof attribute === "string" && attribute.length > 0) values.add(attribute);
|
|
1919
|
+
}
|
|
1920
|
+
if (values.size > 1) {
|
|
1921
|
+
throw new ValidationError(
|
|
1922
|
+
`fromOtelSpans: conflicting scenario ids for run '${runId}': ${[...values].sort().join(", ")}`
|
|
1923
|
+
);
|
|
1924
|
+
}
|
|
1925
|
+
return values.values().next().value;
|
|
1926
|
+
}
|
|
1927
|
+
function resolveTaskScore(runId, spans, scoreForRun) {
|
|
1928
|
+
const orderedSpans = orderSpans(spans);
|
|
1929
|
+
const sources = [];
|
|
1930
|
+
if (scoreForRun) {
|
|
1931
|
+
const supplied = scoreForRun(runId, orderedSpans);
|
|
1932
|
+
if (supplied !== void 0) {
|
|
1933
|
+
if (typeof supplied !== "number" || !Number.isFinite(supplied)) {
|
|
1934
|
+
throw new ValidationError(
|
|
1935
|
+
`fromOtelSpans: scoreForRun returned a non-finite number for run '${runId}'`
|
|
1936
|
+
);
|
|
1877
1937
|
}
|
|
1938
|
+
sources.push({ label: "scoreForRun", value: supplied });
|
|
1878
1939
|
}
|
|
1879
1940
|
}
|
|
1880
|
-
|
|
1941
|
+
for (const span of orderedSpans) {
|
|
1942
|
+
const role = errorRoleForSpan(span);
|
|
1943
|
+
if (span.parentSpanId && role !== "EVALUATOR") continue;
|
|
1944
|
+
if (role === "EVALUATOR" && span.status?.code === "ERROR") continue;
|
|
1945
|
+
for (const key of TASK_SCORE_ATTR_KEYS) {
|
|
1946
|
+
if (!Object.hasOwn(span.attributes, key)) continue;
|
|
1947
|
+
sources.push({
|
|
1948
|
+
label: `span '${span.spanId}' attribute '${key}'`,
|
|
1949
|
+
value: parseTaskScoreAttribute(runId, span.spanId, key, span.attributes[key])
|
|
1950
|
+
});
|
|
1951
|
+
}
|
|
1952
|
+
}
|
|
1953
|
+
if (sources.length === 0) return void 0;
|
|
1954
|
+
sources.sort((left, right) => left.label.localeCompare(right.label));
|
|
1955
|
+
const score = sources[0].value;
|
|
1956
|
+
if (sources.some((source) => source.value !== score)) {
|
|
1957
|
+
const details = sources.map((source) => `${source.label}=${source.value}`).join(", ");
|
|
1958
|
+
throw new ValidationError(
|
|
1959
|
+
`fromOtelSpans: conflicting task-quality scores for run '${runId}': ${details}`
|
|
1960
|
+
);
|
|
1961
|
+
}
|
|
1962
|
+
return score;
|
|
1963
|
+
}
|
|
1964
|
+
function parseTaskScoreAttribute(runId, spanId, key, value) {
|
|
1965
|
+
const source = `span '${spanId}' attribute '${key}'`;
|
|
1966
|
+
if (typeof value === "string") {
|
|
1967
|
+
if (value.trim().length === 0) {
|
|
1968
|
+
throw new ValidationError(
|
|
1969
|
+
`fromOtelSpans: ${source} is blank for run '${runId}'; task quality must be finite`
|
|
1970
|
+
);
|
|
1971
|
+
}
|
|
1972
|
+
const parsed = Number(value);
|
|
1973
|
+
if (Number.isFinite(parsed)) return parsed;
|
|
1974
|
+
} else if (typeof value === "number" && Number.isFinite(value)) {
|
|
1975
|
+
return value;
|
|
1976
|
+
}
|
|
1977
|
+
throw new ValidationError(
|
|
1978
|
+
`fromOtelSpans: ${source} is not a finite task-quality score for run '${runId}'`
|
|
1979
|
+
);
|
|
1980
|
+
}
|
|
1981
|
+
function orderSpans(spans) {
|
|
1982
|
+
return [...spans].sort(
|
|
1983
|
+
(left, right) => compareUnixNano(left.startTimeUnixNano, right.startTimeUnixNano) || left.spanId.localeCompare(right.spanId)
|
|
1984
|
+
);
|
|
1985
|
+
}
|
|
1986
|
+
function parseUnixNano(value) {
|
|
1987
|
+
try {
|
|
1988
|
+
return BigInt(value);
|
|
1989
|
+
} catch {
|
|
1990
|
+
throw new ValidationError(`fromOtelSpans: invalid Unix nanosecond timestamp '${value}'`);
|
|
1991
|
+
}
|
|
1992
|
+
}
|
|
1993
|
+
function compareUnixNano(left, right) {
|
|
1994
|
+
const leftValue = parseUnixNano(left);
|
|
1995
|
+
const rightValue = parseUnixNano(right);
|
|
1996
|
+
return leftValue < rightValue ? -1 : leftValue > rightValue ? 1 : 0;
|
|
1997
|
+
}
|
|
1998
|
+
function unixNanoDurationMs(start, end) {
|
|
1999
|
+
const delta = parseUnixNano(end) - parseUnixNano(start);
|
|
2000
|
+
if (delta <= 0n) return 0;
|
|
2001
|
+
const wholeMs = delta / 1000000n;
|
|
2002
|
+
const fractionalMs = delta % 1000000n;
|
|
2003
|
+
const value = Number(wholeMs) + Number(fractionalMs) / 1e6;
|
|
2004
|
+
if (!Number.isSafeInteger(Number(wholeMs))) {
|
|
2005
|
+
throw new ValidationError("fromOtelSpans: span duration exceeds the safe millisecond range");
|
|
2006
|
+
}
|
|
2007
|
+
return value;
|
|
1881
2008
|
}
|
|
1882
2009
|
function collectNumericAttrs(spans) {
|
|
1883
2010
|
const raw = {};
|