@tangle-network/agent-eval 0.126.7 → 0.128.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +59 -0
- package/README.md +6 -2
- package/dist/analyst/index.d.ts +52 -43
- package/dist/analyst/index.js +4 -4
- package/dist/belief-state/index.d.ts +35 -40
- package/dist/belief-state/index.js +2 -2
- package/dist/benchmarks/index.d.ts +30 -12
- package/dist/benchmarks/index.js +12 -11
- package/dist/builder-eval/index.js +1 -1
- package/dist/campaign/index.d.ts +441 -160
- package/dist/campaign/index.js +11 -10
- package/dist/{chunk-IILEIWGW.js → chunk-2JX3CFMB.js} +100 -40
- package/dist/chunk-2JX3CFMB.js.map +1 -0
- package/dist/chunk-2MKQIFS4.js +183 -0
- package/dist/chunk-2MKQIFS4.js.map +1 -0
- package/dist/{chunk-NUKSVU3W.js → chunk-5DTSBUL2.js} +2 -2
- package/dist/{chunk-H5UD2323.js → chunk-7ZZMD7UK.js} +102 -2
- package/dist/chunk-7ZZMD7UK.js.map +1 -0
- package/dist/{chunk-474LBSOX.js → chunk-BYT7ELPS.js} +45 -18
- package/dist/chunk-BYT7ELPS.js.map +1 -0
- package/dist/{chunk-LUNF2SEL.js → chunk-DJKY2TSY.js} +3 -3
- package/dist/chunk-DPUHNQLN.js +232 -0
- package/dist/chunk-DPUHNQLN.js.map +1 -0
- package/dist/chunk-DRYIUNWY.js +622 -0
- package/dist/chunk-DRYIUNWY.js.map +1 -0
- package/dist/{chunk-IPYXE555.js → chunk-EJGRPCO3.js} +36 -13
- package/dist/chunk-EJGRPCO3.js.map +1 -0
- package/dist/{chunk-OCFJACJU.js → chunk-EOSZT7PL.js} +82 -18
- package/dist/chunk-EOSZT7PL.js.map +1 -0
- package/dist/{chunk-ZVCHKKOP.js → chunk-EZJEIH2R.js} +39 -15
- package/dist/chunk-EZJEIH2R.js.map +1 -0
- package/dist/{chunk-M7AH34KV.js → chunk-IHQDPH7D.js} +32 -16
- package/dist/chunk-IHQDPH7D.js.map +1 -0
- package/dist/{chunk-PJQFMIOX.js → chunk-MHELPNRP.js} +31 -1
- package/dist/chunk-MHELPNRP.js.map +1 -0
- package/dist/{chunk-K6IAZZ6L.js → chunk-NACAGYSY.js} +153 -122
- package/dist/chunk-NACAGYSY.js.map +1 -0
- package/dist/{chunk-BTACVO2W.js → chunk-NKAGIDE2.js} +542 -258
- package/dist/chunk-NKAGIDE2.js.map +1 -0
- package/dist/{chunk-ARU2PZFM.js → chunk-NYLOYM6N.js} +43 -23
- package/dist/chunk-NYLOYM6N.js.map +1 -0
- package/dist/{chunk-QOTFXW5L.js → chunk-P5W7RQKK.js} +2 -2
- package/dist/{chunk-IR3KBHOY.js → chunk-P6FYH6K4.js} +39 -3
- package/dist/chunk-P6FYH6K4.js.map +1 -0
- package/dist/{chunk-J5SQWP6Y.js → chunk-PBE2LOSS.js} +18 -2
- package/dist/chunk-PBE2LOSS.js.map +1 -0
- package/dist/{chunk-X4UCIOTZ.js → chunk-S5YLIBFX.js} +2 -2
- package/dist/{chunk-7VYEGHWF.js → chunk-SZLVEKMJ.js} +41 -20
- package/dist/chunk-SZLVEKMJ.js.map +1 -0
- package/dist/{chunk-P22LJ3Y2.js → chunk-TBL77AUT.js} +8 -8
- package/dist/chunk-TBL77AUT.js.map +1 -0
- package/dist/{chunk-LKKT3IVV.js → chunk-TSN7JT6D.js} +6 -4
- package/dist/{chunk-LKKT3IVV.js.map → chunk-TSN7JT6D.js.map} +1 -1
- package/dist/{chunk-E3IADPLG.js → chunk-UB2LOJ6Q.js} +51 -227
- package/dist/chunk-UB2LOJ6Q.js.map +1 -0
- package/dist/{chunk-MAX3TN3C.js → chunk-UWZZKKU7.js} +7 -19
- package/dist/chunk-UWZZKKU7.js.map +1 -0
- package/dist/{chunk-DPZAEKA6.js → chunk-VLOATJQ2.js} +85 -57
- package/dist/chunk-VLOATJQ2.js.map +1 -0
- package/dist/{chunk-MOXWMGPC.js → chunk-XDWDC2MP.js} +197 -79
- package/dist/chunk-XDWDC2MP.js.map +1 -0
- package/dist/{chunk-CGG5SLH3.js → chunk-XPRT64IE.js} +2 -2
- package/dist/{chunk-FO7HEH76.js → chunk-YJBNWCAA.js} +2 -2
- package/dist/{chunk-KE2VWPZX.js → chunk-ZUUWPZCV.js} +3 -3
- package/dist/cli.js +4 -4
- package/dist/contract/index.d.ts +438 -138
- package/dist/contract/index.js +259 -113
- package/dist/contract/index.js.map +1 -1
- package/dist/control.d.ts +56 -39
- package/dist/control.js +2 -3
- package/dist/hosted/index.d.ts +152 -51
- package/dist/hosted/index.js +24 -1
- package/dist/index.d.ts +399 -277
- package/dist/index.js +363 -244
- package/dist/index.js.map +1 -1
- package/dist/meta-eval/index.d.ts +33 -36
- package/dist/meta-eval/index.js +2 -2
- package/dist/multishot/index.d.ts +1 -0
- package/dist/openapi.json +1 -1
- package/dist/pipelines/index.js +2 -2
- package/dist/reporting.d.ts +101 -80
- package/dist/reporting.js +7 -5
- package/dist/rl.d.ts +198 -142
- package/dist/rl.js +235 -127
- package/dist/rl.js.map +1 -1
- package/dist/rollout/index.d.ts +54 -67
- package/dist/rollout/index.js +6 -5
- package/dist/{run-campaign-FFRM3RH5.js → run-campaign-ISHFZ7FJ.js} +6 -3
- package/dist/supervisor-run/index.d.ts +7 -15
- package/dist/supervisor-run/index.js +2 -2
- package/dist/traces.d.ts +72 -50
- package/dist/traces.js +9 -6
- package/dist/wire/index.d.ts +1 -0
- package/dist/wire/index.js +2 -2
- package/docs/concepts.md +18 -1
- package/docs/customer-journeys.md +7 -6
- package/docs/hosted-ingest-spec.md +37 -40
- package/docs/insight-report.md +78 -7
- package/docs/research-report-methodology.md +20 -21
- package/docs/rollout.md +13 -8
- package/package.json +4 -3
- package/dist/chunk-474LBSOX.js.map +0 -1
- package/dist/chunk-7VYEGHWF.js.map +0 -1
- package/dist/chunk-ARU2PZFM.js.map +0 -1
- package/dist/chunk-BTACVO2W.js.map +0 -1
- package/dist/chunk-DPZAEKA6.js.map +0 -1
- package/dist/chunk-E3IADPLG.js.map +0 -1
- package/dist/chunk-H5UD2323.js.map +0 -1
- package/dist/chunk-IILEIWGW.js.map +0 -1
- package/dist/chunk-IPYXE555.js.map +0 -1
- package/dist/chunk-IR3KBHOY.js.map +0 -1
- package/dist/chunk-J5SQWP6Y.js.map +0 -1
- package/dist/chunk-K6IAZZ6L.js.map +0 -1
- package/dist/chunk-M7AH34KV.js.map +0 -1
- package/dist/chunk-MAX3TN3C.js.map +0 -1
- package/dist/chunk-MOXWMGPC.js.map +0 -1
- package/dist/chunk-OCFJACJU.js.map +0 -1
- package/dist/chunk-P22LJ3Y2.js.map +0 -1
- package/dist/chunk-PJQFMIOX.js.map +0 -1
- package/dist/chunk-VSMTAMNK.js +0 -53
- package/dist/chunk-VSMTAMNK.js.map +0 -1
- package/dist/chunk-ZVCHKKOP.js.map +0 -1
- package/dist/chunk-ZZUXHH3R.js +0 -99
- package/dist/chunk-ZZUXHH3R.js.map +0 -1
- /package/dist/{chunk-NUKSVU3W.js.map → chunk-5DTSBUL2.js.map} +0 -0
- /package/dist/{chunk-LUNF2SEL.js.map → chunk-DJKY2TSY.js.map} +0 -0
- /package/dist/{chunk-QOTFXW5L.js.map → chunk-P5W7RQKK.js.map} +0 -0
- /package/dist/{chunk-X4UCIOTZ.js.map → chunk-S5YLIBFX.js.map} +0 -0
- /package/dist/{chunk-CGG5SLH3.js.map → chunk-XPRT64IE.js.map} +0 -0
- /package/dist/{chunk-FO7HEH76.js.map → chunk-YJBNWCAA.js.map} +0 -0
- /package/dist/{chunk-KE2VWPZX.js.map → chunk-ZUUWPZCV.js.map} +0 -0
- /package/dist/{run-campaign-FFRM3RH5.js.map → run-campaign-ISHFZ7FJ.js.map} +0 -0
package/dist/contract/index.js
CHANGED
|
@@ -7,14 +7,14 @@ import {
|
|
|
7
7
|
fromPigraphSession,
|
|
8
8
|
observeCodeAgentSession,
|
|
9
9
|
parseCodeAgentJsonl
|
|
10
|
-
} from "../chunk-
|
|
10
|
+
} from "../chunk-SZLVEKMJ.js";
|
|
11
11
|
import {
|
|
12
12
|
createHostedClient
|
|
13
|
-
} from "../chunk-
|
|
13
|
+
} from "../chunk-DRYIUNWY.js";
|
|
14
14
|
import {
|
|
15
15
|
analyzeRuns,
|
|
16
16
|
summarizeExecution
|
|
17
|
-
} from "../chunk-
|
|
17
|
+
} from "../chunk-NACAGYSY.js";
|
|
18
18
|
import {
|
|
19
19
|
REFERENCE_EQUIVALENCE_INPUT_LIMITS,
|
|
20
20
|
REFERENCE_EQUIVALENCE_JUDGE_VERSION,
|
|
@@ -40,7 +40,7 @@ import {
|
|
|
40
40
|
skillOptOptimizationMethod,
|
|
41
41
|
surfaceContentHash,
|
|
42
42
|
surfaceHash
|
|
43
|
-
} from "../chunk-
|
|
43
|
+
} from "../chunk-NKAGIDE2.js";
|
|
44
44
|
import {
|
|
45
45
|
campaignSplitDigest,
|
|
46
46
|
createRunCostLedger,
|
|
@@ -48,47 +48,61 @@ import {
|
|
|
48
48
|
inMemoryCampaignStorage,
|
|
49
49
|
resolveRunDir,
|
|
50
50
|
runCampaign
|
|
51
|
-
} from "../chunk-
|
|
51
|
+
} from "../chunk-EZJEIH2R.js";
|
|
52
52
|
import {
|
|
53
53
|
buildDefaultAnalystRegistry,
|
|
54
54
|
createChatClient
|
|
55
|
-
} from "../chunk-
|
|
55
|
+
} from "../chunk-DJKY2TSY.js";
|
|
56
56
|
import "../chunk-HHWE3POT.js";
|
|
57
57
|
import "../chunk-WGXIEX7P.js";
|
|
58
58
|
import {
|
|
59
59
|
FileSystemOutcomeStore,
|
|
60
60
|
InMemoryOutcomeStore
|
|
61
61
|
} from "../chunk-3RF76KTD.js";
|
|
62
|
-
import "../chunk-
|
|
63
|
-
import
|
|
64
|
-
|
|
62
|
+
import "../chunk-NYLOYM6N.js";
|
|
63
|
+
import {
|
|
64
|
+
campaignCellExecutionEvidence,
|
|
65
|
+
campaignCellJudgeDimensions,
|
|
66
|
+
campaignCellTaskScore,
|
|
67
|
+
campaignCellToRunRecord
|
|
68
|
+
} from "../chunk-2MKQIFS4.js";
|
|
69
|
+
import "../chunk-PBE2LOSS.js";
|
|
70
|
+
import "../chunk-VLOATJQ2.js";
|
|
71
|
+
import "../chunk-DPUHNQLN.js";
|
|
65
72
|
import {
|
|
66
73
|
pairedBootstrap
|
|
67
|
-
} from "../chunk-
|
|
74
|
+
} from "../chunk-MHELPNRP.js";
|
|
68
75
|
import "../chunk-WS3NZZQQ.js";
|
|
69
76
|
import "../chunk-VI2UW6B6.js";
|
|
70
77
|
import {
|
|
78
|
+
readTaskFailureLabels,
|
|
71
79
|
recordAggregateMeasurements,
|
|
72
|
-
summarizeExecutionMeasurements
|
|
73
|
-
|
|
80
|
+
summarizeExecutionMeasurements,
|
|
81
|
+
summarizeTraceErrors
|
|
82
|
+
} from "../chunk-7ZZMD7UK.js";
|
|
74
83
|
import "../chunk-PXE2VKMX.js";
|
|
75
84
|
import "../chunk-ZET2UAYW.js";
|
|
76
85
|
import "../chunk-GGE4NNQT.js";
|
|
77
|
-
import
|
|
86
|
+
import {
|
|
87
|
+
classifyOtlpSpanRole,
|
|
88
|
+
isOtlpModelCall
|
|
89
|
+
} from "../chunk-P6FYH6K4.js";
|
|
78
90
|
import "../chunk-PC4UYEBM.js";
|
|
79
91
|
import {
|
|
92
|
+
modelHasSnapshot,
|
|
80
93
|
parseRunRecordSafe
|
|
81
|
-
} from "../chunk-
|
|
94
|
+
} from "../chunk-2JX3CFMB.js";
|
|
82
95
|
import "../chunk-MA6HLL3S.js";
|
|
83
|
-
import "../chunk-VSMTAMNK.js";
|
|
84
|
-
import "../chunk-ONWEPEDO.js";
|
|
85
96
|
import {
|
|
86
|
-
|
|
97
|
+
ValidationError
|
|
98
|
+
} from "../chunk-ONWEPEDO.js";
|
|
99
|
+
import {
|
|
100
|
+
LLM_MODEL_ATTR_KEYS,
|
|
101
|
+
SPAN_KIND_ATTR_KEYS
|
|
87
102
|
} from "../chunk-K4DBDHLK.js";
|
|
88
103
|
import "../chunk-PZ5AY32C.js";
|
|
89
104
|
|
|
90
105
|
// src/contract/self-improve.ts
|
|
91
|
-
import { createHash } from "crypto";
|
|
92
106
|
var SelfImproveRunError = class extends Error {
|
|
93
107
|
cost;
|
|
94
108
|
receipts;
|
|
@@ -343,11 +357,11 @@ async function runSelfImprove(opts, costLedger, startedAt, runDir, storage) {
|
|
|
343
357
|
findings: opts.findings,
|
|
344
358
|
selectionRankKey: opts.selectionRankKey
|
|
345
359
|
});
|
|
346
|
-
const
|
|
347
|
-
const
|
|
348
|
-
|
|
349
|
-
);
|
|
350
|
-
const winnerStats = meanComposite(
|
|
360
|
+
const reportSplit = holdoutDeferred ? "search" : "holdout";
|
|
361
|
+
const reportBaselineCampaign = holdoutDeferred ? result.baselineCampaign : result.baselineOnHoldout;
|
|
362
|
+
const reportWinnerCampaign = holdoutDeferred ? winnerSearchCampaign(result) : result.winnerOnHoldout;
|
|
363
|
+
const baseline = meanComposite(reportBaselineCampaign.aggregates.byScenario);
|
|
364
|
+
const winnerStats = meanComposite(reportWinnerCampaign.aggregates.byScenario);
|
|
351
365
|
let power;
|
|
352
366
|
const baselineHoldoutComposites = result.baselineOnHoldout.cells.filter((cell) => !cell.error).map((cell) => {
|
|
353
367
|
const scores = Object.values(cell.judgeScores);
|
|
@@ -390,12 +404,21 @@ async function runSelfImprove(opts, costLedger, startedAt, runDir, storage) {
|
|
|
390
404
|
const totalCost = cost.totalCostUsd;
|
|
391
405
|
const insight = await analyzeRuns({
|
|
392
406
|
runs: [
|
|
393
|
-
...cellsToRunRecords(result.baselineCampaign.cells, "baseline", runDir, opts.baselineSurface),
|
|
394
407
|
...cellsToRunRecords(
|
|
395
|
-
|
|
408
|
+
reportBaselineCampaign.cells,
|
|
409
|
+
"baseline",
|
|
410
|
+
runDir,
|
|
411
|
+
opts.baselineSurface,
|
|
412
|
+
reportSplit,
|
|
413
|
+
opts.model
|
|
414
|
+
),
|
|
415
|
+
...cellsToRunRecords(
|
|
416
|
+
reportWinnerCampaign.cells,
|
|
396
417
|
"winner",
|
|
397
418
|
runDir,
|
|
398
|
-
result.winnerSurface
|
|
419
|
+
result.winnerSurface,
|
|
420
|
+
reportSplit,
|
|
421
|
+
opts.model
|
|
399
422
|
)
|
|
400
423
|
],
|
|
401
424
|
baselineCandidateId: "baseline",
|
|
@@ -468,19 +491,21 @@ async function shipEvalRunToHosted(tenant, opts, summary, raw, runDir) {
|
|
|
468
491
|
const client = createHostedClient(tenant);
|
|
469
492
|
function snapshotFromCampaign(index, surface, campaign, durationMs) {
|
|
470
493
|
const cells = campaign.cells.map((cell) => {
|
|
471
|
-
const
|
|
472
|
-
const composite = judgeScores.length === 0 ? 0 : judgeScores.reduce((s, j) => s + j.composite, 0) / judgeScores.length;
|
|
494
|
+
const execution = campaignCellExecutionEvidence(cell);
|
|
473
495
|
return {
|
|
474
496
|
scenarioId: cell.scenarioId,
|
|
475
497
|
rep: cell.rep,
|
|
476
|
-
compositeMean:
|
|
477
|
-
dimensions:
|
|
478
|
-
|
|
479
|
-
|
|
498
|
+
compositeMean: campaignCellTaskScore(cell) ?? null,
|
|
499
|
+
dimensions: campaignCellJudgeDimensions(cell),
|
|
500
|
+
terminalOutcome: execution.terminalOutcome,
|
|
501
|
+
executionErrorCount: execution.executionErrorCount ?? null,
|
|
480
502
|
errorMessage: cell.error ?? void 0
|
|
481
503
|
};
|
|
482
504
|
});
|
|
483
|
-
const
|
|
505
|
+
const scoredCells = cells.flatMap(
|
|
506
|
+
(cell) => cell.compositeMean === null ? [] : [cell.compositeMean]
|
|
507
|
+
);
|
|
508
|
+
const compositeMean = scoredCells.length === 0 ? null : scoredCells.reduce((sum, score) => sum + score, 0) / scoredCells.length;
|
|
484
509
|
return {
|
|
485
510
|
index,
|
|
486
511
|
surfaceHash: surfaceHash(surface),
|
|
@@ -531,66 +556,33 @@ function hashString(s) {
|
|
|
531
556
|
}
|
|
532
557
|
return h.toString(16).padStart(8, "0");
|
|
533
558
|
}
|
|
534
|
-
function cellsToRunRecords(cells, candidateId, runId, surface) {
|
|
559
|
+
function cellsToRunRecords(cells, candidateId, runId, surface, splitTag, fallbackModel) {
|
|
535
560
|
const promptHash = surfaceContentHash(surface);
|
|
536
|
-
const configHash =
|
|
561
|
+
const configHash = surfaceContentHash(candidateId);
|
|
537
562
|
return cells.map((cell) => {
|
|
538
|
-
const
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
perJudge[judgeId] = { ...score.dimensions };
|
|
544
|
-
for (const [dim, value] of Object.entries(score.dimensions)) {
|
|
545
|
-
if (!Number.isFinite(value)) continue;
|
|
546
|
-
const accum = perDimMeanAccum[dim] ?? { sum: 0, n: 0 };
|
|
547
|
-
accum.sum += value;
|
|
548
|
-
accum.n += 1;
|
|
549
|
-
perDimMeanAccum[dim] = accum;
|
|
550
|
-
}
|
|
551
|
-
if (Number.isFinite(score.composite)) {
|
|
552
|
-
compositeSum += score.composite;
|
|
553
|
-
compositeCount += 1;
|
|
554
|
-
}
|
|
563
|
+
const model = cell.resolvedModel ?? fallbackModel;
|
|
564
|
+
if (!model) {
|
|
565
|
+
throw new ValidationError(
|
|
566
|
+
`selfImprove.model is required when cell ${cell.cellId} has no paid-call model receipt`
|
|
567
|
+
);
|
|
555
568
|
}
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
569
|
+
if (!modelHasSnapshot(model)) {
|
|
570
|
+
throw new ValidationError(
|
|
571
|
+
`selfImprove model "${model}" lacks a snapshot version for cell ${cell.cellId}`
|
|
572
|
+
);
|
|
559
573
|
}
|
|
560
|
-
|
|
561
|
-
const judgeScores = {
|
|
562
|
-
perJudge,
|
|
563
|
-
perDimMean,
|
|
564
|
-
composite
|
|
565
|
-
};
|
|
566
|
-
return {
|
|
574
|
+
return campaignCellToRunRecord(cell, {
|
|
567
575
|
runId: `${runId}::${candidateId}::${cell.cellId}`,
|
|
568
576
|
experimentId: runId,
|
|
569
577
|
candidateId,
|
|
570
|
-
//
|
|
571
|
-
// Synthesize a stable seed for that pairing.
|
|
578
|
+
// scenarioId is explicit; seed keeps repeated runs distinct.
|
|
572
579
|
seed: cell.rep * 1e6 + hashString(cell.scenarioId).slice(0, 6).split("").reduce((a, c) => a * 31 + c.charCodeAt(0) >>> 0, 0),
|
|
573
|
-
model
|
|
580
|
+
model,
|
|
574
581
|
promptHash,
|
|
575
582
|
configHash,
|
|
576
583
|
commitSha: "cell",
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
tokenUsage: {
|
|
580
|
-
input: cell.tokenUsage.input,
|
|
581
|
-
output: cell.tokenUsage.output,
|
|
582
|
-
...cell.tokenUsage.reasoning === void 0 ? {} : { reasoning: cell.tokenUsage.reasoning },
|
|
583
|
-
...cell.tokenUsage.cached === void 0 ? {} : { cached: cell.tokenUsage.cached },
|
|
584
|
-
...cell.tokenUsage.cacheWrite === void 0 ? {} : { cacheWrite: cell.tokenUsage.cacheWrite }
|
|
585
|
-
},
|
|
586
|
-
outcome: {
|
|
587
|
-
holdoutScore: composite,
|
|
588
|
-
raw: {},
|
|
589
|
-
judgeScores
|
|
590
|
-
},
|
|
591
|
-
splitTag: "holdout",
|
|
592
|
-
...cell.error ? { failureMode: cell.error } : {}
|
|
593
|
-
};
|
|
584
|
+
splitTag
|
|
585
|
+
});
|
|
594
586
|
});
|
|
595
587
|
}
|
|
596
588
|
|
|
@@ -1565,7 +1557,7 @@ function diffGenerations(before, after) {
|
|
|
1565
1557
|
rep: beforeCell.rep,
|
|
1566
1558
|
compositeBefore: beforeCell.compositeMean,
|
|
1567
1559
|
compositeAfter: afterCell.compositeMean,
|
|
1568
|
-
compositeDelta: afterCell.compositeMean - beforeCell.compositeMean,
|
|
1560
|
+
compositeDelta: beforeCell.compositeMean === null || afterCell.compositeMean === null ? null : afterCell.compositeMean - beforeCell.compositeMean,
|
|
1569
1561
|
dimensions: diffDimensions(beforeCell.dimensions, afterCell.dimensions)
|
|
1570
1562
|
});
|
|
1571
1563
|
}
|
|
@@ -1583,7 +1575,7 @@ function diffGenerations(before, after) {
|
|
|
1583
1575
|
added,
|
|
1584
1576
|
compositeBefore: before.compositeMean,
|
|
1585
1577
|
compositeAfter: after.compositeMean,
|
|
1586
|
-
compositeDelta: after.compositeMean - before.compositeMean,
|
|
1578
|
+
compositeDelta: before.compositeMean === null || after.compositeMean === null ? null : after.compositeMean - before.compositeMean,
|
|
1587
1579
|
costUsdBefore: before.costUsd,
|
|
1588
1580
|
costUsdAfter: after.costUsd,
|
|
1589
1581
|
costUsdDelta: after.costUsd - before.costUsd,
|
|
@@ -1736,14 +1728,13 @@ function fromFeedbackTable(opts) {
|
|
|
1736
1728
|
perDimMean: { rating: meanScore },
|
|
1737
1729
|
composite: meanScore
|
|
1738
1730
|
};
|
|
1731
|
+
const splitTag = runMeta.splitTag ?? "holdout";
|
|
1739
1732
|
const outcome = {
|
|
1740
|
-
|
|
1741
|
-
// `holdoutScore` so downstream substrate primitives (`paretoChart`,
|
|
1742
|
-
// promotion gates) read it correctly by default.
|
|
1743
|
-
holdoutScore: meanScore,
|
|
1733
|
+
...splitTag === "holdout" ? { holdoutScore: meanScore } : { searchScore: meanScore },
|
|
1744
1734
|
raw: Object.fromEntries(normalised.map((r) => [`rater:${r.rater}`, r.score])),
|
|
1745
1735
|
judgeScores
|
|
1746
1736
|
};
|
|
1737
|
+
const costUsd = runMeta.costUsd ?? null;
|
|
1747
1738
|
runs.push({
|
|
1748
1739
|
runId,
|
|
1749
1740
|
experimentId: runMeta.experimentId ?? "feedback-corpus",
|
|
@@ -1754,10 +1745,13 @@ function fromFeedbackTable(opts) {
|
|
|
1754
1745
|
configHash: runMeta.configHash ?? "sha256:unknown",
|
|
1755
1746
|
commitSha: runMeta.commitSha ?? "unknown",
|
|
1756
1747
|
wallMs: runMeta.wallMs ?? 0,
|
|
1757
|
-
costUsd
|
|
1748
|
+
costUsd,
|
|
1749
|
+
costProvenance: costUsd === null ? { kind: "uncaptured", usd: null } : { kind: "observed", usd: costUsd },
|
|
1758
1750
|
tokenUsage: { input: 0, output: 0 },
|
|
1751
|
+
terminalOutcome: "unknown",
|
|
1759
1752
|
outcome,
|
|
1760
|
-
splitTag
|
|
1753
|
+
splitTag,
|
|
1754
|
+
scenarioId: runMeta.scenarioId ?? runId
|
|
1761
1755
|
});
|
|
1762
1756
|
if (emitRaterScores) {
|
|
1763
1757
|
for (const r of normalised) raterScores.push({ runId, rater: r.rater, score: r.score });
|
|
@@ -1767,7 +1761,12 @@ function fromFeedbackTable(opts) {
|
|
|
1767
1761
|
}
|
|
1768
1762
|
|
|
1769
1763
|
// src/contract/intake/otel-spans.ts
|
|
1770
|
-
var
|
|
1764
|
+
var TASK_SCORE_ATTR_KEYS = [
|
|
1765
|
+
"gen_ai.evaluation.score.value",
|
|
1766
|
+
"tangle.task.score",
|
|
1767
|
+
"eval.score",
|
|
1768
|
+
"tangle.score"
|
|
1769
|
+
];
|
|
1771
1770
|
var MODEL_KEYS = ["tangle.model", ...LLM_MODEL_ATTR_KEYS, "model"];
|
|
1772
1771
|
var PROMPT_HASH_KEYS = ["tangle.prompt_hash", "prompt.hash"];
|
|
1773
1772
|
var CONFIG_HASH_KEYS = ["tangle.config_hash", "config.hash"];
|
|
@@ -1789,15 +1788,35 @@ function fromOtelSpans(opts) {
|
|
|
1789
1788
|
);
|
|
1790
1789
|
const callSpanIds = new Set(measurements.callSpanIds);
|
|
1791
1790
|
const callSpans = groupSpans2.filter((span) => callSpanIds.has(span.spanId));
|
|
1792
|
-
const wallMs =
|
|
1791
|
+
const wallMs = unixNanoDurationMs(root.startTimeUnixNano, root.endTimeUnixNano);
|
|
1793
1792
|
const model = readAttrString(callSpans, MODEL_KEYS) ?? readAttrString(groupSpans2, MODEL_KEYS) ?? "unknown@unknown";
|
|
1794
1793
|
const capturedCost = (measurements.cost.complete ? measurements.cost.value : void 0) ?? measurements.aggregate?.costUsd;
|
|
1795
|
-
const costUsd = capturedCost ??
|
|
1794
|
+
const costUsd = capturedCost ?? null;
|
|
1795
|
+
const scenarioId = readConsistentScenarioId(groupKey, groupSpans2) ?? groupKey;
|
|
1796
1796
|
const promptHash = readAttrString(groupSpans2, PROMPT_HASH_KEYS) ?? "sha256:unknown";
|
|
1797
1797
|
const configHash = readAttrString(groupSpans2, CONFIG_HASH_KEYS) ?? "sha256:unknown";
|
|
1798
|
-
const score =
|
|
1798
|
+
const score = resolveTaskScore(groupKey, groupSpans2, opts.scoreForRun);
|
|
1799
|
+
const taskFailure = readTaskFailureLabels(
|
|
1800
|
+
groupSpans2.filter((span) => !span.parentSpanId && isTerminalRootCandidate(span)),
|
|
1801
|
+
`fromOtelSpans: run '${groupKey}'`
|
|
1802
|
+
);
|
|
1799
1803
|
const rawNumeric = collectNumericAttrs(groupSpans2);
|
|
1800
|
-
|
|
1804
|
+
const errorSummary = summarizeTraceErrors(
|
|
1805
|
+
groupSpans2.map((span) => ({
|
|
1806
|
+
id: spanIdentity(span),
|
|
1807
|
+
...span.parentSpanId ? { parentId: parentIdentity(span) } : {},
|
|
1808
|
+
role: errorRoleForSpan(span),
|
|
1809
|
+
error: span.status?.code === "ERROR",
|
|
1810
|
+
processRoot: !span.parentSpanId && isTerminalRootCandidate(span)
|
|
1811
|
+
}))
|
|
1812
|
+
);
|
|
1813
|
+
rawNumeric.error_span_count = errorSummary.total;
|
|
1814
|
+
rawNumeric.execution_error_count = errorSummary.execution;
|
|
1815
|
+
rawNumeric.process_error_count = errorSummary.process;
|
|
1816
|
+
rawNumeric.guardrail_error_count = errorSummary.guardrail;
|
|
1817
|
+
rawNumeric.judge_error_count = errorSummary.evaluation;
|
|
1818
|
+
rawNumeric.propagated_error_count = errorSummary.propagated;
|
|
1819
|
+
rawNumeric.unclassified_error_count = errorSummary.unclassified;
|
|
1801
1820
|
rawNumeric.llm_span_count = measurements.modelCallCount;
|
|
1802
1821
|
if (measurements.cost.value !== void 0 && !measurements.cost.complete) {
|
|
1803
1822
|
rawNumeric.partial_observed_cost_usd = measurements.cost.value;
|
|
@@ -1808,12 +1827,18 @@ function fromOtelSpans(opts) {
|
|
|
1808
1827
|
perDimMean: { score },
|
|
1809
1828
|
composite: score
|
|
1810
1829
|
} : void 0;
|
|
1811
|
-
const
|
|
1830
|
+
const terminalOutcome = terminalOutcomeFromRoots(groupSpans2);
|
|
1831
|
+
const failedRoot = terminalOutcome === "failed" ? groupSpans2.find(
|
|
1832
|
+
(span) => !span.parentSpanId && isTerminalRootCandidate(span) && span.status?.code === "ERROR"
|
|
1833
|
+
) : void 0;
|
|
1812
1834
|
const outcome = {
|
|
1813
|
-
...opts.defaultSplit === "search" ? { searchScore: score } : { holdoutScore: score },
|
|
1814
1835
|
raw: rawNumeric,
|
|
1815
1836
|
...judgeScores ? { judgeScores } : {}
|
|
1816
1837
|
};
|
|
1838
|
+
if (score !== void 0) {
|
|
1839
|
+
if (defaultSplit === "holdout") outcome.holdoutScore = score;
|
|
1840
|
+
else outcome.searchScore = score;
|
|
1841
|
+
}
|
|
1817
1842
|
runs.push({
|
|
1818
1843
|
runId: groupKey,
|
|
1819
1844
|
experimentId,
|
|
@@ -1827,22 +1852,56 @@ function fromOtelSpans(opts) {
|
|
|
1827
1852
|
costUsd,
|
|
1828
1853
|
costProvenance: capturedCost === void 0 ? { kind: "uncaptured", usd: null } : { kind: "observed", usd: capturedCost },
|
|
1829
1854
|
tokenUsage: measurements.tokenUsage,
|
|
1855
|
+
terminalOutcome,
|
|
1856
|
+
...failedRoot ? { terminalFailureReason: failedRoot.status?.message ?? failedRoot.name } : {},
|
|
1830
1857
|
outcome,
|
|
1858
|
+
...taskFailure,
|
|
1831
1859
|
splitTag: defaultSplit,
|
|
1832
|
-
|
|
1860
|
+
scenarioId
|
|
1833
1861
|
});
|
|
1834
1862
|
}
|
|
1835
1863
|
return runs;
|
|
1836
1864
|
}
|
|
1865
|
+
function terminalOutcomeFromRoots(spans) {
|
|
1866
|
+
const roots = spans.filter((span) => !span.parentSpanId && isTerminalRootCandidate(span));
|
|
1867
|
+
if (roots.length !== 1) return "unknown";
|
|
1868
|
+
if (roots[0].status?.code === "OK") return "succeeded";
|
|
1869
|
+
if (roots[0].status?.code === "ERROR") return "failed";
|
|
1870
|
+
return "unknown";
|
|
1871
|
+
}
|
|
1872
|
+
function isTerminalRootCandidate(span) {
|
|
1873
|
+
const role = errorRoleForSpan(span);
|
|
1874
|
+
if (role === "LLM" || role === "TOOL" || role === "GUARDRAIL" || role === "EVALUATOR") {
|
|
1875
|
+
return false;
|
|
1876
|
+
}
|
|
1877
|
+
return true;
|
|
1878
|
+
}
|
|
1879
|
+
function readSpanKind(span) {
|
|
1880
|
+
return readAttrString([span], [...SPAN_KIND_ATTR_KEYS, "span.kind"])?.toUpperCase();
|
|
1881
|
+
}
|
|
1882
|
+
function errorRoleForSpan(span) {
|
|
1883
|
+
return classifyOtlpSpanRole({
|
|
1884
|
+
kind: readSpanKind(span),
|
|
1885
|
+
name: span.name,
|
|
1886
|
+
attributes: span.attributes
|
|
1887
|
+
});
|
|
1888
|
+
}
|
|
1889
|
+
function spanIdentity(span) {
|
|
1890
|
+
return `${span.traceId}:${span.spanId}`;
|
|
1891
|
+
}
|
|
1892
|
+
function parentIdentity(span) {
|
|
1893
|
+
return `${span.traceId}:${span.parentSpanId}`;
|
|
1894
|
+
}
|
|
1837
1895
|
function isExplicitModelCall(span) {
|
|
1838
|
-
|
|
1839
|
-
|
|
1840
|
-
|
|
1841
|
-
|
|
1896
|
+
return isOtlpModelCall({
|
|
1897
|
+
kind: readSpanKind(span),
|
|
1898
|
+
name: span.name,
|
|
1899
|
+
attributes: span.attributes
|
|
1900
|
+
});
|
|
1842
1901
|
}
|
|
1843
1902
|
function isExplicitAggregate(span) {
|
|
1844
|
-
const kind = span
|
|
1845
|
-
return
|
|
1903
|
+
const kind = readSpanKind(span);
|
|
1904
|
+
return kind !== void 0 && kind !== "LLM";
|
|
1846
1905
|
}
|
|
1847
1906
|
function groupSpans(spans) {
|
|
1848
1907
|
const m = /* @__PURE__ */ new Map();
|
|
@@ -1855,7 +1914,10 @@ function groupSpans(spans) {
|
|
|
1855
1914
|
return m;
|
|
1856
1915
|
}
|
|
1857
1916
|
function findRoot(group) {
|
|
1858
|
-
|
|
1917
|
+
const structuralRoots = group.filter((span) => !span.parentSpanId);
|
|
1918
|
+
const terminalRoots = structuralRoots.filter(isTerminalRootCandidate);
|
|
1919
|
+
const pool = terminalRoots.length > 0 ? terminalRoots : structuralRoots.length > 0 ? structuralRoots : group;
|
|
1920
|
+
return orderSpans(pool)[0];
|
|
1859
1921
|
}
|
|
1860
1922
|
function readAttrString(spans, keys) {
|
|
1861
1923
|
for (const span of spans) {
|
|
@@ -1866,18 +1928,102 @@ function readAttrString(spans, keys) {
|
|
|
1866
1928
|
}
|
|
1867
1929
|
return void 0;
|
|
1868
1930
|
}
|
|
1869
|
-
function
|
|
1931
|
+
function readConsistentScenarioId(runId, spans) {
|
|
1932
|
+
const values = /* @__PURE__ */ new Set();
|
|
1870
1933
|
for (const span of spans) {
|
|
1871
|
-
|
|
1872
|
-
|
|
1873
|
-
|
|
1874
|
-
|
|
1875
|
-
|
|
1876
|
-
|
|
1934
|
+
const topLevel = span["tangle.scenarioId"];
|
|
1935
|
+
if (typeof topLevel === "string" && topLevel.length > 0) values.add(topLevel);
|
|
1936
|
+
const attribute = span.attributes["tangle.scenarioId"];
|
|
1937
|
+
if (typeof attribute === "string" && attribute.length > 0) values.add(attribute);
|
|
1938
|
+
}
|
|
1939
|
+
if (values.size > 1) {
|
|
1940
|
+
throw new ValidationError(
|
|
1941
|
+
`fromOtelSpans: conflicting scenario ids for run '${runId}': ${[...values].sort().join(", ")}`
|
|
1942
|
+
);
|
|
1943
|
+
}
|
|
1944
|
+
return values.values().next().value;
|
|
1945
|
+
}
|
|
1946
|
+
function resolveTaskScore(runId, spans, scoreForRun) {
|
|
1947
|
+
const orderedSpans = orderSpans(spans);
|
|
1948
|
+
const sources = [];
|
|
1949
|
+
if (scoreForRun) {
|
|
1950
|
+
const supplied = scoreForRun(runId, orderedSpans);
|
|
1951
|
+
if (supplied !== void 0) {
|
|
1952
|
+
if (typeof supplied !== "number" || !Number.isFinite(supplied)) {
|
|
1953
|
+
throw new ValidationError(
|
|
1954
|
+
`fromOtelSpans: scoreForRun returned a non-finite number for run '${runId}'`
|
|
1955
|
+
);
|
|
1877
1956
|
}
|
|
1957
|
+
sources.push({ label: "scoreForRun", value: supplied });
|
|
1878
1958
|
}
|
|
1879
1959
|
}
|
|
1880
|
-
|
|
1960
|
+
for (const span of orderedSpans) {
|
|
1961
|
+
const role = errorRoleForSpan(span);
|
|
1962
|
+
if (span.parentSpanId && role !== "EVALUATOR") continue;
|
|
1963
|
+
if (role === "EVALUATOR" && span.status?.code === "ERROR") continue;
|
|
1964
|
+
for (const key of TASK_SCORE_ATTR_KEYS) {
|
|
1965
|
+
if (!Object.hasOwn(span.attributes, key)) continue;
|
|
1966
|
+
sources.push({
|
|
1967
|
+
label: `span '${span.spanId}' attribute '${key}'`,
|
|
1968
|
+
value: parseTaskScoreAttribute(runId, span.spanId, key, span.attributes[key])
|
|
1969
|
+
});
|
|
1970
|
+
}
|
|
1971
|
+
}
|
|
1972
|
+
if (sources.length === 0) return void 0;
|
|
1973
|
+
sources.sort((left, right) => left.label.localeCompare(right.label));
|
|
1974
|
+
const score = sources[0].value;
|
|
1975
|
+
if (sources.some((source) => source.value !== score)) {
|
|
1976
|
+
const details = sources.map((source) => `${source.label}=${source.value}`).join(", ");
|
|
1977
|
+
throw new ValidationError(
|
|
1978
|
+
`fromOtelSpans: conflicting task-quality scores for run '${runId}': ${details}`
|
|
1979
|
+
);
|
|
1980
|
+
}
|
|
1981
|
+
return score;
|
|
1982
|
+
}
|
|
1983
|
+
function parseTaskScoreAttribute(runId, spanId, key, value) {
|
|
1984
|
+
const source = `span '${spanId}' attribute '${key}'`;
|
|
1985
|
+
if (typeof value === "string") {
|
|
1986
|
+
if (value.trim().length === 0) {
|
|
1987
|
+
throw new ValidationError(
|
|
1988
|
+
`fromOtelSpans: ${source} is blank for run '${runId}'; task quality must be finite`
|
|
1989
|
+
);
|
|
1990
|
+
}
|
|
1991
|
+
const parsed = Number(value);
|
|
1992
|
+
if (Number.isFinite(parsed)) return parsed;
|
|
1993
|
+
} else if (typeof value === "number" && Number.isFinite(value)) {
|
|
1994
|
+
return value;
|
|
1995
|
+
}
|
|
1996
|
+
throw new ValidationError(
|
|
1997
|
+
`fromOtelSpans: ${source} is not a finite task-quality score for run '${runId}'`
|
|
1998
|
+
);
|
|
1999
|
+
}
|
|
2000
|
+
function orderSpans(spans) {
|
|
2001
|
+
return [...spans].sort(
|
|
2002
|
+
(left, right) => compareUnixNano(left.startTimeUnixNano, right.startTimeUnixNano) || left.spanId.localeCompare(right.spanId)
|
|
2003
|
+
);
|
|
2004
|
+
}
|
|
2005
|
+
function parseUnixNano(value) {
|
|
2006
|
+
try {
|
|
2007
|
+
return BigInt(value);
|
|
2008
|
+
} catch {
|
|
2009
|
+
throw new ValidationError(`fromOtelSpans: invalid Unix nanosecond timestamp '${value}'`);
|
|
2010
|
+
}
|
|
2011
|
+
}
|
|
2012
|
+
function compareUnixNano(left, right) {
|
|
2013
|
+
const leftValue = parseUnixNano(left);
|
|
2014
|
+
const rightValue = parseUnixNano(right);
|
|
2015
|
+
return leftValue < rightValue ? -1 : leftValue > rightValue ? 1 : 0;
|
|
2016
|
+
}
|
|
2017
|
+
function unixNanoDurationMs(start, end) {
|
|
2018
|
+
const delta = parseUnixNano(end) - parseUnixNano(start);
|
|
2019
|
+
if (delta <= 0n) return 0;
|
|
2020
|
+
const wholeMs = delta / 1000000n;
|
|
2021
|
+
const fractionalMs = delta % 1000000n;
|
|
2022
|
+
const value = Number(wholeMs) + Number(fractionalMs) / 1e6;
|
|
2023
|
+
if (!Number.isSafeInteger(Number(wholeMs))) {
|
|
2024
|
+
throw new ValidationError("fromOtelSpans: span duration exceeds the safe millisecond range");
|
|
2025
|
+
}
|
|
2026
|
+
return value;
|
|
1881
2027
|
}
|
|
1882
2028
|
function collectNumericAttrs(spans) {
|
|
1883
2029
|
const raw = {};
|