@tangle-network/agent-eval 0.132.0 → 0.133.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/README.md +1 -1
- package/dist/agent-profile-cell-OhuTee9n.js +335 -0
- package/dist/agent-profile-cell-OhuTee9n.js.map +1 -0
- package/dist/analyst/index.js +3 -3
- package/dist/{analyze-runs-BjPn_fOS.js → analyze-runs-B-afTpCv.js} +6 -19
- package/dist/analyze-runs-B-afTpCv.js.map +1 -0
- package/dist/{analyze-runs-AFDI5RI0.d.ts → analyze-runs-DZr7JW-m.d.ts} +2 -2
- package/dist/{analyze-runs-AFDI5RI0.d.ts.map → analyze-runs-DZr7JW-m.d.ts.map} +1 -1
- package/dist/{baseline-HsBvw_dk.js → baseline-DcX5hQDv.js} +2 -70
- package/dist/{baseline-HsBvw_dk.js.map → baseline-DcX5hQDv.js.map} +1 -1
- package/dist/baseline-hG3K85h4.d.ts.map +1 -1
- package/dist/benchmarks/index.d.ts +1 -1
- package/dist/benchmarks/index.js +1 -1
- package/dist/{benchmarks-DTrT3UH-.js → benchmarks-BU7P6PCW.js} +3 -3
- package/dist/{benchmarks-DTrT3UH-.js.map → benchmarks-BU7P6PCW.js.map} +1 -1
- package/dist/builder-eval/index.js +1 -1
- package/dist/campaign/index.d.ts +3 -3
- package/dist/campaign/index.js +2 -2
- package/dist/{campaign-Cx6CfMR4.js → campaign-CnzHQndg.js} +21 -15
- package/dist/{campaign-Cx6CfMR4.js.map → campaign-CnzHQndg.js.map} +1 -1
- package/dist/cli.js +1 -1
- package/dist/{client-aZDHJiKO.d.ts → client-D4F9hdzR.d.ts} +2 -2
- package/dist/{client-aZDHJiKO.d.ts.map → client-D4F9hdzR.d.ts.map} +1 -1
- package/dist/contract/index.d.ts +145 -6
- package/dist/contract/index.d.ts.map +1 -1
- package/dist/contract/index.js +1395 -11
- package/dist/contract/index.js.map +1 -1
- package/dist/control.js +1 -1
- package/dist/{cost-ledger-ZAa_P4r0.js → cost-ledger-BrJxbrMy.js} +238 -3
- package/dist/cost-ledger-BrJxbrMy.js.map +1 -0
- package/dist/{default-registry-B1JcpnRv.js → default-registry-D3T9XbuY.js} +3 -3
- package/dist/{default-registry-B1JcpnRv.js.map → default-registry-D3T9XbuY.js.map} +1 -1
- package/dist/{eval-campaign-mDKhkdUq.js → eval-campaign-DXhpZghy.js} +5 -6
- package/dist/{eval-campaign-mDKhkdUq.js.map → eval-campaign-DXhpZghy.js.map} +1 -1
- package/dist/{task-failure-attributes-CQZlB3et.js → extract-usage-2j25whHw.js} +154 -2
- package/dist/extract-usage-2j25whHw.js.map +1 -0
- package/dist/fuzz.js +1 -1
- package/dist/hosted/index.d.ts +1 -1
- package/dist/{index-3cdlURSk2.d.ts → index-3cdlURSk.d.ts} +1 -1
- package/dist/index-3cdlURSk.d.ts.map +1 -0
- package/dist/{index-FpfWFsKm.d.ts → index-Ba636PKl.d.ts} +29 -6
- package/dist/{index-FpfWFsKm.d.ts.map → index-Ba636PKl.d.ts.map} +1 -1
- package/dist/{index-p2TR_iWJ.d.ts → index-Wek5mU0y.d.ts} +3 -3
- package/dist/{index-p2TR_iWJ.d.ts.map → index-Wek5mU0y.d.ts.map} +1 -1
- package/dist/{index-BAvgST_9.d.ts → index-nhIYz9hn.d.ts} +67 -9
- package/dist/index-nhIYz9hn.d.ts.map +1 -0
- package/dist/index.d.ts +55 -9
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +111 -24
- package/dist/index.js.map +1 -1
- package/dist/ledger-core/index.d.ts +2 -2
- package/dist/ledger-core/index.js +2 -2
- package/dist/ledger-core-CPZfcrC2.js +620 -0
- package/dist/ledger-core-CPZfcrC2.js.map +1 -0
- package/dist/{llm-client-BNcP4v08.js → llm-client-ClPW-dWB.js} +2 -2
- package/dist/{llm-client-BNcP4v08.js.map → llm-client-ClPW-dWB.js.map} +1 -1
- package/dist/meta-eval/index.d.ts +215 -2
- package/dist/{index-CXs7QlR5.d.ts.map → meta-eval/index.d.ts.map} +1 -1
- package/dist/meta-eval/index.js +93 -3
- package/dist/meta-eval/index.js.map +1 -1
- package/dist/{mint-D5_87M5L.js → mint-BvkwcYZU.js} +2 -2
- package/dist/{mint-D5_87M5L.js.map → mint-BvkwcYZU.js.map} +1 -1
- package/dist/openapi.json +1 -1
- package/dist/{paired-arms-D9D0wXj2.js → paired-arms-6XItKzd1.js} +2 -2
- package/dist/{paired-arms-D9D0wXj2.js.map → paired-arms-6XItKzd1.js.map} +1 -1
- package/dist/pipelines/index.js +2 -2
- package/dist/profile-cell.js +1 -242
- package/dist/{propose-review-control-Bqb7daEJ.js → propose-review-control-SQ-n9-We.js} +2 -2
- package/dist/{propose-review-control-Bqb7daEJ.js.map → propose-review-control-SQ-n9-We.js.map} +1 -1
- package/dist/{release-report-oWt9f2k-.js → release-report-wuilQkvK.js} +3 -3
- package/dist/{release-report-oWt9f2k-.js.map → release-report-wuilQkvK.js.map} +1 -1
- package/dist/{replay-D18-pBAA.js → replay-CJfGLdx4.js} +4 -5
- package/dist/{replay-D18-pBAA.js.map → replay-CJfGLdx4.js.map} +1 -1
- package/dist/reporting.d.ts +1 -1
- package/dist/reporting.js +4 -4
- package/dist/{reward-hacking-BEvjdUtD.js → reward-hacking-Dl2UBzej.js} +3 -3
- package/dist/{reward-hacking-BEvjdUtD.js.map → reward-hacking-Dl2UBzej.js.map} +1 -1
- package/dist/rl.d.ts +153 -3
- package/dist/rl.d.ts.map +1 -1
- package/dist/rl.js +222 -7
- package/dist/rl.js.map +1 -1
- package/dist/rollout/index.d.ts +1 -1
- package/dist/rollout/index.js +2 -2
- package/dist/{rollout-VEo41J0N.js → rollout-CeTlDrf6.js} +2 -2
- package/dist/{rollout-VEo41J0N.js.map → rollout-CeTlDrf6.js.map} +1 -1
- package/dist/{rubric-predictive-validity-B3xmbmS1.js → rubric-predictive-validity-QG7ydk0s.js} +2 -2
- package/dist/{rubric-predictive-validity-B3xmbmS1.js.map → rubric-predictive-validity-QG7ydk0s.js.map} +1 -1
- package/dist/{run-record-CN8Zd21B.js → run-record-BIwU2wdV.js} +2 -2
- package/dist/{run-record-CN8Zd21B.js.map → run-record-BIwU2wdV.js.map} +1 -1
- package/dist/{semantic-concept-judge-DKCtoOz8.js → semantic-concept-judge-BypLt6Fw.js} +4 -4
- package/dist/{semantic-concept-judge-DKCtoOz8.js.map → semantic-concept-judge-BypLt6Fw.js.map} +1 -1
- package/dist/{server-Dc_lsOYd.js → server-BPqlDBWK.js} +3 -3
- package/dist/{server-Dc_lsOYd.js.map → server-BPqlDBWK.js.map} +1 -1
- package/dist/{skillopt-optimization-method-CQlz8GQM.js → skillopt-optimization-method-BoIzh7Dl.js} +6 -6
- package/dist/{skillopt-optimization-method-CQlz8GQM.js.map → skillopt-optimization-method-BoIzh7Dl.js.map} +1 -1
- package/dist/{skillopt-optimization-method-C9M_lxdo.d.ts → skillopt-optimization-method-CAASpcS3.d.ts} +3 -3
- package/dist/{skillopt-optimization-method-C9M_lxdo.d.ts.map → skillopt-optimization-method-CAASpcS3.d.ts.map} +1 -1
- package/dist/{statistics-CnnxdpOg.js → statistics-DWM_AyLe.js} +90 -76
- package/dist/statistics-DWM_AyLe.js.map +1 -0
- package/dist/statistics-DbvkkDPa.d.ts.map +1 -1
- package/dist/{summary-report-BNs5nmXI.js → summary-report-Ci17nIdU.js} +4 -4
- package/dist/{summary-report-BNs5nmXI.js.map → summary-report-Ci17nIdU.js.map} +1 -1
- package/dist/traces.js +2 -2
- package/dist/wire/index.js +1 -1
- package/package.json +2 -7
- package/dist/analyze-runs-BjPn_fOS.js.map +0 -1
- package/dist/belief-state/index.d.ts +0 -622
- package/dist/belief-state/index.d.ts.map +0 -1
- package/dist/belief-state/index.js +0 -1819
- package/dist/belief-state/index.js.map +0 -1
- package/dist/calibration-CNWWA6K8.js +0 -94
- package/dist/calibration-CNWWA6K8.js.map +0 -1
- package/dist/code-agent-session-BjkMTQ7H.js +0 -1390
- package/dist/code-agent-session-BjkMTQ7H.js.map +0 -1
- package/dist/code-agent-session-D5URqc3_.d.ts +0 -143
- package/dist/code-agent-session-D5URqc3_.d.ts.map +0 -1
- package/dist/cost-ledger-ZAa_P4r0.js.map +0 -1
- package/dist/extract-usage-BrQ8mCLX.js +0 -155
- package/dist/extract-usage-BrQ8mCLX.js.map +0 -1
- package/dist/index-3cdlURSk2.d.ts.map +0 -1
- package/dist/index-BAvgST_9.d.ts.map +0 -1
- package/dist/index-CXs7QlR5.d.ts +0 -217
- package/dist/ledger-core-eqaI3PCD.js +0 -388
- package/dist/ledger-core-eqaI3PCD.js.map +0 -1
- package/dist/metrics-C9YY1OcL.js +0 -239
- package/dist/metrics-C9YY1OcL.js.map +0 -1
- package/dist/off-policy-DvgzvtIx.js +0 -220
- package/dist/off-policy-DvgzvtIx.js.map +0 -1
- package/dist/off-policy-mskQw8Mb.d.ts +0 -153
- package/dist/off-policy-mskQw8Mb.d.ts.map +0 -1
- package/dist/pre-registration-DakwTRXk.js +0 -96
- package/dist/pre-registration-DakwTRXk.js.map +0 -1
- package/dist/profile-cell.js.map +0 -1
- package/dist/runtime-trajectory-1gyaTOoC.js +0 -93
- package/dist/runtime-trajectory-1gyaTOoC.js.map +0 -1
- package/dist/runtime-trajectory-BW9Wszb-.d.ts +0 -50
- package/dist/runtime-trajectory-BW9Wszb-.d.ts.map +0 -1
- package/dist/statistics-CnnxdpOg.js.map +0 -1
- package/dist/task-failure-attributes-CQZlB3et.js.map +0 -1
|
@@ -1,155 +0,0 @@
|
|
|
1
|
-
import { TOKEN_USAGE_INPUT_KEYS, TOKEN_USAGE_OUTPUT_KEYS, firstTokenCount, tokenUsageSource } from "@tangle-network/agent-core/telemetry";
|
|
2
|
-
import { SSEChunkParser } from "@tangle-network/agent-core/sse";
|
|
3
|
-
//#region src/trace/extract-usage.ts
|
|
4
|
-
/**
|
|
5
|
-
* Provider response and SSE usage extraction.
|
|
6
|
-
*
|
|
7
|
-
* Missing usage returns `null`; reported zeroes and cache-only activity remain
|
|
8
|
-
* distinguishable from absent telemetry.
|
|
9
|
-
*/
|
|
10
|
-
const INPUT_OTHER_KEYS = ["input_other"];
|
|
11
|
-
const EXCLUSIVE_REASONING_KEYS = [
|
|
12
|
-
"reasoning",
|
|
13
|
-
"reasoning_tokens",
|
|
14
|
-
"reasoningTokens"
|
|
15
|
-
];
|
|
16
|
-
const INCLUSIVE_REASONING_KEYS = ["reasoning_output_tokens", "reasoningOutputTokens"];
|
|
17
|
-
const INPUT_DETAIL_KEYS = ["prompt_tokens_details", "input_tokens_details"];
|
|
18
|
-
const OUTPUT_DETAIL_KEYS = ["completion_tokens_details", "output_tokens_details"];
|
|
19
|
-
const CACHED_KEYS = [
|
|
20
|
-
"cache",
|
|
21
|
-
"cached_tokens",
|
|
22
|
-
"cached_input_tokens",
|
|
23
|
-
"cache_read_tokens",
|
|
24
|
-
"cache_read_input_tokens",
|
|
25
|
-
"cachedTokens",
|
|
26
|
-
"cachedInputTokens",
|
|
27
|
-
"cacheReadTokens",
|
|
28
|
-
"cacheReadInputTokens",
|
|
29
|
-
"input_cache_read"
|
|
30
|
-
];
|
|
31
|
-
const CACHE_WRITE_KEYS = [
|
|
32
|
-
"cache_creation_tokens",
|
|
33
|
-
"cache_creation_input_tokens",
|
|
34
|
-
"cacheCreationTokens",
|
|
35
|
-
"cacheCreationInputTokens",
|
|
36
|
-
"input_cache_creation"
|
|
37
|
-
];
|
|
38
|
-
function nestedRecord(source, key) {
|
|
39
|
-
const value = source[key];
|
|
40
|
-
return value && typeof value === "object" && !Array.isArray(value) ? value : void 0;
|
|
41
|
-
}
|
|
42
|
-
function nestedNumber(source, recordKeys, valueKeys) {
|
|
43
|
-
for (const recordKey of recordKeys) {
|
|
44
|
-
const value = firstTokenCount(nestedRecord(source, recordKey), valueKeys);
|
|
45
|
-
if (value !== void 0) return value;
|
|
46
|
-
}
|
|
47
|
-
}
|
|
48
|
-
/**
|
|
49
|
-
* Pull `{ input, output, cached?, cacheWrite? }` from a parsed response
|
|
50
|
-
* body. Accepts a top-level `usage` object (the common case) or a body that IS
|
|
51
|
-
* the usage object. Returns null only when none of those categories is present.
|
|
52
|
-
*/
|
|
53
|
-
function extractUsage(body) {
|
|
54
|
-
if (!body || typeof body !== "object") return null;
|
|
55
|
-
const usage = tokenUsageSource(body);
|
|
56
|
-
const input = firstTokenCount(usage, TOKEN_USAGE_INPUT_KEYS);
|
|
57
|
-
const inputOther = firstTokenCount(usage, INPUT_OTHER_KEYS);
|
|
58
|
-
const output = firstTokenCount(usage, TOKEN_USAGE_OUTPUT_KEYS);
|
|
59
|
-
const exclusiveReasoning = firstTokenCount(usage, EXCLUSIVE_REASONING_KEYS);
|
|
60
|
-
const inclusiveReasoning = firstTokenCount(usage, INCLUSIVE_REASONING_KEYS) ?? nestedNumber(usage, OUTPUT_DETAIL_KEYS, ["reasoning_tokens", "reasoningTokens"]);
|
|
61
|
-
const reasoning = exclusiveReasoning ?? inclusiveReasoning;
|
|
62
|
-
const nestedCache = nestedRecord(usage, "cache");
|
|
63
|
-
const cached = firstTokenCount(usage, CACHED_KEYS) ?? firstTokenCount(nestedCache, ["read"]) ?? nestedNumber(usage, INPUT_DETAIL_KEYS, ["cached_tokens", "cachedTokens"]);
|
|
64
|
-
const cacheWrite = firstTokenCount(usage, CACHE_WRITE_KEYS) ?? firstTokenCount(nestedCache, ["write"]);
|
|
65
|
-
if (input === void 0 && inputOther === void 0 && output === void 0 && reasoning === void 0 && cached === void 0 && cacheWrite === void 0) return null;
|
|
66
|
-
const result = {
|
|
67
|
-
input: (input ?? 0) + (inputOther ?? 0),
|
|
68
|
-
output: (output ?? (inclusiveReasoning !== void 0 ? inclusiveReasoning : 0)) + (exclusiveReasoning ?? 0)
|
|
69
|
-
};
|
|
70
|
-
if (reasoning !== void 0) result.reasoning = reasoning;
|
|
71
|
-
if (cached !== void 0) result.cached = cached;
|
|
72
|
-
if (cacheWrite !== void 0) result.cacheWrite = cacheWrite;
|
|
73
|
-
return result;
|
|
74
|
-
}
|
|
75
|
-
/**
|
|
76
|
-
* Extract token usage from a complete SSE response body using the shared SSE
|
|
77
|
-
* frame parser. Cumulative snapshots are the fail-safe default because summing
|
|
78
|
-
* them inflates billing; callers with explicit delta events opt into `mode: 'delta'`.
|
|
79
|
-
*/
|
|
80
|
-
function extractUsageFromSse(text, options = {}) {
|
|
81
|
-
const mode = options.mode ?? "cumulative";
|
|
82
|
-
let input = 0;
|
|
83
|
-
let output = 0;
|
|
84
|
-
let reasoning = 0;
|
|
85
|
-
let sawReasoning = false;
|
|
86
|
-
let cached = 0;
|
|
87
|
-
let sawCached = false;
|
|
88
|
-
let cacheWrite = 0;
|
|
89
|
-
let sawCacheWrite = false;
|
|
90
|
-
let found = false;
|
|
91
|
-
const parser = new SSEChunkParser({ transform: parseSseJson });
|
|
92
|
-
const events = [...parser.push(text), ...parser.flush()];
|
|
93
|
-
const merge = mode === "delta" ? (current, next) => current + next : Math.max;
|
|
94
|
-
for (const event of events) {
|
|
95
|
-
const usage = extractUsage(event.data);
|
|
96
|
-
if (!usage) continue;
|
|
97
|
-
input = merge(input, usage.input);
|
|
98
|
-
output = merge(output, usage.output);
|
|
99
|
-
if (usage.reasoning !== void 0) {
|
|
100
|
-
reasoning = merge(reasoning, usage.reasoning);
|
|
101
|
-
sawReasoning = true;
|
|
102
|
-
}
|
|
103
|
-
if (usage.cached !== void 0) {
|
|
104
|
-
cached = merge(cached, usage.cached);
|
|
105
|
-
sawCached = true;
|
|
106
|
-
}
|
|
107
|
-
if (usage.cacheWrite !== void 0) {
|
|
108
|
-
cacheWrite = merge(cacheWrite, usage.cacheWrite);
|
|
109
|
-
sawCacheWrite = true;
|
|
110
|
-
}
|
|
111
|
-
found = true;
|
|
112
|
-
}
|
|
113
|
-
if (!found) return null;
|
|
114
|
-
return {
|
|
115
|
-
input,
|
|
116
|
-
output,
|
|
117
|
-
...sawReasoning ? { reasoning } : {},
|
|
118
|
-
...sawCached ? { cached } : {},
|
|
119
|
-
...sawCacheWrite ? { cacheWrite } : {}
|
|
120
|
-
};
|
|
121
|
-
}
|
|
122
|
-
function parseSseJson(raw) {
|
|
123
|
-
const payload = raw.trim();
|
|
124
|
-
if (!payload || payload === "[DONE]") return null;
|
|
125
|
-
try {
|
|
126
|
-
return JSON.parse(payload);
|
|
127
|
-
} catch {
|
|
128
|
-
return null;
|
|
129
|
-
}
|
|
130
|
-
}
|
|
131
|
-
/**
|
|
132
|
-
* Extract usage from an HTTP `Response` without consuming the caller's body:
|
|
133
|
-
* clones, reads the text, and tries the JSON parser first, then the SSE
|
|
134
|
-
* accumulator. Best-effort — returns null on any read/parse miss so a usage tee
|
|
135
|
-
* never takes down the underlying call.
|
|
136
|
-
*/
|
|
137
|
-
async function extractUsageFromResponse(response, sseOptions) {
|
|
138
|
-
let text;
|
|
139
|
-
try {
|
|
140
|
-
text = await response.clone().text();
|
|
141
|
-
} catch {
|
|
142
|
-
return null;
|
|
143
|
-
}
|
|
144
|
-
let json;
|
|
145
|
-
try {
|
|
146
|
-
json = JSON.parse(text);
|
|
147
|
-
} catch {
|
|
148
|
-
json = void 0;
|
|
149
|
-
}
|
|
150
|
-
return extractUsage(json) ?? extractUsageFromSse(text, sseOptions);
|
|
151
|
-
}
|
|
152
|
-
//#endregion
|
|
153
|
-
export { extractUsageFromResponse as n, extractUsageFromSse as r, extractUsage as t };
|
|
154
|
-
|
|
155
|
-
//# sourceMappingURL=extract-usage-BrQ8mCLX.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"extract-usage-BrQ8mCLX.js","names":["obj"],"sources":["../src/trace/extract-usage.ts"],"sourcesContent":["/**\n * Provider response and SSE usage extraction.\n *\n * Missing usage returns `null`; reported zeroes and cache-only activity remain\n * distinguishable from absent telemetry.\n */\n\nimport { SSEChunkParser } from '@tangle-network/agent-core/sse'\nimport {\n firstTokenCount,\n TOKEN_USAGE_INPUT_KEYS,\n TOKEN_USAGE_OUTPUT_KEYS,\n tokenUsageSource,\n} from '@tangle-network/agent-core/telemetry'\nimport type { RunTokenUsage } from '../run-record'\n\nexport type ExtractedUsage = RunTokenUsage\nexport type SseUsageMode = 'cumulative' | 'delta'\n\nexport interface ExtractUsageFromSseOptions {\n /** Provider usage events are cumulative snapshots unless explicitly marked as deltas. */\n mode?: SseUsageMode\n}\n\nconst INPUT_OTHER_KEYS = ['input_other'] as const\nconst EXCLUSIVE_REASONING_KEYS = ['reasoning', 'reasoning_tokens', 'reasoningTokens'] as const\nconst INCLUSIVE_REASONING_KEYS = ['reasoning_output_tokens', 'reasoningOutputTokens'] as const\nconst INPUT_DETAIL_KEYS = ['prompt_tokens_details', 'input_tokens_details'] as const\nconst OUTPUT_DETAIL_KEYS = ['completion_tokens_details', 'output_tokens_details'] as const\nconst CACHED_KEYS = [\n 'cache',\n 'cached_tokens',\n 'cached_input_tokens',\n 'cache_read_tokens',\n 'cache_read_input_tokens',\n 'cachedTokens',\n 'cachedInputTokens',\n 'cacheReadTokens',\n 'cacheReadInputTokens',\n 'input_cache_read',\n] as const\nconst CACHE_WRITE_KEYS = [\n 'cache_creation_tokens',\n 'cache_creation_input_tokens',\n 'cacheCreationTokens',\n 'cacheCreationInputTokens',\n 'input_cache_creation',\n] as const\n\nfunction nestedRecord(\n source: Record<string, unknown>,\n key: string,\n): Record<string, unknown> | undefined {\n const value = source[key]\n return value && typeof value === 'object' && !Array.isArray(value)\n ? (value as Record<string, unknown>)\n : undefined\n}\n\nfunction nestedNumber(\n source: Record<string, unknown>,\n recordKeys: readonly string[],\n valueKeys: readonly string[],\n): number | undefined {\n for (const recordKey of recordKeys) {\n const value = firstTokenCount(nestedRecord(source, recordKey), valueKeys)\n if (value !== undefined) return value\n }\n return undefined\n}\n\n/**\n * Pull `{ input, output, cached?, cacheWrite? }` from a parsed response\n * body. Accepts a top-level `usage` object (the common case) or a body that IS\n * the usage object. Returns null only when none of those categories is present.\n */\nexport function extractUsage(body: unknown): ExtractedUsage | null {\n if (!body || typeof body !== 'object') return null\n const obj = body as Record<string, unknown>\n const usage = tokenUsageSource(obj)\n const input = firstTokenCount(usage, TOKEN_USAGE_INPUT_KEYS)\n const inputOther = firstTokenCount(usage, INPUT_OTHER_KEYS)\n const output = firstTokenCount(usage, TOKEN_USAGE_OUTPUT_KEYS)\n const exclusiveReasoning = firstTokenCount(usage, EXCLUSIVE_REASONING_KEYS)\n const inclusiveReasoning =\n firstTokenCount(usage, INCLUSIVE_REASONING_KEYS) ??\n nestedNumber(usage, OUTPUT_DETAIL_KEYS, ['reasoning_tokens', 'reasoningTokens'])\n const reasoning = exclusiveReasoning ?? inclusiveReasoning\n const nestedCache = nestedRecord(usage, 'cache')\n const cached =\n firstTokenCount(usage, CACHED_KEYS) ??\n firstTokenCount(nestedCache, ['read']) ??\n nestedNumber(usage, INPUT_DETAIL_KEYS, ['cached_tokens', 'cachedTokens'])\n const cacheWrite =\n firstTokenCount(usage, CACHE_WRITE_KEYS) ?? firstTokenCount(nestedCache, ['write'])\n if (\n input === undefined &&\n inputOther === undefined &&\n output === undefined &&\n reasoning === undefined &&\n cached === undefined &&\n cacheWrite === undefined\n )\n return null\n const result: ExtractedUsage = {\n input: (input ?? 0) + (inputOther ?? 0),\n output:\n (output ?? (inclusiveReasoning !== undefined ? inclusiveReasoning : 0)) +\n (exclusiveReasoning ?? 0),\n }\n if (reasoning !== undefined) result.reasoning = reasoning\n if (cached !== undefined) result.cached = cached\n if (cacheWrite !== undefined) result.cacheWrite = cacheWrite\n return result\n}\n\n/**\n * Extract token usage from a complete SSE response body using the shared SSE\n * frame parser. Cumulative snapshots are the fail-safe default because summing\n * them inflates billing; callers with explicit delta events opt into `mode: 'delta'`.\n */\nexport function extractUsageFromSse(\n text: string,\n options: ExtractUsageFromSseOptions = {},\n): ExtractedUsage | null {\n const mode = options.mode ?? 'cumulative'\n let input = 0\n let output = 0\n let reasoning = 0\n let sawReasoning = false\n let cached = 0\n let sawCached = false\n let cacheWrite = 0\n let sawCacheWrite = false\n let found = false\n const parser = new SSEChunkParser<unknown>({ transform: parseSseJson })\n const events = [...parser.push(text), ...parser.flush()]\n const merge = mode === 'delta' ? (current: number, next: number) => current + next : Math.max\n\n for (const event of events) {\n const usage = extractUsage(event.data)\n if (!usage) continue\n input = merge(input, usage.input)\n output = merge(output, usage.output)\n if (usage.reasoning !== undefined) {\n reasoning = merge(reasoning, usage.reasoning)\n sawReasoning = true\n }\n if (usage.cached !== undefined) {\n cached = merge(cached, usage.cached)\n sawCached = true\n }\n if (usage.cacheWrite !== undefined) {\n cacheWrite = merge(cacheWrite, usage.cacheWrite)\n sawCacheWrite = true\n }\n found = true\n }\n if (!found) return null\n return {\n input,\n output,\n ...(sawReasoning ? { reasoning } : {}),\n ...(sawCached ? { cached } : {}),\n ...(sawCacheWrite ? { cacheWrite } : {}),\n }\n}\n\nfunction parseSseJson(raw: string): unknown | null {\n const payload = raw.trim()\n if (!payload || payload === '[DONE]') return null\n try {\n return JSON.parse(payload)\n } catch {\n return null\n }\n}\n\n/**\n * Extract usage from an HTTP `Response` without consuming the caller's body:\n * clones, reads the text, and tries the JSON parser first, then the SSE\n * accumulator. Best-effort — returns null on any read/parse miss so a usage tee\n * never takes down the underlying call.\n */\nexport async function extractUsageFromResponse(\n response: Response,\n sseOptions?: ExtractUsageFromSseOptions,\n): Promise<ExtractedUsage | null> {\n let text: string\n try {\n text = await response.clone().text()\n } catch {\n return null\n }\n let json: unknown\n try {\n json = JSON.parse(text)\n } catch {\n json = undefined\n }\n return extractUsage(json) ?? extractUsageFromSse(text, sseOptions)\n}\n"],"mappings":";;;;;;;;;AAwBA,MAAM,mBAAmB,CAAC,aAAa;AACvC,MAAM,2BAA2B;CAAC;CAAa;CAAoB;AAAiB;AACpF,MAAM,2BAA2B,CAAC,2BAA2B,uBAAuB;AACpF,MAAM,oBAAoB,CAAC,yBAAyB,sBAAsB;AAC1E,MAAM,qBAAqB,CAAC,6BAA6B,uBAAuB;AAChF,MAAM,cAAc;CAClB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF;AACA,MAAM,mBAAmB;CACvB;CACA;CACA;CACA;CACA;AACF;AAEA,SAAS,aACP,QACA,KACqC;CACrC,MAAM,QAAQ,OAAO;CACrB,OAAO,SAAS,OAAO,UAAU,YAAY,CAAC,MAAM,QAAQ,KAAK,IAC5D,QACD,KAAA;AACN;AAEA,SAAS,aACP,QACA,YACA,WACoB;CACpB,KAAK,MAAM,aAAa,YAAY;EAClC,MAAM,QAAQ,gBAAgB,aAAa,QAAQ,SAAS,GAAG,SAAS;EACxE,IAAI,UAAU,KAAA,GAAW,OAAO;CAClC;AAEF;;;;;;AAOA,SAAgB,aAAa,MAAsC;CACjE,IAAI,CAAC,QAAQ,OAAO,SAAS,UAAU,OAAO;CAE9C,MAAM,QAAQ,iBAAiBA,IAAG;CAClC,MAAM,QAAQ,gBAAgB,OAAO,sBAAsB;CAC3D,MAAM,aAAa,gBAAgB,OAAO,gBAAgB;CAC1D,MAAM,SAAS,gBAAgB,OAAO,uBAAuB;CAC7D,MAAM,qBAAqB,gBAAgB,OAAO,wBAAwB;CAC1E,MAAM,qBACJ,gBAAgB,OAAO,wBAAwB,KAC/C,aAAa,OAAO,oBAAoB,CAAC,oBAAoB,iBAAiB,CAAC;CACjF,MAAM,YAAY,sBAAsB;CACxC,MAAM,cAAc,aAAa,OAAO,OAAO;CAC/C,MAAM,SACJ,gBAAgB,OAAO,WAAW,KAClC,gBAAgB,aAAa,CAAC,MAAM,CAAC,KACrC,aAAa,OAAO,mBAAmB,CAAC,iBAAiB,cAAc,CAAC;CAC1E,MAAM,aACJ,gBAAgB,OAAO,gBAAgB,KAAK,gBAAgB,aAAa,CAAC,OAAO,CAAC;CACpF,IACE,UAAU,KAAA,KACV,eAAe,KAAA,KACf,WAAW,KAAA,KACX,cAAc,KAAA,KACd,WAAW,KAAA,KACX,eAAe,KAAA,GAEf,OAAO;CACT,MAAM,SAAyB;EAC7B,QAAQ,SAAS,MAAM,cAAc;EACrC,SACG,WAAW,uBAAuB,KAAA,IAAY,qBAAqB,OACnE,sBAAsB;CAC3B;CACA,IAAI,cAAc,KAAA,GAAW,OAAO,YAAY;CAChD,IAAI,WAAW,KAAA,GAAW,OAAO,SAAS;CAC1C,IAAI,eAAe,KAAA,GAAW,OAAO,aAAa;CAClD,OAAO;AACT;;;;;;AAOA,SAAgB,oBACd,MACA,UAAsC,CAAC,GAChB;CACvB,MAAM,OAAO,QAAQ,QAAQ;CAC7B,IAAI,QAAQ;CACZ,IAAI,SAAS;CACb,IAAI,YAAY;CAChB,IAAI,eAAe;CACnB,IAAI,SAAS;CACb,IAAI,YAAY;CAChB,IAAI,aAAa;CACjB,IAAI,gBAAgB;CACpB,IAAI,QAAQ;CACZ,MAAM,SAAS,IAAI,eAAwB,EAAE,WAAW,aAAa,CAAC;CACtE,MAAM,SAAS,CAAC,GAAG,OAAO,KAAK,IAAI,GAAG,GAAG,OAAO,MAAM,CAAC;CACvD,MAAM,QAAQ,SAAS,WAAW,SAAiB,SAAiB,UAAU,OAAO,KAAK;CAE1F,KAAK,MAAM,SAAS,QAAQ;EAC1B,MAAM,QAAQ,aAAa,MAAM,IAAI;EACrC,IAAI,CAAC,OAAO;EACZ,QAAQ,MAAM,OAAO,MAAM,KAAK;EAChC,SAAS,MAAM,QAAQ,MAAM,MAAM;EACnC,IAAI,MAAM,cAAc,KAAA,GAAW;GACjC,YAAY,MAAM,WAAW,MAAM,SAAS;GAC5C,eAAe;EACjB;EACA,IAAI,MAAM,WAAW,KAAA,GAAW;GAC9B,SAAS,MAAM,QAAQ,MAAM,MAAM;GACnC,YAAY;EACd;EACA,IAAI,MAAM,eAAe,KAAA,GAAW;GAClC,aAAa,MAAM,YAAY,MAAM,UAAU;GAC/C,gBAAgB;EAClB;EACA,QAAQ;CACV;CACA,IAAI,CAAC,OAAO,OAAO;CACnB,OAAO;EACL;EACA;EACA,GAAI,eAAe,EAAE,UAAU,IAAI,CAAC;EACpC,GAAI,YAAY,EAAE,OAAO,IAAI,CAAC;EAC9B,GAAI,gBAAgB,EAAE,WAAW,IAAI,CAAC;CACxC;AACF;AAEA,SAAS,aAAa,KAA6B;CACjD,MAAM,UAAU,IAAI,KAAK;CACzB,IAAI,CAAC,WAAW,YAAY,UAAU,OAAO;CAC7C,IAAI;EACF,OAAO,KAAK,MAAM,OAAO;CAC3B,QAAQ;EACN,OAAO;CACT;AACF;;;;;;;AAQA,eAAsB,yBACpB,UACA,YACgC;CAChC,IAAI;CACJ,IAAI;EACF,OAAO,MAAM,SAAS,MAAM,CAAC,CAAC,KAAK;CACrC,QAAQ;EACN,OAAO;CACT;CACA,IAAI;CACJ,IAAI;EACF,OAAO,KAAK,MAAM,IAAI;CACxB,QAAQ;EACN,OAAO,KAAA;CACT;CACA,OAAO,aAAa,IAAI,KAAK,oBAAoB,MAAM,UAAU;AACnE"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index-3cdlURSk2.d.ts","names":[],"sources":["../src/rollout/exporters.ts","../src/rollout/gate-checks.ts","../src/rollout/interchange/harbor.ts","../src/rollout/ledger.ts","../src/rollout/reward.ts","../src/rollout/mint.ts","../src/rollout/readers/claude-jsonl.ts","../src/rollout/readers/opencode-sqlite.ts","../src/rollout/release/gate-report.ts","../src/rollout/release/scrub.ts","../src/rollout/release/card.ts","../src/rollout/release/hf-dataset.ts"],"mappings":";;;;;;;;;;;;;;;;;;;;;UAiDiB;;EAEf;;;;;;EAMA;;iBAGc,eAAe,MAAM,oBAAoB;UAcxC;;EAEf;;EAEA;;;;;;;;;;;;;;;;;;;;;;KAuBU;UAEK,kCAAkC;;EAEjD,aAAa;;KAGH,mBAAmB;UAEd;EACf,UAAU;EACV;IACE;IACA;IACA;IACA;IACA;MACE;;;;;;;;;;;;;;;;;iBAkBU,UAAU,OAAO,qBAAqB,UAAS,mBAAwB;UAyBtE;;EAEf;EACA,OAAO;EACP;EACA;IACE;IACA;IACA;IACA;IACA,OAAO;MAUL;;;;;iBAMU,aACd,OAAO,qBACP,UAAS,wBACR;UAgCc;EACf;EACA;EACA;EACA;EACA;;UAGe;;EAEf,QAAQ;;EAER,YAAY;EACZ;EACA,SAAS;EACT,WAAW;EACX,aAAa;EACb;IACE,MAAM;IACN,QAAQ;IACR;IACA;IACA;IACA;IACA;IACA;IACA,MAAM;MACJ;;iBASU,yBAAyB,MAAM,oBAAoB;iBA+BnD,0BACd,OAAO,qBACP,UAAS,4BACR;UAoBc;;EAEf,UAAU;;EAEV;IACE;IACA;IACA;IACA;IACA;IACA,OAAO;IACP;MACE;;iBAGU,UAAU,MAAM,oBAAoB;;iBAmBpC,WACd,OAAO,qBACP,UAAS,4BACR;iBAkBa,QAAQ,MAAM;;;;;;;;;;cCpUjB;KAOD,sBAAsB;;;;;;;;;KAUtB,qBAAqB,QAAQ,kBAAkB,SAAS;;;;;;;;;;;;;;;;;UAkBnD;EACf,SAAS;;EAET;;UAOe;EACf,IAAI;;EAEJ;;EAEA,SAAS,SAAS;;;;;;;;;;;;EAYlB,WAAW;;;;;;;;;;;;;iBA6LG,gBAAgB,SAAS,cAAc;;;;;;cA6I1C,yBAAyB,KAAK,cAAc;;;;;;;;;KAe7C;WACG;;;;WAEA;WAAyB;;;;WAEzB;WAAuB;;;KAU1B,yBAAyB,KAAK,cAAc;;;;;;;;;cAU3C;;;;;;;aAWT;eAnCiB;;aAoCjB,kBAAgB;aAMhB,2BAAyB;aAMzB,qBAAmB;;;;;;;;aAenB;;;aACA,kBAAgB;aAChB,2BAAyB;aACzB;;;;;;;;;;;aASA;;;aACA;;;aACA;;;aACA;;;;;;;;;;;aASA;;;aACA;;;aACA;;;aACA;;;;;;KAKQ,8BAA8B;;;;;;iBAO1B,WAAW,SAAS,aAAa,QAAQ;;;cCxd5C;;cAGA;KAwBD;UAEK;EACf;EACA;;UAGe;EACf;EACA;EACA,SAAS;;UAGM;EACf;EACA;;EAEA,WAAW;EACX,QAAQ;;UAGO;EACf;EACA;;EAEA;EACA,QAAQ;;UAGO;EACf;EACA,mBAAmB;EACnB,0BAA0B;EAC1B,QAAQ;;UAGO;EACf,SAAS;;UAGM;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA,QAAQ;;UAGO;;EAEf;EACA;EACA,QAAQ;EACR;EACA;EACA,kBAAkB;EAClB;EACA,aAAa;EACb,cAAc;EACd,UAAU;EACV;EACA;EACA,QAAQ;;UAGO;EACf;EACA;EACA;;EAEA,mBAAmB;EACnB,QAAQ;;UAGO;EACf;EACA;EACA;EACA;EACA;EACA,QAAQ;;UAGO;EACf;EACA;;EAEA;EACA,OAAO;EACP,OAAO;EACP;EACA,gBAAgB;EAChB;EACA,wBAAwB;EACxB,QAAQ;;;;;;;;;;;;;;;;;;iBAmXM,mBAAmB,OAAO,gBAAgB;;iBAa1C,qBAAqB,OAAO,gBAAgB;UA2O3C;;EAEf,YAAY;;;;;;;;;;;iBAwGE,qBACd,YAAY,kBACZ,UAAS,oBACR;;;;;;;;;;;;;;;iBAiCa,qBACd,gBAAgB,eAChB,OAAO,eACN;;;;iBC/6BmB,mBAAmB,cAAc,OAAO,gBAAgB;;iBAOxD,mBAAmB,cAAc,OAAO,gBAAgB;;;;;;;;;;;iBAiBxD,kBAAkB,eAAe,QAAQ;;;;;;;;;;;;;;;iBAkBzC,mBAAmB,eAAe,QAAQ;;;;;;;;KCzCpD;;KAGP,SAAS,KAAK;;iBAGH,gBAAgB,QAAQ;;;;;;;;;;;;;;iBAiBxB,mBAAmB,QAAQ,QAAQ,OAAO;;;;;;;;;;;;;;;;iBAmB1C,cACd,QAAQ,QACR,SAAQ;;KAOE;;;;;;;iBAQI,YAAY,QAAQ,QAAQ,SAAQ,kBAA8B;;;;;;;;;iBAelE,cACd,QAAQ,QACR,SAAQ;;;;;;;;;;;;iBAiBM,eAAe,QAAQ;EAAW;EAAuB;;;;;KCvE7D,mBAAmB;UAEd;EACf,QAAQ;;EAER;;EAEA,OAAO;;EAEP;;EAEA,YAAY;;UAGG;EACf,MAAM;;EAEN;;;;;;;;iBA2LoB,gBACpB,SAAS,aACT,OAAO,YACP,UAAS,qBACR,QAAQ;;;cC/OE;;iBAGG,kBAAkB;UAIjB;EACf;EACA;;;iBAIoB,sBACpB,aACA,uBACC,QAAQ;UASM;EACf;EACA;EACA;EACA;;UAGe;EACf,UAAU;EACV,OAAO;;EAEP;EACA;EACA;;UAyDe;;;;;;WAMN;;;iBAgBW,qBACpB,cACA,UAAS,8BACR,QAAQ;;;cCnHE;UAEI;EACf;EACA;EACA;EACA;;EAEA;IAAS;IAAa;;EACtB;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;;iBAcoB,eACpB,gBACC,QAAQ;;iBA8CK,gCACd,IAAI,cACJ,oBACC;iBAOa,wBACd,IAAI,cACJ,oBACC;;;;;;;iBA2Ba,4BAA4B,IAAI,cAAc,oBAAoB;;;;KCrFtE;cAEC,yBAAyB,OAAO,eAAe;;UAQ3C;EACf;EACA;;;;;;;;;;;;EAYA;;;;;;;;EAQA;;;;;;;;;;;;EAYA;;;UAIe;;EAEf;EACA;;UAGe;;EAEf;;EAEA;;;;;;EAMA;;EAEA;;;;;;;;;;;;;;EAcA,oBAAoB;;;;;;;;;;;;EAYpB;;EAEA;;;;;;;;;;EAUA,wBAAwB;;UAGT;;EAEf;EACA,UAAU,QAAQ,OAAO,eAAe;;;iBAI1B,gBAAgB,gBAAgB,sBAAsB;;;;;;;cAUzD;EAIX,MAAG,eAAkB,aAAW;EAShC,YAAS,eAAkB,6BAA2B;EAStD,MAAG,eAAkB,cAAY;EAOjC,MAAG,gBAAmB,wBAAsB;;;iBAgC9B,kBACd,OAAO,qBACP,eAAe,kBACd;;;;;;;;;;;;;;;;iBA+Fa,iBAAiB,QAAQ;;;UCrUxB;EACf;EACA,SAAS;;EAET,UAAU,eAAe;;cAGd,sBAAsB;;KAkEvB,cAAc;iBAEV,oBAAoB;iBAIpB,eAAe,MAAM,aAAa,MAAM,cAAc;iBAKtD,UAAU,cAAc,QAAQ;;;;;;;;;iBA8BhC,iBAAiB,MAAM,mBAAmB,QAAQ,cAAc;iBAIhE,WAAW,OAAO;EAChC,OAAO;EACP,QAAQ;;;;;;;;;iBAaM,uBAAuB;;;cCrI1B;KACD,wBAAwB;;cAGvB,cAAc,OAAO;UAejB;;EAEf,OAAO;EACP,SAAS;EACT;;EAEA;EACA,aAAa;EACb;IAAY;IAAmB;;EAC/B,cAAc,QAAQ,OAAO;;;;;;;EAO7B,MAAM;;iBA4GQ,iBAAiB,QAAQ;;;UC1HxB;EACf;EACA,SAAS;EACT;;UAGe;;EAEf,OAAO,eAAe;EACtB,QAAQ;EACR;IAAY;IAAmB;;;UAGhB;EACf;EACA;EACA;EACA,OAAO;EACP,cAAc,QAAQ,OAAO;;EAE7B,MAAM;EACN;;iBAGoB,eACpB,kBACA,SAAS,eACR,QAAQ;iBAuGK,gBAAgB,cAAc;iBAI9B,YAAY,cAAc;UAgBzB,8BAA8B;EAC7C;EACA;;cAGW;iBAGG,wBAAwB,iBAAiB;;iBA0CnC,qBAAqB,iBAAiB"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index-BAvgST_9.d.ts","names":[],"sources":["../src/ledger-core/atomic-file-lock.ts","../src/ledger-core/journal-file.ts","../src/ledger-core/journal.ts"],"mappings":";;UAiBiB;WACN;WACA;WACA;;UAGM;WACN,OAAO;EAChB;;KAGU;WACG;WAA0B;WAAyB,QAAQ;;WAC3D;WAA0B;;KAE7B;WACG;WAAyB,MAAM;IAC1C;UAEa;WACN;WACA;;cAGE,4BAA4B;WACrB;;;iBASJ,oBACd,SAAS,wBACR;;;;;;;iBAea,yBACd,SAAS,wBACR;;;;;UC/Dc;;EAEf;EACA,eAAe,iBAAiB;IAAY;MAAoB;;;;iBAKlD,iBAAiB,cAAc,cAAc,SAAS;iBAYtD,mBAAmB,GAAG,cAAc,SAAS,mBAAmB,WAAW,IAAI;KAQnF,eAAe;EAAO;EAAgB,OAAO;;EAAQ;;iBAEjD,sBAAsB,GACpC,cACA,SAAS,mBACT,WAAW,IACV,eAAe;;;KClBN;;;iBAII,gBAAgB;iBAIhB,cAAc,iBAAiB;;UAK9B;EACf;;;;;UAMe,kBAAkB,cAAc;EAC/C;EACA,cAAc;EACd,OAAO;EACP,WAAW;;KAGD,cAAc,uBAAuB,cAAc,mBAAmB,SAChF,kBAAkB;UAEH;EACf;EACA;;;;;UAMe,gBAAgB,OAAO;EACtC,MAAM,OAAO,OAAO;EACpB,OAAO,SAAS,UAAU;;UAGX,mBACf,uBACA,cAAc,iBACd,oBACQ;;EAER,QAAQ;EACR,cAAc,kBAAkB;;;;EAIhC,WAAW,cAAc,SAAS,oBAAoB,cAAc,QAAQ;;EAE5E,iBAAiB,OAAO,cAAc,QAAQ,QAAQ;EACtD,mBAAmB,gBAAgB,cAAc,QAAQ,QAAQ;;UAGlD,mBAAmB,OAAO;EACzC,SAAS;EACT,YAAY;;UAGG,mBAAmB,OAAO;EACzC,OAAO;;EAEP;EACA,YAAY;;;;cAiBD,kBAAkB,uBAAuB,cAAc,iBAAiB;WAC1E;mBACQ;mBACA;EAEjB,YAAY,cAAc,OAAO,mBAAmB,QAAQ,OAAO;EAM7D,UAAU,QAAQ,mBAAmB,cAAc,QAAQ,QAAQ;EAMnE,OACJ,OAAO,QACN,QAAQ,mBAAmB,cAAc,QAAQ,QAAQ;UA+BpD;UAKA;UAuCA"}
|
package/dist/index-CXs7QlR5.d.ts
DELETED
|
@@ -1,217 +0,0 @@
|
|
|
1
|
-
import { f as Run } from "./schema-BtVldJ3T.js";
|
|
2
|
-
import { s as TraceStore } from "./store-CT9YIIve.js";
|
|
3
|
-
import { a as OutcomeFilter, o as OutcomeStore } from "./outcome-store-BYHIuO0e.js";
|
|
4
|
-
import "./rubric-predictive-validity-C1dCLcvb.js";
|
|
5
|
-
import { a as ContinuousCalibrationResult, n as CandidateScore, o as GoldenItem, r as ContinuousAgreement, t as CalibrationResult } from "./judge-calibration-DFtEMlde.js";
|
|
6
|
-
import { n as SeriesConvergenceResult, t as SeriesConvergenceOptions } from "./series-convergence-ofsqPWhs.js";
|
|
7
|
-
import { i as CorpusAgreementReport } from "./statistics-DbvkkDPa.js";
|
|
8
|
-
//#region src/meta-eval/correlation-study.d.ts
|
|
9
|
-
interface EvalMetricSpec {
|
|
10
|
-
id: string;
|
|
11
|
-
/** Extract a scalar from a run (defaults cover score/pass/durationMs/costUsd/tokens). */
|
|
12
|
-
extract?: (run: Run, store: TraceStore) => Promise<number | null>;
|
|
13
|
-
}
|
|
14
|
-
interface OutcomePair {
|
|
15
|
-
evalMetric: string;
|
|
16
|
-
outcomeMetric: string;
|
|
17
|
-
}
|
|
18
|
-
interface CorrelationResult {
|
|
19
|
-
evalMetric: string;
|
|
20
|
-
outcomeMetric: string;
|
|
21
|
-
n: number;
|
|
22
|
-
pearson: number;
|
|
23
|
-
spearman: number;
|
|
24
|
-
/** 95% bootstrap CI for Pearson. */
|
|
25
|
-
pearsonCi95: {
|
|
26
|
-
lower: number;
|
|
27
|
-
upper: number;
|
|
28
|
-
};
|
|
29
|
-
/** Rough verdict: 'strong' ≥ 0.7, 'moderate' ≥ 0.4, else 'weak'. */
|
|
30
|
-
verdict: 'strong' | 'moderate' | 'weak';
|
|
31
|
-
}
|
|
32
|
-
interface CorrelationStudyResult {
|
|
33
|
-
pairs: CorrelationResult[];
|
|
34
|
-
joinedSamples: number;
|
|
35
|
-
skippedRuns: number;
|
|
36
|
-
}
|
|
37
|
-
interface CorrelationStudyOptions {
|
|
38
|
-
/** Only join outcomes captured within this window after run.startedAt. */
|
|
39
|
-
maxCaptureLagMs?: number;
|
|
40
|
-
/** Restrict to a subset of outcomes (cohort, region, source). */
|
|
41
|
-
outcomeFilter?: OutcomeFilter;
|
|
42
|
-
/** Which outcome per run to use when multiple exist. Default 'latest'. */
|
|
43
|
-
reduction?: 'latest' | 'mean' | 'max';
|
|
44
|
-
/** Bootstrap iterations for the CI. Default 500. */
|
|
45
|
-
bootstrapIterations?: number;
|
|
46
|
-
}
|
|
47
|
-
declare function correlationStudy(traceStore: TraceStore, outcomeStore: OutcomeStore, evalMetrics: EvalMetricSpec[], outcomeMetricNames: string[], options?: CorrelationStudyOptions): Promise<CorrelationStudyResult>;
|
|
48
|
-
//#endregion
|
|
49
|
-
//#region src/meta-eval/calibration.d.ts
|
|
50
|
-
interface CalibrationBin {
|
|
51
|
-
lower: number;
|
|
52
|
-
upper: number;
|
|
53
|
-
n: number;
|
|
54
|
-
evalMean: number;
|
|
55
|
-
outcomeMean: number;
|
|
56
|
-
/** |outcomeMean − evalMean|; contributes to ECE weighted by n/total. */
|
|
57
|
-
gap: number;
|
|
58
|
-
}
|
|
59
|
-
interface CalibrationReport {
|
|
60
|
-
evalMetric: string;
|
|
61
|
-
outcomeMetric: string;
|
|
62
|
-
n: number;
|
|
63
|
-
bins: CalibrationBin[];
|
|
64
|
-
/** Expected Calibration Error — Σ (n_i/N) × |outcomeMean_i − evalMean_i|. */
|
|
65
|
-
ece: number;
|
|
66
|
-
/** Max bin gap — upper bound on miscalibration. */
|
|
67
|
-
maxGap: number;
|
|
68
|
-
}
|
|
69
|
-
interface CalibrationOptions {
|
|
70
|
-
bins?: number;
|
|
71
|
-
/** Equal-width (fixed bin edges) or equal-frequency (quantile bins). */
|
|
72
|
-
binning?: 'equal-width' | 'equal-frequency';
|
|
73
|
-
/** Clip eval values to [lo, hi] before binning. */
|
|
74
|
-
range?: {
|
|
75
|
-
lo: number;
|
|
76
|
-
hi: number;
|
|
77
|
-
};
|
|
78
|
-
}
|
|
79
|
-
interface CalibrationPair {
|
|
80
|
-
evalScore: number;
|
|
81
|
-
outcome: number;
|
|
82
|
-
}
|
|
83
|
-
declare function calibrationCurve(traceStore: TraceStore, outcomeStore: OutcomeStore, evalMetric: EvalMetricSpec, outcomeMetric: string, options?: CalibrationOptions): Promise<CalibrationReport | null>;
|
|
84
|
-
declare function calibrationFromPairs(inputPairs: CalibrationPair[], evalMetric: string, outcomeMetric: string, options?: CalibrationOptions): CalibrationReport | null;
|
|
85
|
-
//#endregion
|
|
86
|
-
//#region src/meta-eval/sentinel.d.ts
|
|
87
|
-
declare const SENTINEL_METRIC_NAMES: readonly ['irr', 'calibrationKappa', 'sentinelPassRate'];
|
|
88
|
-
type SentinelMetricName = (typeof SENTINEL_METRIC_NAMES)[number];
|
|
89
|
-
interface SentinelMetrics {
|
|
90
|
-
/** Inter-rater reliability — ICC(2,1) from agreement instruments. */
|
|
91
|
-
irr?: number;
|
|
92
|
-
/** Weighted κ vs the human golden set (`calibrateJudge*`). */
|
|
93
|
-
calibrationKappa?: number;
|
|
94
|
-
/** Fraction of a fixed sentinel golden set the judge still scores within tolerance. */
|
|
95
|
-
sentinelPassRate?: number;
|
|
96
|
-
}
|
|
97
|
-
interface SentinelSnapshot {
|
|
98
|
-
/** Caller-supplied ISO-8601 timestamp — the module never reads a clock. */
|
|
99
|
-
at: string;
|
|
100
|
-
judgeId: string;
|
|
101
|
-
/** Exact model identity behind the judge (pin the full version string). */
|
|
102
|
-
judgeModel: string;
|
|
103
|
-
/**
|
|
104
|
-
* Metrics measured at `at`. May be empty: an empty-metrics snapshot is a
|
|
105
|
-
* model-change marker — it records the new `judgeModel` immediately and
|
|
106
|
-
* the silent-upgrade alarm stays raised until a golden-grounded snapshot
|
|
107
|
-
* (calibrationKappa or sentinelPassRate) follows.
|
|
108
|
-
*/
|
|
109
|
-
metrics: SentinelMetrics;
|
|
110
|
-
}
|
|
111
|
-
/** Identity + timestamp the caller supplies alongside an instrument output. */
|
|
112
|
-
interface SnapshotMeta {
|
|
113
|
-
at: string;
|
|
114
|
-
judgeId: string;
|
|
115
|
-
judgeModel: string;
|
|
116
|
-
}
|
|
117
|
-
/** Shape guard for snapshots crossing a boundary (store append, JSONL load, report input). */
|
|
118
|
-
declare function validateSentinelSnapshot(snapshot: SentinelSnapshot, source?: string): void;
|
|
119
|
-
/**
|
|
120
|
-
* From `calibrateJudge` / `calibrateJudgeContinuous` output. Prefers the
|
|
121
|
-
* un-rounded continuous κ_w when the report carries one (integer κ
|
|
122
|
-
* discards information for [0,1] judges).
|
|
123
|
-
*/
|
|
124
|
-
declare function snapshotFromCalibration(report: CalibrationResult | ContinuousCalibrationResult, meta: SnapshotMeta): SentinelSnapshot;
|
|
125
|
-
/**
|
|
126
|
-
* From `corpusInterRaterAgreement` (statistics.ts) or `continuousAgreement`
|
|
127
|
-
* (judge-calibration.ts) output. IRR = ICC(2,1) — the reliability
|
|
128
|
-
* coefficient both instruments compute. For ensemble-level agreement,
|
|
129
|
-
* `meta.judgeId` names the ensemble and `meta.judgeModel` pins its
|
|
130
|
-
* composition (e.g. a joined list of member model strings).
|
|
131
|
-
*/
|
|
132
|
-
declare function snapshotFromAgreement(report: CorpusAgreementReport | ContinuousAgreement, meta: SnapshotMeta): SentinelSnapshot;
|
|
133
|
-
interface SentinelSetOptions {
|
|
134
|
-
/** Max |judge − human| for an item to count as passed. Default 0.1 (tuned for [0,1] scores). */
|
|
135
|
-
tolerance?: number;
|
|
136
|
-
}
|
|
137
|
-
/**
|
|
138
|
-
* From a rerun of a fixed sentinel golden set: the same `GoldenItem`s the
|
|
139
|
-
* judge was originally calibrated on, re-scored periodically. Pass rate =
|
|
140
|
-
* fraction of joined items where the judge stays within `tolerance` of the
|
|
141
|
-
* human score. Items the judge didn't score are excluded from the
|
|
142
|
-
* denominator; zero joined items is an error, not a 0% pass rate.
|
|
143
|
-
*/
|
|
144
|
-
declare function snapshotFromSentinelSet(scores: CandidateScore[], golden: GoldenItem[], meta: SnapshotMeta, options?: SentinelSetOptions): SentinelSnapshot;
|
|
145
|
-
interface SentinelStore {
|
|
146
|
-
append(snapshot: SentinelSnapshot): Promise<void>;
|
|
147
|
-
/** All snapshots (optionally for one judge), in append order. */
|
|
148
|
-
history(judgeId?: string): Promise<SentinelSnapshot[]>;
|
|
149
|
-
}
|
|
150
|
-
declare function inMemorySentinelStore(initial?: SentinelSnapshot[]): SentinelStore;
|
|
151
|
-
/**
|
|
152
|
-
* JSONL store — one snapshot per line, appended atomically per call.
|
|
153
|
-
* A corrupt or shape-invalid line is a loud error naming the file and
|
|
154
|
-
* line number: a sentinel history that silently drops records would
|
|
155
|
-
* defeat the drift detection it exists to provide.
|
|
156
|
-
*/
|
|
157
|
-
declare function fileSentinelStore(path: string): SentinelStore;
|
|
158
|
-
interface SentinelThresholds {
|
|
159
|
-
/** Floor for irr — below it the judge ensemble is unreliable regardless of trend. Default 0.6. */
|
|
160
|
-
minIrr?: number;
|
|
161
|
-
/** Max tolerated calibrationKappa drop vs the series baseline. Default 0.15. */
|
|
162
|
-
maxKappaDrop?: number;
|
|
163
|
-
/** Max tolerated sentinelPassRate drop vs the series baseline. Default 0.1. */
|
|
164
|
-
maxSentinelDrop?: number;
|
|
165
|
-
/** Newest snapshot older than this (days, vs asOf) = the sentinel itself went blind. Default 30. */
|
|
166
|
-
staleAfterDays?: number;
|
|
167
|
-
}
|
|
168
|
-
interface JudgeSentinelOptions {
|
|
169
|
-
/** Caller-supplied ISO timestamp staleness is measured against. */
|
|
170
|
-
asOf: string;
|
|
171
|
-
thresholds?: SentinelThresholds;
|
|
172
|
-
/** Forwarded to `analyzeSeries` (window, stableCv, driftRun). */
|
|
173
|
-
convergence?: SeriesConvergenceOptions;
|
|
174
|
-
}
|
|
175
|
-
interface SentinelTrend {
|
|
176
|
-
judgeId: string;
|
|
177
|
-
metric: SentinelMetricName;
|
|
178
|
-
/** Verbatim `analyzeSeries` state for this judge×metric history. */
|
|
179
|
-
state: SeriesConvergenceResult['state'];
|
|
180
|
-
/** Newest value in the series. */
|
|
181
|
-
current: number;
|
|
182
|
-
/** Oldest value in the series — the reference the drop thresholds compare against. */
|
|
183
|
-
baseline: number;
|
|
184
|
-
/** current − baseline (signed; negative = decayed). */
|
|
185
|
-
drift: number;
|
|
186
|
-
alarmed: boolean;
|
|
187
|
-
reason?: string;
|
|
188
|
-
}
|
|
189
|
-
interface SentinelReport {
|
|
190
|
-
perJudge: SentinelTrend[];
|
|
191
|
-
alarms: string[];
|
|
192
|
-
healthy: boolean;
|
|
193
|
-
/**
|
|
194
|
-
* `judgeId:metric` pairs with too few snapshots for the convergence
|
|
195
|
-
* machine. Named, not counted — a blind spot you can't see is a blind
|
|
196
|
-
* spot you won't fix. Floor/drop checks still apply to thin series, so
|
|
197
|
-
* insufficient history alone never masks an alarm.
|
|
198
|
-
*/
|
|
199
|
-
insufficientHistory: string[];
|
|
200
|
-
}
|
|
201
|
-
declare function judgeSentinelReport(history: SentinelSnapshot[], opts: JudgeSentinelOptions): SentinelReport;
|
|
202
|
-
interface EvalHealthStamp {
|
|
203
|
-
healthy: boolean;
|
|
204
|
-
alarms: string[];
|
|
205
|
-
}
|
|
206
|
-
/**
|
|
207
|
-
* The object a campaign attaches to its verdicts. Compute it from the
|
|
208
|
-
* sentinel report in a post-campaign hook (or a nightly job feeding the
|
|
209
|
-
* next day's campaigns) and store it alongside the verdict payload.
|
|
210
|
-
* `healthy: false` means the judges that produced those verdicts have an
|
|
211
|
-
* unresolved drift / decay / silent-upgrade / staleness alarm — treat the
|
|
212
|
-
* verdicts as carrying an integrity warning until recalibration clears it.
|
|
213
|
-
*/
|
|
214
|
-
declare function evalHealthStamp(report: SentinelReport): EvalHealthStamp;
|
|
215
|
-
//#endregion
|
|
216
|
-
export { EvalMetricSpec as A, CalibrationPair as C, CorrelationResult as D, calibrationFromPairs as E, correlationStudy as M, CorrelationStudyOptions as O, CalibrationOptions as S, calibrationCurve as T, snapshotFromAgreement as _, SentinelMetrics as a, validateSentinelSnapshot as b, SentinelSnapshot as c, SentinelTrend as d, SnapshotMeta as f, judgeSentinelReport as g, inMemorySentinelStore as h, SentinelMetricName as i, OutcomePair as j, CorrelationStudyResult as k, SentinelStore as l, fileSentinelStore as m, JudgeSentinelOptions as n, SentinelReport as o, evalHealthStamp as p, SENTINEL_METRIC_NAMES as r, SentinelSetOptions as s, EvalHealthStamp as t, SentinelThresholds as u, snapshotFromCalibration as v, CalibrationReport as w, CalibrationBin as x, snapshotFromSentinelSet as y };
|
|
217
|
-
//# sourceMappingURL=index-CXs7QlR5.d.ts.map
|