margin-meter 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +292 -2
- package/dist/batching.d.ts +94 -0
- package/dist/batching.d.ts.map +1 -0
- package/dist/batching.js +181 -0
- package/dist/batching.js.map +1 -0
- package/dist/braintrust.d.ts +153 -0
- package/dist/braintrust.d.ts.map +1 -0
- package/dist/braintrust.js +424 -0
- package/dist/braintrust.js.map +1 -0
- package/dist/capture.d.ts +28 -0
- package/dist/capture.d.ts.map +1 -0
- package/dist/capture.js +177 -0
- package/dist/capture.js.map +1 -0
- package/dist/index.d.ts +334 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +697 -23
- package/dist/index.js.map +1 -1
- package/dist/instrument.d.ts +8 -7
- package/dist/instrument.d.ts.map +1 -1
- package/dist/instrument.js +8 -7
- package/dist/instrument.js.map +1 -1
- package/dist/langsmith.d.ts +150 -0
- package/dist/langsmith.d.ts.map +1 -0
- package/dist/langsmith.js +415 -0
- package/dist/langsmith.js.map +1 -0
- package/dist/promptfoo.d.ts +161 -0
- package/dist/promptfoo.d.ts.map +1 -0
- package/dist/promptfoo.js +446 -0
- package/dist/promptfoo.js.map +1 -0
- package/dist/providers.d.ts +36 -0
- package/dist/providers.d.ts.map +1 -1
- package/dist/providers.js +229 -3
- package/dist/providers.js.map +1 -1
- package/dist/vitest.d.ts +232 -0
- package/dist/vitest.d.ts.map +1 -0
- package/dist/vitest.js +502 -0
- package/dist/vitest.js.map +1 -0
- package/package.json +24 -6
|
@@ -0,0 +1,446 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* margin-meter/promptfoo — turn a `promptfoo eval` result file into outcomes.
|
|
3
|
+
*
|
|
4
|
+
* The TypeScript twin of `sdk/python/margin_meter/promptfoo.py`. promptfoo is a
|
|
5
|
+
* Node tool, so a promptfoo user is already in the JS toolchain and this SDK is
|
|
6
|
+
* their natural bridge. `promptfoo eval -o results.json` writes a code-graded
|
|
7
|
+
* pass/fail file that maps one-to-one onto `POST /api/ingest/outcomes`
|
|
8
|
+
* (`docs/INTEGRATION_LADDER.md` rung 2), so the whole of rung 2 — "their evals
|
|
9
|
+
* report pass/fail" — is available with no hand-written POST:
|
|
10
|
+
*
|
|
11
|
+
* promptfoo eval -o results.json
|
|
12
|
+
*
|
|
13
|
+
* import { MarginMeter, promptfoo } from "margin-meter";
|
|
14
|
+
* const meter = new MarginMeter(); // MARGIN_INGEST_URL/KEY from env
|
|
15
|
+
* const data = JSON.parse(await readFile("results.json", "utf8"));
|
|
16
|
+
* await promptfoo.recordResults(meter, data, "my-eval-suite");
|
|
17
|
+
*
|
|
18
|
+
* The mapping is identical to the Python adapter's, on purpose — the two SDKs
|
|
19
|
+
* must not drift:
|
|
20
|
+
*
|
|
21
|
+
* - `passed` ← `success` (the top-level aggregate verdict).
|
|
22
|
+
* - `qualityScore` ← `score` passed through when in [0, 1]; a score outside
|
|
23
|
+
* that range (a raw metric from a custom assert) falls back to the coherent
|
|
24
|
+
* binary, because the ingest layer rejects an out-of-range score and losing
|
|
25
|
+
* the row is worse than losing the decimal. Always supplied (1.0/0.0 at
|
|
26
|
+
* least): the ingest layer discards `quality_method`/`grader_kind` on a
|
|
27
|
+
* scoreless outcome, so the score is what lights the grading provenance.
|
|
28
|
+
* - `qualityMethod`/`graderKind` ← DERIVED from the assert types, never
|
|
29
|
+
* assumed (see the honesty rule below).
|
|
30
|
+
* - `link` ← `testCase.description`, else the prompt id + index — never
|
|
31
|
+
* `prompt.label`, which promptfoo defaults to the prompt text (MAR-1548).
|
|
32
|
+
* - `valueUsd` ← opt-in via `vars.margin_value` or
|
|
33
|
+
* `testCase.metadata.margin_value` — the only route to value-per-outcome
|
|
34
|
+
* (rung 8). Absent is the cost-per-outcome default, never coerced to 0.
|
|
35
|
+
* - a result with no boolean `success` is not a verdict and is SKIPPED —
|
|
36
|
+
* recording a non-verdict would fabricate an outcome the eval did not
|
|
37
|
+
* produce.
|
|
38
|
+
*
|
|
39
|
+
* ## The honesty rule
|
|
40
|
+
*
|
|
41
|
+
* promptfoo grades two different ways and the adapter must not conflate them:
|
|
42
|
+
* code-graded asserts (`contains`, `equals`, `is-json`, `javascript`, `regex`,
|
|
43
|
+
* …) are `ground_truth`/`code`; model-graded asserts (`llm-rubric`,
|
|
44
|
+
* `factuality`, `similar`, `classifier`, `g-eval`, …) are `llm_judge`/`judge`.
|
|
45
|
+
* The WEAKER honest claim wins: any model-graded assert makes the whole outcome
|
|
46
|
+
* a judge grade; a code assert mixed with an assert type we do not recognise is
|
|
47
|
+
* downgraded to `unknown` (we cannot prove the unknown one was code); nothing
|
|
48
|
+
* recognisable at all is `unknown`. A rubric-graded eval reported as a
|
|
49
|
+
* ground-truth code grade would be the exact productivity illusion Margin
|
|
50
|
+
* exists to puncture.
|
|
51
|
+
*
|
|
52
|
+
* Two design promises, same as the Python adapter: it consumes the file the
|
|
53
|
+
* customer already produces (no new harness), and it never fabricates and is
|
|
54
|
+
* fail-safe — a delivery fault on one row warns and is counted, never aborting
|
|
55
|
+
* the batch.
|
|
56
|
+
*/
|
|
57
|
+
// Grading provenance strings — mirror `src/margin/grading.py`, kept as literals
|
|
58
|
+
// so the SDK has no dependency on the server package.
|
|
59
|
+
const GROUND_TRUTH = "ground_truth";
|
|
60
|
+
const LLM_JUDGE = "llm_judge";
|
|
61
|
+
const UNKNOWN_METHOD = "unknown";
|
|
62
|
+
const GRADER_CODE = "code";
|
|
63
|
+
const GRADER_JUDGE = "judge";
|
|
64
|
+
const GRADER_UNKNOWN = "unknown";
|
|
65
|
+
// `Outcome.link` is a `String(512)`; a longer link is truncated, not dropped (a
|
|
66
|
+
// lossy link beats a lost outcome) — same rule as the pytest adapter.
|
|
67
|
+
const MAX_LINK_LEN = 512;
|
|
68
|
+
const VALUE_KEY = "margin_value";
|
|
69
|
+
/** Mirror of `src/margin/ingest.MAX_VALUE_USD`: ingest refuses a value past this. */
|
|
70
|
+
const MAX_VALUE_USD = 10_000_000;
|
|
71
|
+
/** promptfoo assert types that are pure code: a deterministic comparison with no
|
|
72
|
+
* model in the loop. Normalised lower-case, separators stripped so `is-json`
|
|
73
|
+
* and `isjson` collapse together and the `not-` negation prefix is ignored. */
|
|
74
|
+
const CODE_ASSERTS = new Set([
|
|
75
|
+
"equals",
|
|
76
|
+
"contains",
|
|
77
|
+
"icontains",
|
|
78
|
+
"containsall",
|
|
79
|
+
"containsany",
|
|
80
|
+
"icontainsall",
|
|
81
|
+
"icontainsany",
|
|
82
|
+
"startswith",
|
|
83
|
+
"regex",
|
|
84
|
+
"isjson",
|
|
85
|
+
"containsjson",
|
|
86
|
+
"issql",
|
|
87
|
+
"containssql",
|
|
88
|
+
"isxml",
|
|
89
|
+
"containsxml",
|
|
90
|
+
"isvalidopenaifunctioncall",
|
|
91
|
+
"isvalidopenaitoolscall",
|
|
92
|
+
"javascript",
|
|
93
|
+
"python",
|
|
94
|
+
"webhook",
|
|
95
|
+
"levenshtein",
|
|
96
|
+
"rougen",
|
|
97
|
+
"bleu",
|
|
98
|
+
"gleu",
|
|
99
|
+
"meteor",
|
|
100
|
+
"latency",
|
|
101
|
+
"cost",
|
|
102
|
+
"perplexity",
|
|
103
|
+
"perplexityscore",
|
|
104
|
+
"finishreason",
|
|
105
|
+
"guardrails",
|
|
106
|
+
"trace",
|
|
107
|
+
]);
|
|
108
|
+
/** promptfoo assert types that put a MODEL in the grading loop — an LLM judge,
|
|
109
|
+
* an embedding-similarity model, or a classifier. Any one present means the
|
|
110
|
+
* strongest honest claim is `llm_judge`/`judge`, never a code grade. */
|
|
111
|
+
const MODEL_ASSERTS = new Set([
|
|
112
|
+
"llmrubric",
|
|
113
|
+
"modelgradedclosedqa",
|
|
114
|
+
"modelgradedfactuality",
|
|
115
|
+
"factuality",
|
|
116
|
+
"answerrelevance",
|
|
117
|
+
"contextrelevance",
|
|
118
|
+
"contextfaithfulness",
|
|
119
|
+
"contextrecall",
|
|
120
|
+
"similar",
|
|
121
|
+
"classifier",
|
|
122
|
+
"moderation",
|
|
123
|
+
"selectbest",
|
|
124
|
+
"pi",
|
|
125
|
+
"geval",
|
|
126
|
+
"gevalfactuality",
|
|
127
|
+
]);
|
|
128
|
+
const defaultWarn = (message) => {
|
|
129
|
+
// `process` may be absent in a non-Node runtime; fall back to console.
|
|
130
|
+
const p = globalThis
|
|
131
|
+
.process;
|
|
132
|
+
if (p && typeof p.emitWarning === "function")
|
|
133
|
+
p.emitWarning(message);
|
|
134
|
+
else
|
|
135
|
+
console.warn(message);
|
|
136
|
+
};
|
|
137
|
+
function isObject(x) {
|
|
138
|
+
return typeof x === "object" && x !== null && !Array.isArray(x);
|
|
139
|
+
}
|
|
140
|
+
/** Normalise a promptfoo assert type for set membership: lower-case, drop
|
|
141
|
+
* dashes/underscores/spaces, strip a leading `not` negation (`not-contains`
|
|
142
|
+
* grades the same WAY as `contains`). Returns undefined for a missing/blank
|
|
143
|
+
* type. */
|
|
144
|
+
export function normAssertType(raw) {
|
|
145
|
+
if (typeof raw !== "string")
|
|
146
|
+
return undefined;
|
|
147
|
+
let t = raw.trim().toLowerCase().replace(/[-_ ]/g, "");
|
|
148
|
+
if (!t)
|
|
149
|
+
return undefined;
|
|
150
|
+
if (t.startsWith("not") && t !== "not")
|
|
151
|
+
t = t.slice(3);
|
|
152
|
+
return t || undefined;
|
|
153
|
+
}
|
|
154
|
+
/** Derive `(qualityMethod, graderKind)` from the assert types behind a result,
|
|
155
|
+
* applying the weaker-claim-wins honesty rule:
|
|
156
|
+
* - any model-graded assert → llm_judge/judge
|
|
157
|
+
* - only recognised code asserts → ground_truth/code
|
|
158
|
+
* - anything else (no asserts, or a code assert mixed with an unrecognised
|
|
159
|
+
* type we cannot vouch for) → unknown/unknown */
|
|
160
|
+
export function gradingSignature(assertTypes) {
|
|
161
|
+
let hasModel = false;
|
|
162
|
+
let hasCode = false;
|
|
163
|
+
let hasUnknown = false;
|
|
164
|
+
for (const raw of assertTypes) {
|
|
165
|
+
const t = normAssertType(raw);
|
|
166
|
+
if (t === undefined)
|
|
167
|
+
continue;
|
|
168
|
+
if (MODEL_ASSERTS.has(t))
|
|
169
|
+
hasModel = true;
|
|
170
|
+
else if (CODE_ASSERTS.has(t))
|
|
171
|
+
hasCode = true;
|
|
172
|
+
else
|
|
173
|
+
hasUnknown = true;
|
|
174
|
+
}
|
|
175
|
+
if (hasModel)
|
|
176
|
+
return { qualityMethod: LLM_JUDGE, graderKind: GRADER_JUDGE };
|
|
177
|
+
if (hasCode && !hasUnknown)
|
|
178
|
+
return { qualityMethod: GROUND_TRUTH, graderKind: GRADER_CODE };
|
|
179
|
+
return { qualityMethod: UNKNOWN_METHOD, graderKind: GRADER_UNKNOWN };
|
|
180
|
+
}
|
|
181
|
+
/** Collect the assert types behind one promptfoo result. Prefers
|
|
182
|
+
* `gradingResult.componentResults[].assertion.type` (what actually ran,
|
|
183
|
+
* including expanded asserts) and its single top-level `assertion`; falls back
|
|
184
|
+
* to the configured `testCase.assert[].type`. */
|
|
185
|
+
export function assertTypesFromResult(result) {
|
|
186
|
+
const types = [];
|
|
187
|
+
if (!isObject(result))
|
|
188
|
+
return types;
|
|
189
|
+
const grading = result.gradingResult;
|
|
190
|
+
if (isObject(grading)) {
|
|
191
|
+
const components = grading.componentResults;
|
|
192
|
+
if (Array.isArray(components)) {
|
|
193
|
+
for (const comp of components) {
|
|
194
|
+
if (isObject(comp) && isObject(comp.assertion)) {
|
|
195
|
+
types.push(comp.assertion.type);
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
if (isObject(grading.assertion))
|
|
200
|
+
types.push(grading.assertion.type);
|
|
201
|
+
}
|
|
202
|
+
if (types.length > 0)
|
|
203
|
+
return types;
|
|
204
|
+
const testCase = result.testCase;
|
|
205
|
+
if (isObject(testCase) && Array.isArray(testCase.assert)) {
|
|
206
|
+
for (const a of testCase.assert) {
|
|
207
|
+
if (isObject(a))
|
|
208
|
+
types.push(a.type);
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
return types;
|
|
212
|
+
}
|
|
213
|
+
// A finite DECIMAL number, the only string shape accepted. Deliberately narrower
|
|
214
|
+
// than JS `Number()`, which also parses `0x10`/`0o7`/`0b1`/`Infinity`/`""` — a
|
|
215
|
+
// `margin_value` typo like `"0x10"` must be a DROPPED value, never a silent $16.
|
|
216
|
+
// This is the acceptance set of Python `float()` minus `inf`/`nan`, so the two
|
|
217
|
+
// SDKs treat a stringly-typed tag the same and neither lets a non-number through.
|
|
218
|
+
const DECIMAL_RE = /^[+-]?(\d+\.?\d*|\.\d+)([eE][+-]?\d+)?$/;
|
|
219
|
+
/** Coerce a raw `margin_value` tag to a finite number, tolerating a stringly-typed
|
|
220
|
+
* var. A negative is a loss and is kept (MAR-1198). Throws on a bool, a non-decimal
|
|
221
|
+
* string, a non-finite value, or one past +/-MAX_VALUE_USD — the caller warns and
|
|
222
|
+
* drops the value while keeping the pass/fail. */
|
|
223
|
+
function coerceValue(raw) {
|
|
224
|
+
if (typeof raw === "boolean")
|
|
225
|
+
throw new Error(`${VALUE_KEY} must be a number, got ${JSON.stringify(raw)}`);
|
|
226
|
+
let val;
|
|
227
|
+
if (typeof raw === "number") {
|
|
228
|
+
val = raw;
|
|
229
|
+
}
|
|
230
|
+
else if (typeof raw === "string") {
|
|
231
|
+
const s = raw.trim();
|
|
232
|
+
if (!DECIMAL_RE.test(s))
|
|
233
|
+
throw new Error(`${VALUE_KEY} must be a number, got ${JSON.stringify(raw)}`);
|
|
234
|
+
val = Number(s);
|
|
235
|
+
}
|
|
236
|
+
else {
|
|
237
|
+
throw new Error(`${VALUE_KEY} must be a number, got ${JSON.stringify(raw)}`);
|
|
238
|
+
}
|
|
239
|
+
if (!Number.isFinite(val))
|
|
240
|
+
throw new Error(`${VALUE_KEY} must be a finite number, got ${JSON.stringify(raw)}`);
|
|
241
|
+
// A loss is a real value (MAR-1198): ingest takes any finite amount within +/-MAX.
|
|
242
|
+
if (Math.abs(val) > MAX_VALUE_USD)
|
|
243
|
+
throw new Error(`${VALUE_KEY} implausible (|value| > ${MAX_VALUE_USD}), got ${val}`);
|
|
244
|
+
return val;
|
|
245
|
+
}
|
|
246
|
+
/** Read an opt-in `margin_value` off a result's `vars` or its test case's
|
|
247
|
+
* `metadata`/`vars` — the value-per-outcome numerator (rung 8). Returns the USD
|
|
248
|
+
* amount, or undefined when absent (the normal cost-per-outcome case). Throws
|
|
249
|
+
* on a malformed value so the caller can warn and drop it. */
|
|
250
|
+
export function valueFromResult(result) {
|
|
251
|
+
if (!isObject(result))
|
|
252
|
+
return undefined;
|
|
253
|
+
let raw = undefined;
|
|
254
|
+
const vars = result.vars;
|
|
255
|
+
if (isObject(vars) && VALUE_KEY in vars) {
|
|
256
|
+
raw = vars[VALUE_KEY];
|
|
257
|
+
}
|
|
258
|
+
else {
|
|
259
|
+
const testCase = result.testCase;
|
|
260
|
+
if (isObject(testCase)) {
|
|
261
|
+
const meta = testCase.metadata;
|
|
262
|
+
if (isObject(meta) && VALUE_KEY in meta)
|
|
263
|
+
raw = meta[VALUE_KEY];
|
|
264
|
+
const tv = testCase.vars;
|
|
265
|
+
// `== null` (not `=== undefined`) so an explicit `null` in metadata falls
|
|
266
|
+
// through to testCase.vars, matching the Python adapter's `raw is None`.
|
|
267
|
+
if (raw == null && isObject(tv) && VALUE_KEY in tv)
|
|
268
|
+
raw = tv[VALUE_KEY];
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
if (raw === undefined || raw === null)
|
|
272
|
+
return undefined;
|
|
273
|
+
return coerceValue(raw);
|
|
274
|
+
}
|
|
275
|
+
/** A traceable link back to the exact eval case: the test description if the
|
|
276
|
+
* author wrote one, else the prompt id plus the row index. ⛔ Never
|
|
277
|
+
* `prompt.label` (MAR-1548): promptfoo defaults it to the raw prompt text. */
|
|
278
|
+
export function linkFromResult(result, index) {
|
|
279
|
+
if (isObject(result)) {
|
|
280
|
+
const testCase = result.testCase;
|
|
281
|
+
if (isObject(testCase) && typeof testCase.description === "string") {
|
|
282
|
+
const desc = testCase.description.trim();
|
|
283
|
+
if (desc)
|
|
284
|
+
return desc.slice(0, MAX_LINK_LEN);
|
|
285
|
+
}
|
|
286
|
+
// `||` (not `??`) so an empty-string label falls through to the next
|
|
287
|
+
// candidate, matching the Python adapter's `or` chain.
|
|
288
|
+
let label = undefined;
|
|
289
|
+
const prompt = result.prompt;
|
|
290
|
+
if (isObject(prompt))
|
|
291
|
+
label = prompt.id;
|
|
292
|
+
if (!label) {
|
|
293
|
+
label = result.promptId || result.provider;
|
|
294
|
+
if (isObject(label))
|
|
295
|
+
label = label.id || label.label;
|
|
296
|
+
}
|
|
297
|
+
if (typeof label === "string" && label)
|
|
298
|
+
return `${label}:${index}`.slice(0, MAX_LINK_LEN);
|
|
299
|
+
if (typeof label === "number")
|
|
300
|
+
return `${label}:${index}`.slice(0, MAX_LINK_LEN);
|
|
301
|
+
}
|
|
302
|
+
return `promptfoo:${index}`;
|
|
303
|
+
}
|
|
304
|
+
/** Return the `{ passed, score }` verdict for a real result, else undefined.
|
|
305
|
+
* `passed` comes from a boolean `success`; a result without one is not a
|
|
306
|
+
* verdict. `score` is passed through when in [0, 1]; out of that range (a raw
|
|
307
|
+
* metric) falls back to the coherent binary and warns, because the ingest layer
|
|
308
|
+
* rejects an out-of-range score and a lost row is worse than a lost decimal. */
|
|
309
|
+
export function passedAndScore(result, onWarn = defaultWarn) {
|
|
310
|
+
if (!isObject(result))
|
|
311
|
+
return undefined;
|
|
312
|
+
const success = result.success;
|
|
313
|
+
if (typeof success !== "boolean")
|
|
314
|
+
return undefined;
|
|
315
|
+
const passed = success;
|
|
316
|
+
const score = result.score;
|
|
317
|
+
if (typeof score !== "number" || Number.isNaN(score)) {
|
|
318
|
+
return { passed, score: passed ? 1.0 : 0.0 };
|
|
319
|
+
}
|
|
320
|
+
if (score >= 0.0 && score <= 1.0)
|
|
321
|
+
return { passed, score };
|
|
322
|
+
onWarn(`promptfoo score ${score} is outside [0, 1]; recording the binary ` +
|
|
323
|
+
"pass/fail instead so the outcome is not dropped");
|
|
324
|
+
return { passed, score: passed ? 1.0 : 0.0 };
|
|
325
|
+
}
|
|
326
|
+
/** Translate one graded promptfoo result into `MarginMeter.recordOutcome` input.
|
|
327
|
+
* The grader signature is derived from `assertTypes` (never assumed);
|
|
328
|
+
* `valueUsd` rides only when a value was supplied. */
|
|
329
|
+
export function outcomeInput(args) {
|
|
330
|
+
const { qualityMethod, graderKind } = gradingSignature(args.assertTypes);
|
|
331
|
+
const input = {
|
|
332
|
+
workflowId: args.workflowId,
|
|
333
|
+
passed: Boolean(args.passed),
|
|
334
|
+
qualityScore: Number(args.qualityScore),
|
|
335
|
+
qualityMethod,
|
|
336
|
+
graderKind,
|
|
337
|
+
link: String(args.link).slice(0, MAX_LINK_LEN),
|
|
338
|
+
isSimulated: Boolean(args.isSimulated),
|
|
339
|
+
};
|
|
340
|
+
if (args.valueUsd !== undefined)
|
|
341
|
+
input.valueUsd = Number(args.valueUsd);
|
|
342
|
+
return input;
|
|
343
|
+
}
|
|
344
|
+
/** Full translation of one promptfoo result, or undefined if it is not a real
|
|
345
|
+
* verdict. A malformed `margin_value` warns and drops the value (keeping the
|
|
346
|
+
* pass/fail), never throwing into the caller. */
|
|
347
|
+
export function resultToOutcomeInput(result, opts) {
|
|
348
|
+
const onWarn = opts.onWarn ?? defaultWarn;
|
|
349
|
+
const index = opts.index ?? 0;
|
|
350
|
+
const verdict = passedAndScore(result, onWarn);
|
|
351
|
+
if (verdict === undefined)
|
|
352
|
+
return undefined;
|
|
353
|
+
let value;
|
|
354
|
+
try {
|
|
355
|
+
value = valueFromResult(result);
|
|
356
|
+
}
|
|
357
|
+
catch (exc) {
|
|
358
|
+
value = undefined;
|
|
359
|
+
onWarn(`${linkFromResult(result, index)}: ${exc.message}`);
|
|
360
|
+
}
|
|
361
|
+
return outcomeInput({
|
|
362
|
+
workflowId: opts.workflowId,
|
|
363
|
+
passed: verdict.passed,
|
|
364
|
+
qualityScore: verdict.score,
|
|
365
|
+
assertTypes: assertTypesFromResult(result),
|
|
366
|
+
link: linkFromResult(result, index),
|
|
367
|
+
valueUsd: value,
|
|
368
|
+
isSimulated: opts.isSimulated,
|
|
369
|
+
});
|
|
370
|
+
}
|
|
371
|
+
/** Pull the flat list of per-test results out of a parsed promptfoo output.
|
|
372
|
+
* promptfoo has shipped several envelope shapes (the top-level OutputFile with
|
|
373
|
+
* `results.results`, an older `results` list, a bare list); this walks the
|
|
374
|
+
* known shapes rather than pinning one version. */
|
|
375
|
+
export function extractResults(data) {
|
|
376
|
+
if (Array.isArray(data))
|
|
377
|
+
return data.filter(isObject);
|
|
378
|
+
if (!isObject(data))
|
|
379
|
+
return [];
|
|
380
|
+
const results = data.results;
|
|
381
|
+
// New shape: { results: { results: [...] } }
|
|
382
|
+
if (isObject(results) && Array.isArray(results.results)) {
|
|
383
|
+
return results.results.filter(isObject);
|
|
384
|
+
}
|
|
385
|
+
// Older shape: { results: [...] }
|
|
386
|
+
if (Array.isArray(results))
|
|
387
|
+
return results.filter(isObject);
|
|
388
|
+
return [];
|
|
389
|
+
}
|
|
390
|
+
/** Yield `recordOutcome` inputs for every real verdict in a parsed promptfoo
|
|
391
|
+
* output (any envelope shape {@link extractResults} understands). */
|
|
392
|
+
export function iterOutcomes(data, opts) {
|
|
393
|
+
const out = [];
|
|
394
|
+
const results = extractResults(data);
|
|
395
|
+
for (let i = 0; i < results.length; i++) {
|
|
396
|
+
const input = resultToOutcomeInput(results[i], {
|
|
397
|
+
workflowId: opts.workflowId,
|
|
398
|
+
index: i,
|
|
399
|
+
isSimulated: opts.isSimulated,
|
|
400
|
+
onWarn: opts.onWarn,
|
|
401
|
+
});
|
|
402
|
+
if (input !== undefined)
|
|
403
|
+
out.push(input);
|
|
404
|
+
}
|
|
405
|
+
return out;
|
|
406
|
+
}
|
|
407
|
+
/** Emit an outcome per verdict in a parsed promptfoo output through `meter`.
|
|
408
|
+
* Fail-safe: a delivery fault on one row warns and counts an error; it never
|
|
409
|
+
* aborts the batch. Non-verdict rows are counted as skipped. */
|
|
410
|
+
export async function recordResults(meter, data, workflowId, opts = {}) {
|
|
411
|
+
const onWarn = opts.onWarn ?? defaultWarn;
|
|
412
|
+
const results = extractResults(data);
|
|
413
|
+
let recorded = 0;
|
|
414
|
+
let errors = 0;
|
|
415
|
+
let skipped = 0;
|
|
416
|
+
for (let i = 0; i < results.length; i++) {
|
|
417
|
+
const input = resultToOutcomeInput(results[i], {
|
|
418
|
+
workflowId,
|
|
419
|
+
index: i,
|
|
420
|
+
isSimulated: opts.isSimulated,
|
|
421
|
+
onWarn,
|
|
422
|
+
});
|
|
423
|
+
if (input === undefined) {
|
|
424
|
+
skipped++;
|
|
425
|
+
continue;
|
|
426
|
+
}
|
|
427
|
+
try {
|
|
428
|
+
await meter.recordOutcome(input);
|
|
429
|
+
recorded++;
|
|
430
|
+
}
|
|
431
|
+
catch (exc) {
|
|
432
|
+
// Never let a delivery fault abort the batch. (The meter is fail-safe by
|
|
433
|
+
// default and resolves rather than rejects; this guards a strict meter.)
|
|
434
|
+
errors++;
|
|
435
|
+
onWarn(`failed to record outcome for ${input.link}: ${exc.message}`);
|
|
436
|
+
}
|
|
437
|
+
}
|
|
438
|
+
try {
|
|
439
|
+
await meter.flush();
|
|
440
|
+
}
|
|
441
|
+
catch (exc) {
|
|
442
|
+
onWarn(`flush failed: ${exc.message}`);
|
|
443
|
+
}
|
|
444
|
+
return { recorded, errors, skipped };
|
|
445
|
+
}
|
|
446
|
+
//# sourceMappingURL=promptfoo.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"promptfoo.js","sourceRoot":"","sources":["../src/promptfoo.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuDG;AAMH,gFAAgF;AAChF,sDAAsD;AACtD,MAAM,YAAY,GAAG,cAAc,CAAC;AACpC,MAAM,SAAS,GAAG,WAAW,CAAC;AAC9B,MAAM,cAAc,GAAG,SAAS,CAAC;AACjC,MAAM,WAAW,GAAG,MAAM,CAAC;AAC3B,MAAM,YAAY,GAAG,OAAO,CAAC;AAC7B,MAAM,cAAc,GAAG,SAAS,CAAC;AAEjC,gFAAgF;AAChF,sEAAsE;AACtE,MAAM,YAAY,GAAG,GAAG,CAAC;AAEzB,MAAM,SAAS,GAAG,cAAc,CAAC;AACjC,qFAAqF;AACrF,MAAM,aAAa,GAAG,UAAU,CAAC;AAEjC;;gFAEgF;AAChF,MAAM,YAAY,GAAwB,IAAI,GAAG,CAAC;IAChD,QAAQ;IACR,UAAU;IACV,WAAW;IACX,aAAa;IACb,aAAa;IACb,cAAc;IACd,cAAc;IACd,YAAY;IACZ,OAAO;IACP,QAAQ;IACR,cAAc;IACd,OAAO;IACP,aAAa;IACb,OAAO;IACP,aAAa;IACb,2BAA2B;IAC3B,wBAAwB;IACxB,YAAY;IACZ,QAAQ;IACR,SAAS;IACT,aAAa;IACb,QAAQ;IACR,MAAM;IACN,MAAM;IACN,QAAQ;IACR,SAAS;IACT,MAAM;IACN,YAAY;IACZ,iBAAiB;IACjB,cAAc;IACd,YAAY;IACZ,OAAO;CACR,CAAC,CAAC;AAEH;;yEAEyE;AACzE,MAAM,aAAa,GAAwB,IAAI,GAAG,CAAC;IACjD,WAAW;IACX,qBAAqB;IACrB,uBAAuB;IACvB,YAAY;IACZ,iBAAiB;IACjB,kBAAkB;IAClB,qBAAqB;IACrB,eAAe;IACf,SAAS;IACT,YAAY;IACZ,YAAY;IACZ,YAAY;IACZ,IAAI;IACJ,OAAO;IACP,iBAAiB;CAClB,CAAC,CAAC;AAOH,MAAM,WAAW,GAAW,CAAC,OAAO,EAAE,EAAE;IACtC,uEAAuE;IACvE,MAAM,CAAC,GAAI,UAAkE;SAC1E,OAAO,CAAC;IACX,IAAI,CAAC,IAAI,OAAO,CAAC,CAAC,WAAW,KAAK,UAAU;QAAE,CAAC,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;;QAChE,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AAC7B,CAAC,CAAC;AAcF,SAAS,QAAQ,CAAC,CAAU;IAC1B,OAAO,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AAClE,CAAC;AAED;;;YAGY;AACZ,MAAM,UAAU,cAAc,CAAC,GAAY;IACzC,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAC;IAC9C,IAAI,CAAC,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IACvD,IAAI,CAAC,CAAC;QAAE,OAAO,SAAS,CAAC;IACzB,IAAI,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK;QAAE,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACvD,OAAO,CAAC,IAAI,SAAS,CAAC;AACxB,CAAC;AAED;;;;;qDAKqD;AACrD,MAAM,UAAU,gBAAgB,CAAC,WAA8B;IAC7D,IAAI,QAAQ,GAAG,KAAK,CAAC;IACrB,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,IAAI,UAAU,GAAG,KAAK,CAAC;IACvB,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;QAC9B,MAAM,CAAC,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;QAC9B,IAAI,CAAC,KAAK,SAAS;YAAE,SAAS;QAC9B,IAAI,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,QAAQ,GAAG,IAAI,CAAC;aACrC,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,OAAO,GAAG,IAAI,CAAC;;YACxC,UAAU,GAAG,IAAI,CAAC;IACzB,CAAC;IACD,IAAI,QAAQ;QAAE,OAAO,EAAE,aAAa,EAAE,SAAS,EAAE,UAAU,EAAE,YAAY,EAAE,CAAC;IAC5E,IAAI,OAAO,IAAI,CAAC,UAAU;QACxB,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,UAAU,EAAE,WAAW,EAAE,CAAC;IAClE,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,UAAU,EAAE,cAAc,EAAE,CAAC;AACvE,CAAC;AAED;;;kDAGkD;AAClD,MAAM,UAAU,qBAAqB,CAAC,MAAe;IACnD,MAAM,KAAK,GAAc,EAAE,CAAC;IAC5B,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;QAAE,OAAO,KAAK,CAAC;IACpC,MAAM,OAAO,GAAG,MAAM,CAAC,aAAa,CAAC;IACrC,IAAI,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;QACtB,MAAM,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC;QAC5C,IAAI,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;YAC9B,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;gBAC9B,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;oBAC/C,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;gBAClC,CAAC;YACH,CAAC;QACH,CAAC;QACD,IAAI,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACtE,CAAC;IACD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IACnC,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;IACjC,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QACzD,KAAK,MAAM,CAAC,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;YAChC,IAAI,QAAQ,CAAC,CAAC,CAAC;gBAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,iFAAiF;AACjF,+EAA+E;AAC/E,iFAAiF;AACjF,+EAA+E;AAC/E,kFAAkF;AAClF,MAAM,UAAU,GAAG,yCAAyC,CAAC;AAE7D;;;mDAGmD;AACnD,SAAS,WAAW,CAAC,GAAY;IAC/B,IAAI,OAAO,GAAG,KAAK,SAAS;QAC1B,MAAM,IAAI,KAAK,CAAC,GAAG,SAAS,0BAA0B,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC/E,IAAI,GAAW,CAAC;IAChB,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QAC5B,GAAG,GAAG,GAAG,CAAC;IACZ,CAAC;SAAM,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QACnC,MAAM,CAAC,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;QACrB,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC;YACrB,MAAM,IAAI,KAAK,CAAC,GAAG,SAAS,0BAA0B,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC/E,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CAAC,GAAG,SAAS,0BAA0B,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC/E,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,GAAG,SAAS,iCAAiC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACtF,mFAAmF;IACnF,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,aAAa;QAC/B,MAAM,IAAI,KAAK,CAAC,GAAG,SAAS,2BAA2B,aAAa,UAAU,GAAG,EAAE,CAAC,CAAC;IACvF,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;+DAG+D;AAC/D,MAAM,UAAU,eAAe,CAAC,MAAe;IAC7C,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;QAAE,OAAO,SAAS,CAAC;IACxC,IAAI,GAAG,GAAY,SAAS,CAAC;IAC7B,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;IACzB,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,SAAS,IAAI,IAAI,EAAE,CAAC;QACxC,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC;IACxB,CAAC;SAAM,CAAC;QACN,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;QACjC,IAAI,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,CAAC;YAC/B,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,SAAS,IAAI,IAAI;gBAAE,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC;YAC/D,MAAM,EAAE,GAAG,QAAQ,CAAC,IAAI,CAAC;YACzB,0EAA0E;YAC1E,yEAAyE;YACzE,IAAI,GAAG,IAAI,IAAI,IAAI,QAAQ,CAAC,EAAE,CAAC,IAAI,SAAS,IAAI,EAAE;gBAAE,GAAG,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAC1E,CAAC;IACH,CAAC;IACD,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI;QAAE,OAAO,SAAS,CAAC;IACxD,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC;AAC1B,CAAC;AAED;;+EAE+E;AAC/E,MAAM,UAAU,cAAc,CAAC,MAAe,EAAE,KAAa;IAC3D,IAAI,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QACrB,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;QACjC,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,OAAO,QAAQ,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;YACnE,MAAM,IAAI,GAAG,QAAQ,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;YACzC,IAAI,IAAI;gBAAE,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;QAC/C,CAAC;QACD,qEAAqE;QACrE,uDAAuD;QACvD,IAAI,KAAK,GAAY,SAAS,CAAC;QAC/B,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAC7B,IAAI,QAAQ,CAAC,MAAM,CAAC;YAAE,KAAK,GAAG,MAAM,CAAC,EAAE,CAAC;QACxC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,KAAK,GAAG,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC;YAC3C,IAAI,QAAQ,CAAC,KAAK,CAAC;gBAAE,KAAK,GAAG,KAAK,CAAC,EAAE,IAAI,KAAK,CAAC,KAAK,CAAC;QACvD,CAAC;QACD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK;YAAE,OAAO,GAAG,KAAK,IAAI,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;QAC1F,IAAI,OAAO,KAAK,KAAK,QAAQ;YAAE,OAAO,GAAG,KAAK,IAAI,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;IACnF,CAAC;IACD,OAAO,aAAa,KAAK,EAAE,CAAC;AAC9B,CAAC;AAED;;;;iFAIiF;AACjF,MAAM,UAAU,cAAc,CAC5B,MAAe,EACf,SAAiB,WAAW;IAE5B,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;QAAE,OAAO,SAAS,CAAC;IACxC,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;IAC/B,IAAI,OAAO,OAAO,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IACnD,MAAM,MAAM,GAAG,OAAO,CAAC;IACvB,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;IAC3B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;QACrD,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;IAC/C,CAAC;IACD,IAAI,KAAK,IAAI,GAAG,IAAI,KAAK,IAAI,GAAG;QAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;IAC3D,MAAM,CACJ,mBAAmB,KAAK,2CAA2C;QACjE,iDAAiD,CACpD,CAAC;IACF,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;AAC/C,CAAC;AAaD;;uDAEuD;AACvD,MAAM,UAAU,YAAY,CAAC,IAAsB;IACjD,MAAM,EAAE,aAAa,EAAE,UAAU,EAAE,GAAG,gBAAgB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACzE,MAAM,KAAK,GAAuB;QAChC,UAAU,EAAE,IAAI,CAAC,UAAU;QAC3B,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC;QAC5B,YAAY,EAAE,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC;QACvC,aAAa;QACb,UAAU;QACV,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC;QAC9C,WAAW,EAAE,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC;KACvC,CAAC;IACF,IAAI,IAAI,CAAC,QAAQ,KAAK,SAAS;QAAE,KAAK,CAAC,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACxE,OAAO,KAAK,CAAC;AACf,CAAC;AAUD;;kDAEkD;AAClD,MAAM,UAAU,oBAAoB,CAClC,MAAe,EACf,IAAsB;IAEtB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,WAAW,CAAC;IAC1C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;IAC9B,MAAM,OAAO,GAAG,cAAc,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC/C,IAAI,OAAO,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC5C,IAAI,KAAyB,CAAC;IAC9B,IAAI,CAAC;QACH,KAAK,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC;IAClC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,KAAK,GAAG,SAAS,CAAC;QAClB,MAAM,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,KAAK,CAAC,KAAM,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;IACxE,CAAC;IACD,OAAO,YAAY,CAAC;QAClB,UAAU,EAAE,IAAI,CAAC,UAAU;QAC3B,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,YAAY,EAAE,OAAO,CAAC,KAAK;QAC3B,WAAW,EAAE,qBAAqB,CAAC,MAAM,CAAC;QAC1C,IAAI,EAAE,cAAc,CAAC,MAAM,EAAE,KAAK,CAAC;QACnC,QAAQ,EAAE,KAAK;QACf,WAAW,EAAE,IAAI,CAAC,WAAW;KAC9B,CAAC,CAAC;AACL,CAAC;AAED;;;oDAGoD;AACpD,MAAM,UAAU,cAAc,CAAC,IAAa;IAC1C,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IACtD,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;QAAE,OAAO,EAAE,CAAC;IAC/B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;IAC7B,6CAA6C;IAC7C,IAAI,QAAQ,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACxD,OAAO,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC1C,CAAC;IACD,kCAAkC;IAClC,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;QAAE,OAAO,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC5D,OAAO,EAAE,CAAC;AACZ,CAAC;AAED;sEACsE;AACtE,MAAM,UAAU,YAAY,CAC1B,IAAa,EACb,IAAoE;IAEpE,MAAM,GAAG,GAAyB,EAAE,CAAC;IACrC,MAAM,OAAO,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;IACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,KAAK,GAAG,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;YAC7C,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,KAAK,EAAE,CAAC;YACR,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,MAAM,EAAE,IAAI,CAAC,MAAM;SACpB,CAAC,CAAC;QACH,IAAI,KAAK,KAAK,SAAS;YAAE,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC3C,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAiBD;;iEAEiE;AACjE,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,KAAkB,EAClB,IAAa,EACb,UAAkB,EAClB,OAAmD,EAAE;IAErD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,WAAW,CAAC;IAC1C,MAAM,OAAO,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;IACrC,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,KAAK,GAAG,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;YAC7C,UAAU;YACV,KAAK,EAAE,CAAC;YACR,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,MAAM;SACP,CAAC,CAAC;QACH,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,OAAO,EAAE,CAAC;YACV,SAAS;QACX,CAAC;QACD,IAAI,CAAC;YACH,MAAM,KAAK,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;YACjC,QAAQ,EAAE,CAAC;QACb,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,yEAAyE;YACzE,yEAAyE;YACzE,MAAM,EAAE,CAAC;YACT,MAAM,CAAC,gCAAgC,KAAK,CAAC,IAAI,KAAM,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;QAClF,CAAC;IACH,CAAC;IACD,IAAI,CAAC;QACH,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;IACtB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,CAAC,iBAAkB,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;IACpD,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;AACvC,CAAC"}
|
package/dist/providers.d.ts
CHANGED
|
@@ -80,6 +80,28 @@ export interface Usage {
|
|
|
80
80
|
* ordinary verbose output. Optional/0 on a call with no reasoning breakdown.
|
|
81
81
|
*/
|
|
82
82
|
reasoningTokens?: number;
|
|
83
|
+
/**
|
|
84
|
+
* The provider's OWN reason the generation ended, read VERBATIM — Anthropic
|
|
85
|
+
* `stop_reason` (`end_turn` / `max_tokens` / `tool_use` / `refusal`), OpenAI &
|
|
86
|
+
* Gemini `finish_reason`, Cohere `finish_reason`, Bedrock `stopReason`. A raw
|
|
87
|
+
* SIGNAL, never a billing verdict: a blocked-and-fell-back call (Fable 5's safety
|
|
88
|
+
* fallback) is often NOT billed, but whether it was depends on the customer's
|
|
89
|
+
* opt-in config, so the stop reason alone can never be turned into "this was
|
|
90
|
+
* free" — that is settled by the invoice (MAR-209). Captured only so a downstream
|
|
91
|
+
* reconcile CAN ask the question; `undefined` is an honest UNKNOWN, first-class
|
|
92
|
+
* and never defaulted to a value.
|
|
93
|
+
*/
|
|
94
|
+
stopReason?: string;
|
|
95
|
+
/**
|
|
96
|
+
* OpenRouter's own billed cost (`usage.cost`) for a FAN-OUT slug
|
|
97
|
+
* (`openrouter/fusion`), which runs N panel calls plus an analyst call per request,
|
|
98
|
+
* so no token rate prices it (MAR-1309). `undefined` for every other call — the
|
|
99
|
+
* server then prices tokens off its table exactly as before.
|
|
100
|
+
*/
|
|
101
|
+
billedCostUsd?: number;
|
|
102
|
+
/** Who served the call when the RESPONSE says so: OpenRouter's top-level `provider` (the
|
|
103
|
+
* upstream host it routed to). `undefined` when the response is silent, never inferred. */
|
|
104
|
+
servingHost?: string;
|
|
83
105
|
}
|
|
84
106
|
/** The response object matched no provider shape we can meter. */
|
|
85
107
|
export declare class UsageExtractionError extends Error {
|
|
@@ -94,6 +116,8 @@ type Any = Record<string, unknown> | null | undefined | unknown;
|
|
|
94
116
|
* instrumenter always does).
|
|
95
117
|
*/
|
|
96
118
|
export declare function detectProvider(response: Any): ProviderName | undefined;
|
|
119
|
+
/** True for `openrouter/fusion` and its siblings (many billed calls per request). */
|
|
120
|
+
export declare function isFanoutSlug(model?: string): boolean;
|
|
97
121
|
/**
|
|
98
122
|
* Recover the metered `Usage` of one call from its provider response. `provider`
|
|
99
123
|
* forces the shape (the instrumenter always knows which client it patched);
|
|
@@ -134,6 +158,16 @@ export declare function usageFromStreamChunk(chunk: Any, provider?: ProviderName
|
|
|
134
158
|
* the largest REAL count each field reached.
|
|
135
159
|
*/
|
|
136
160
|
export declare function mergeStreamUsage(a: Usage, b: Usage): Usage;
|
|
161
|
+
/**
|
|
162
|
+
* The stop reason one stream chunk carries, if any. Never throws.
|
|
163
|
+
*
|
|
164
|
+
* A stream rarely puts the reason and the usage on the same chunk (MAR-1516). OpenAI's
|
|
165
|
+
* finish chunk has no `usage` and the usage chunk after it has empty `choices`; Bedrock
|
|
166
|
+
* `converseStream` sends `stopReason` in `messageStop` and usage in a separate
|
|
167
|
+
* `metadata` event. A reader that only looks at usage-bearing chunks records every
|
|
168
|
+
* streamed call with no reason, so this reads it from whichever chunk holds it.
|
|
169
|
+
*/
|
|
170
|
+
export declare function streamChunkStopReason(chunk: Any): string | undefined;
|
|
137
171
|
/**
|
|
138
172
|
* Accumulates the metered usage of one STREAMING call across its chunks.
|
|
139
173
|
*
|
|
@@ -148,6 +182,8 @@ export declare class StreamUsageAccumulator {
|
|
|
148
182
|
private provider?;
|
|
149
183
|
private model?;
|
|
150
184
|
private usage;
|
|
185
|
+
private stopReason;
|
|
186
|
+
private host;
|
|
151
187
|
constructor(provider?: ProviderName, model?: string);
|
|
152
188
|
/** Fold one chunk's usage (if any) into the running total. Never throws. */
|
|
153
189
|
observe(chunk: Any): void;
|
package/dist/providers.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"providers.d.ts","sourceRoot":"","sources":["../src/providers.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AAEH,eAAO,MAAM,MAAM,WAAW,CAAC;AAC/B,eAAO,MAAM,SAAS,cAAc,CAAC;AACrC,eAAO,MAAM,MAAM,WAAW,CAAC;AAC/B,eAAO,MAAM,MAAM,WAAW,CAAC;AAC/B,eAAO,MAAM,OAAO,YAAY,CAAC;AAEjC,MAAM,MAAM,YAAY,GAAG,QAAQ,GAAG,WAAW,GAAG,QAAQ,GAAG,QAAQ,GAAG,SAAS,CAAC;AAEpF,6EAA6E;AAC7E,MAAM,WAAW,KAAK;IACpB,QAAQ,EAAE,YAAY,CAAC;IACvB,mFAAmF;IACnF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB;;;;;;;OAOG;IACH,mBAAmB,EAAE,MAAM,CAAC;IAC5B;;;;;OAKG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B;;;;;;;OAOG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"providers.d.ts","sourceRoot":"","sources":["../src/providers.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AAEH,eAAO,MAAM,MAAM,WAAW,CAAC;AAC/B,eAAO,MAAM,SAAS,cAAc,CAAC;AACrC,eAAO,MAAM,MAAM,WAAW,CAAC;AAC/B,eAAO,MAAM,MAAM,WAAW,CAAC;AAC/B,eAAO,MAAM,OAAO,YAAY,CAAC;AAEjC,MAAM,MAAM,YAAY,GAAG,QAAQ,GAAG,WAAW,GAAG,QAAQ,GAAG,QAAQ,GAAG,SAAS,CAAC;AAEpF,6EAA6E;AAC7E,MAAM,WAAW,KAAK;IACpB,QAAQ,EAAE,YAAY,CAAC;IACvB,mFAAmF;IACnF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB;;;;;;;OAOG;IACH,mBAAmB,EAAE,MAAM,CAAC;IAC5B;;;;;OAKG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B;;;;;;;OAOG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;;;;;;;;OAUG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;;;OAKG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;gGAC4F;IAC5F,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,kEAAkE;AAClE,qBAAa,oBAAqB,SAAQ,KAAK;gBACjC,OAAO,EAAE,MAAM;CAI5B;AAED,KAAK,GAAG,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,GAAG,SAAS,GAAG,OAAO,CAAC;AA2DhE;;;;;;GAMG;AACH,wBAAgB,cAAc,CAAC,QAAQ,EAAE,GAAG,GAAG,YAAY,GAAG,SAAS,CAuBtE;AAoBD,qFAAqF;AACrF,wBAAgB,YAAY,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAGpD;AAkND;;;;;;;GAOG;AACH,wBAAgB,YAAY,CAC1B,QAAQ,EAAE,GAAG,EACb,IAAI,GAAE;IAAE,QAAQ,CAAC,EAAE,YAAY,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAO,GACrD,KAAK,CAUP;AAeD;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,oBAAoB,CAClC,KAAK,EAAE,GAAG,EACV,QAAQ,CAAC,EAAE,YAAY,EACvB,KAAK,CAAC,EAAE,MAAM,GACb,KAAK,GAAG,SAAS,CAoBnB;AA+ED;;;;;;;;GAQG;AACH,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,GAAG,KAAK,CAsB1D;AAED;;;;;;;;GAQG;AACH,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,GAAG,GAAG,MAAM,GAAG,SAAS,CAqBpE;AAED;;;;;;;;;GASG;AACH,qBAAa,sBAAsB;IACjC,OAAO,CAAC,QAAQ,CAAC,CAAe;IAChC,OAAO,CAAC,KAAK,CAAC,CAAS;IACvB,OAAO,CAAC,KAAK,CAAgC;IAC7C,OAAO,CAAC,UAAU,CAAiC;IACnD,OAAO,CAAC,IAAI,CAAiC;gBAIjC,QAAQ,CAAC,EAAE,YAAY,EAAE,KAAK,CAAC,EAAE,MAAM;IAKnD,4EAA4E;IAC5E,OAAO,CAAC,KAAK,EAAE,GAAG,GAAG,IAAI;IAazB,4EAA4E;IAC5E,QAAQ,IAAI,KAAK,GAAG,SAAS;CAS9B"}
|