@artemiskit/core 0.5.0 → 0.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/dist/artifacts/manifest.d.ts +5 -1
- package/dist/artifacts/manifest.d.ts.map +1 -1
- package/dist/artifacts/types.d.ts +87 -0
- package/dist/artifacts/types.d.ts.map +1 -1
- package/dist/index.js +622 -400
- package/dist/provenance/execution-provenance.d.ts +11 -0
- package/dist/provenance/execution-provenance.d.ts.map +1 -0
- package/dist/provenance/index.d.ts +1 -0
- package/dist/provenance/index.d.ts.map +1 -1
- package/dist/runner/executor.d.ts.map +1 -1
- package/dist/runner/runner.d.ts.map +1 -1
- package/dist/runner/types.d.ts +15 -1
- package/dist/runner/types.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/artifacts/manifest.test.ts +123 -1
- package/src/artifacts/manifest.ts +25 -29
- package/src/artifacts/types.ts +262 -0
- package/src/provenance/execution-provenance.test.ts +59 -0
- package/src/provenance/execution-provenance.ts +53 -0
- package/src/provenance/index.ts +1 -0
- package/src/runner/executor.test.ts +60 -0
- package/src/runner/executor.ts +145 -11
- package/src/runner/release-validation.test.ts +36 -0
- package/src/runner/runner.ts +31 -1
- package/src/runner/types.ts +9 -1
- package/src/scenario/schema.ts +1 -1
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/** Bounded execution provenance derived from declared and observed run evidence. */
|
|
2
|
+
import type { CaseResult, ExecutionProvenance } from '../artifacts/types';
|
|
3
|
+
export declare function createExecutionProvenance(options: {
|
|
4
|
+
provider: string;
|
|
5
|
+
requestedModel?: string;
|
|
6
|
+
temperature?: number;
|
|
7
|
+
maxTokens?: number;
|
|
8
|
+
seed?: number;
|
|
9
|
+
cases: CaseResult[];
|
|
10
|
+
}): ExecutionProvenance;
|
|
11
|
+
//# sourceMappingURL=execution-provenance.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"execution-provenance.d.ts","sourceRoot":"","sources":["../../src/provenance/execution-provenance.ts"],"names":[],"mappings":"AAAA,oFAAoF;AAEpF,OAAO,KAAK,EAAE,UAAU,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAE1E,wBAAgB,yBAAyB,CAAC,OAAO,EAAE;IACjD,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,UAAU,EAAE,CAAC;CACrB,GAAG,mBAAmB,CA2BtB"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/provenance/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,OAAO,CAAC;AACnC,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/provenance/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,OAAO,CAAC;AACnC,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAC7D,OAAO,EAAE,yBAAyB,EAAE,MAAM,wBAAwB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../src/runner/executor.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../src/runner/executor.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAKV,UAAU,EACX,MAAM,oBAAoB,CAAC;AAI5B,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAQnD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAwE/C;;GAEG;AACH,wBAAsB,WAAW,CAC/B,QAAQ,EAAE,QAAQ,EAClB,OAAO,EAAE,eAAe,GACvB,OAAO,CAAC,UAAU,CAAC,CA6ErB"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/runner/runner.ts"],"names":[],"mappings":"AAAA;;GAEG;
|
|
1
|
+
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/runner/runner.ts"],"names":[],"mappings":"AAAA;;GAEG;AAQH,OAAO,KAAK,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAErD;;GAEG;AACH,wBAAsB,WAAW,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,CAyJzE;AAED;;GAEG;AACH,wBAAsB,YAAY,CAAC,WAAW,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CASlF"}
|
package/dist/runner/types.d.ts
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* Runner types and interfaces
|
|
3
3
|
*/
|
|
4
4
|
import type { ModelClient } from '../adapters/types';
|
|
5
|
-
import type { CaseResult, ResolvedConfig, RunManifest } from '../artifacts/types';
|
|
5
|
+
import type { CaseResult, CostProvenance, ResolvedConfig, RunManifest } from '../artifacts/types';
|
|
6
6
|
import type { RedactionConfig } from '../redaction/types';
|
|
7
7
|
import type { Scenario } from '../scenario/schema';
|
|
8
8
|
import type { ToolExecutor } from '../tools';
|
|
@@ -26,6 +26,13 @@ export interface RunOptions {
|
|
|
26
26
|
timeout?: number;
|
|
27
27
|
/** Number of retries per case */
|
|
28
28
|
retries?: number;
|
|
29
|
+
/** One-based coordinate for an independently planned repetition. */
|
|
30
|
+
repetition?: {
|
|
31
|
+
index: number;
|
|
32
|
+
total: number;
|
|
33
|
+
};
|
|
34
|
+
/** Attested or operator-supplied monetary evidence; omitted means unavailable. */
|
|
35
|
+
costProvenance?: CostProvenance;
|
|
29
36
|
/** Redaction configuration (CLI overrides scenario) */
|
|
30
37
|
redaction?: RedactionConfig;
|
|
31
38
|
/** SDK-only executor for explicitly supplied real tools. */
|
|
@@ -52,8 +59,15 @@ export interface RunResult {
|
|
|
52
59
|
export interface ExecutorContext {
|
|
53
60
|
client: ModelClient;
|
|
54
61
|
scenario: Scenario;
|
|
62
|
+
/** Effective model selected outside the scenario, such as a CLI override. */
|
|
63
|
+
requestedModel?: string;
|
|
55
64
|
timeout?: number;
|
|
56
65
|
retries?: number;
|
|
66
|
+
runId?: string;
|
|
67
|
+
repetition?: {
|
|
68
|
+
index: number;
|
|
69
|
+
total: number;
|
|
70
|
+
};
|
|
57
71
|
/** Redaction configuration for this execution */
|
|
58
72
|
redaction?: RedactionConfig;
|
|
59
73
|
toolExecutor?: ToolExecutor;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/runner/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAAE,UAAU,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/runner/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAAE,UAAU,EAAE,cAAc,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAClG,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAE7C;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,0BAA0B;IAC1B,QAAQ,EAAE,QAAQ,CAAC;IACnB,0BAA0B;IAC1B,MAAM,EAAE,WAAW,CAAC;IACpB,oCAAoC;IACpC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,kDAAkD;IAClD,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,2BAA2B;IAC3B,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,oCAAoC;IACpC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iCAAiC;IACjC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,oEAAoE;IACpE,UAAU,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IAC9C,kFAAkF;IAClF,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,uDAAuD;IACvD,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,4DAA4D;IAC5D,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,oCAAoC;IACpC,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IAC5E,oCAAoC;IACpC,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;CACxC;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,6BAA6B;IAC7B,QAAQ,EAAE,WAAW,CAAC;IACtB,8BAA8B;IAC9B,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,+BAA+B;IAC/B,OAAO,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,WAAW,CAAC;IACpB,QAAQ,EAAE,QAAQ,CAAC;IACnB,6EAA6E;IAC7E,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IAC9C,iDAAiD;IACjD,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,YAAY,CAAC,EAAE,YAAY,CAAC;CAC7B"}
|
package/package.json
CHANGED
|
@@ -55,13 +55,19 @@ describe('createRunManifest', () => {
|
|
|
55
55
|
endTime,
|
|
56
56
|
});
|
|
57
57
|
|
|
58
|
-
expect(manifest.version).toBe('1.
|
|
58
|
+
expect(manifest.version).toBe('1.4');
|
|
59
59
|
expect(manifest.project).toBe('test-project');
|
|
60
60
|
expect(manifest.run_id).toBeTruthy();
|
|
61
61
|
expect(manifest.run_id.length).toBe(12);
|
|
62
62
|
expect(manifest.config.scenario).toBe('test-scenario');
|
|
63
63
|
expect(manifest.config.provider).toBe('openai');
|
|
64
64
|
expect(manifest.config.model).toBe('gpt-4');
|
|
65
|
+
expect(manifest.metrics.cost).toBeUndefined();
|
|
66
|
+
expect(manifest.metrics.cost_provenance).toEqual({
|
|
67
|
+
schema_version: '1',
|
|
68
|
+
status: 'unavailable',
|
|
69
|
+
unavailable_reason: 'provider_billing_not_recorded',
|
|
70
|
+
});
|
|
65
71
|
});
|
|
66
72
|
|
|
67
73
|
test('calculates metrics correctly', () => {
|
|
@@ -189,6 +195,30 @@ describe('createRunManifest', () => {
|
|
|
189
195
|
expect(() => assertRunManifestIntegrity(manifest)).not.toThrow();
|
|
190
196
|
});
|
|
191
197
|
|
|
198
|
+
test('retains target and evaluator execution provenance separately', () => {
|
|
199
|
+
const manifest = createRunManifest({
|
|
200
|
+
project: 'test-project',
|
|
201
|
+
config: { scenario: 'test-scenario', provider: 'openai' },
|
|
202
|
+
executionProvenance: {
|
|
203
|
+
schema_version: '1',
|
|
204
|
+
target: {
|
|
205
|
+
provider: 'openai',
|
|
206
|
+
requested_models: ['gpt-requested'],
|
|
207
|
+
observed_models: ['gpt-observed'],
|
|
208
|
+
generation: { temperature: 0, max_tokens: 100, seed: 42 },
|
|
209
|
+
},
|
|
210
|
+
evaluator: { models: ['judge-model'] },
|
|
211
|
+
},
|
|
212
|
+
cases: mockCases,
|
|
213
|
+
startTime: new Date(),
|
|
214
|
+
endTime: new Date(),
|
|
215
|
+
});
|
|
216
|
+
|
|
217
|
+
expect(manifest.execution_provenance?.target.observed_models).toEqual(['gpt-observed']);
|
|
218
|
+
expect(manifest.execution_provenance?.evaluator?.models).toEqual(['judge-model']);
|
|
219
|
+
expect(() => assertRunManifestIntegrity(manifest)).not.toThrow();
|
|
220
|
+
});
|
|
221
|
+
|
|
192
222
|
test('includes provenance information', () => {
|
|
193
223
|
const manifest = createRunManifest({
|
|
194
224
|
project: 'test-project',
|
|
@@ -303,5 +333,97 @@ describe('createRunManifest', () => {
|
|
|
303
333
|
expect(() =>
|
|
304
334
|
assertRunManifestIntegrity({ ...historical, workload_identity: { schema_version: '1' } })
|
|
305
335
|
).toThrow('malformed workload identity');
|
|
336
|
+
expect(() =>
|
|
337
|
+
assertRunManifestIntegrity({
|
|
338
|
+
...historical,
|
|
339
|
+
execution_provenance: { schema_version: '1', target: { provider: '' } },
|
|
340
|
+
})
|
|
341
|
+
).toThrow('malformed execution provenance');
|
|
342
|
+
expect(() =>
|
|
343
|
+
assertRunManifestIntegrity({
|
|
344
|
+
...historical,
|
|
345
|
+
cases: [{ ...historical.cases[0], target: { provider: '', observed_models: ['x'] } }],
|
|
346
|
+
})
|
|
347
|
+
).toThrow('malformed target evidence');
|
|
348
|
+
});
|
|
349
|
+
|
|
350
|
+
test('retains attested cost evidence but rejects malformed monetary claims', () => {
|
|
351
|
+
const manifest = createRunManifest({
|
|
352
|
+
project: 'test-project',
|
|
353
|
+
config: { scenario: 'cost', provider: 'openai' },
|
|
354
|
+
costProvenance: {
|
|
355
|
+
schema_version: '1',
|
|
356
|
+
status: 'known',
|
|
357
|
+
amount: 1.25,
|
|
358
|
+
currency: 'USD',
|
|
359
|
+
source: 'provider_billing',
|
|
360
|
+
recorded_at: '2026-09-10T00:00:00.000Z',
|
|
361
|
+
},
|
|
362
|
+
cases: mockCases,
|
|
363
|
+
startTime: new Date(),
|
|
364
|
+
endTime: new Date(),
|
|
365
|
+
});
|
|
366
|
+
|
|
367
|
+
expect(manifest.metrics.cost_provenance?.status).toBe('known');
|
|
368
|
+
expect(() => assertRunManifestIntegrity(manifest)).not.toThrow();
|
|
369
|
+
expect(() =>
|
|
370
|
+
assertRunManifestIntegrity({
|
|
371
|
+
...manifest,
|
|
372
|
+
metrics: {
|
|
373
|
+
...manifest.metrics,
|
|
374
|
+
cost_provenance: { schema_version: '1', status: 'known', amount: 1 },
|
|
375
|
+
},
|
|
376
|
+
})
|
|
377
|
+
).toThrow('malformed cost provenance');
|
|
378
|
+
});
|
|
379
|
+
|
|
380
|
+
test('validates bounded retry-chain evidence without requiring it from historical manifests', () => {
|
|
381
|
+
const manifest = createRunManifest({
|
|
382
|
+
project: 'test-project',
|
|
383
|
+
config: { scenario: 'retry', provider: 'fixture' },
|
|
384
|
+
attemptEvidence: {
|
|
385
|
+
schema_version: '1',
|
|
386
|
+
repetition: { index: 1, total: 2 },
|
|
387
|
+
retry_policy: { default_max_retries: 1, backoff: 'exponential', initial_delay_ms: 1000 },
|
|
388
|
+
timeout: { default_ms: 5000 },
|
|
389
|
+
},
|
|
390
|
+
cases: [
|
|
391
|
+
{
|
|
392
|
+
...mockCases[0],
|
|
393
|
+
attempts: 2,
|
|
394
|
+
attempt_evidence: [
|
|
395
|
+
{
|
|
396
|
+
attempt_id: 'run:case-1:1',
|
|
397
|
+
retry_chain_id: 'run:case-1',
|
|
398
|
+
repetition_index: 1,
|
|
399
|
+
attempt_number: 1,
|
|
400
|
+
status: 'error',
|
|
401
|
+
included_in_outcome: false,
|
|
402
|
+
latency_ms: 3,
|
|
403
|
+
error_code: 'target_error',
|
|
404
|
+
},
|
|
405
|
+
{
|
|
406
|
+
attempt_id: 'run:case-1:2',
|
|
407
|
+
retry_chain_id: 'run:case-1',
|
|
408
|
+
repetition_index: 1,
|
|
409
|
+
attempt_number: 2,
|
|
410
|
+
status: 'passed',
|
|
411
|
+
included_in_outcome: true,
|
|
412
|
+
latency_ms: 4,
|
|
413
|
+
},
|
|
414
|
+
],
|
|
415
|
+
},
|
|
416
|
+
],
|
|
417
|
+
startTime: new Date(),
|
|
418
|
+
endTime: new Date(),
|
|
419
|
+
});
|
|
420
|
+
|
|
421
|
+
expect(() => assertRunManifestIntegrity(manifest)).not.toThrow();
|
|
422
|
+
expect(() =>
|
|
423
|
+
assertRunManifestIntegrity({
|
|
424
|
+
...manifest,
|
|
425
|
+
cases: [{ ...manifest.cases[0], attempt_evidence: [] }],
|
|
426
|
+
})
|
|
427
|
+
).toThrow('malformed attempt evidence');
|
|
306
428
|
});
|
|
307
429
|
});
|
|
@@ -3,14 +3,15 @@
|
|
|
3
3
|
*/
|
|
4
4
|
|
|
5
5
|
import { nanoid } from 'nanoid';
|
|
6
|
-
import { estimateCost, getModelPricing } from '../cost/pricing';
|
|
7
6
|
import { getEnvironmentInfo } from '../provenance/environment';
|
|
8
7
|
import { getGitInfo } from '../provenance/git';
|
|
9
8
|
import type {
|
|
10
9
|
CaseResult,
|
|
11
|
-
|
|
10
|
+
CostProvenance,
|
|
11
|
+
ExecutionProvenance,
|
|
12
12
|
ManifestRedactionInfo,
|
|
13
13
|
ResolvedConfig,
|
|
14
|
+
RunAttemptEvidence,
|
|
14
15
|
RunConfig,
|
|
15
16
|
RunManifest,
|
|
16
17
|
RunMetrics,
|
|
@@ -26,6 +27,10 @@ export function createRunManifest(options: {
|
|
|
26
27
|
config: RunConfig;
|
|
27
28
|
resolvedConfig?: ResolvedConfig;
|
|
28
29
|
workloadIdentity?: WorkloadIdentity;
|
|
30
|
+
executionProvenance?: ExecutionProvenance;
|
|
31
|
+
attemptEvidence?: RunAttemptEvidence;
|
|
32
|
+
costProvenance?: CostProvenance;
|
|
33
|
+
runId?: string;
|
|
29
34
|
cases: CaseResult[];
|
|
30
35
|
startTime: Date;
|
|
31
36
|
endTime: Date;
|
|
@@ -38,6 +43,10 @@ export function createRunManifest(options: {
|
|
|
38
43
|
config,
|
|
39
44
|
resolvedConfig,
|
|
40
45
|
workloadIdentity,
|
|
46
|
+
executionProvenance,
|
|
47
|
+
attemptEvidence,
|
|
48
|
+
costProvenance,
|
|
49
|
+
runId,
|
|
41
50
|
cases,
|
|
42
51
|
startTime,
|
|
43
52
|
endTime,
|
|
@@ -46,15 +55,13 @@ export function createRunManifest(options: {
|
|
|
46
55
|
redaction,
|
|
47
56
|
} = options;
|
|
48
57
|
|
|
49
|
-
|
|
50
|
-
const modelForCost = resolvedConfig?.model || config.model;
|
|
51
|
-
const metrics = calculateMetrics(cases, modelForCost);
|
|
58
|
+
const metrics = calculateMetrics(cases, costProvenance);
|
|
52
59
|
const git = getGitInfo();
|
|
53
60
|
const environment = getEnvironmentInfo();
|
|
54
61
|
|
|
55
62
|
return {
|
|
56
|
-
version: '1.
|
|
57
|
-
run_id: nanoid(12),
|
|
63
|
+
version: '1.4',
|
|
64
|
+
run_id: runId ?? nanoid(12),
|
|
58
65
|
project,
|
|
59
66
|
start_time: startTime.toISOString(),
|
|
60
67
|
end_time: endTime.toISOString(),
|
|
@@ -62,6 +69,8 @@ export function createRunManifest(options: {
|
|
|
62
69
|
config,
|
|
63
70
|
resolved_config: resolvedConfig,
|
|
64
71
|
workload_identity: workloadIdentity,
|
|
72
|
+
execution_provenance: executionProvenance,
|
|
73
|
+
attempt_evidence: attemptEvidence,
|
|
65
74
|
metrics,
|
|
66
75
|
git,
|
|
67
76
|
provenance: {
|
|
@@ -78,7 +87,7 @@ export function createRunManifest(options: {
|
|
|
78
87
|
/**
|
|
79
88
|
* Calculate metrics from case results
|
|
80
89
|
*/
|
|
81
|
-
function calculateMetrics(cases: CaseResult[],
|
|
90
|
+
function calculateMetrics(cases: CaseResult[], costProvenance?: CostProvenance): RunMetrics {
|
|
82
91
|
const passedCases = cases.filter((c) => getCaseEvaluationStatus(c) === 'passed');
|
|
83
92
|
const validCases = cases.filter((c) => {
|
|
84
93
|
const status = getCaseEvaluationStatus(c);
|
|
@@ -94,26 +103,13 @@ function calculateMetrics(cases: CaseResult[], model?: string): RunMetrics {
|
|
|
94
103
|
const totalPromptTokens = cases.reduce((sum, c) => sum + c.tokens.prompt, 0);
|
|
95
104
|
const totalCompletionTokens = cases.reduce((sum, c) => sum + c.tokens.completion, 0);
|
|
96
105
|
|
|
97
|
-
//
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
const costEstimate = estimateCost(totalPromptTokens, totalCompletionTokens, model);
|
|
105
|
-
const pricing = getModelPricing(model);
|
|
106
|
-
cost = {
|
|
107
|
-
total_usd: costEstimate.totalUsd,
|
|
108
|
-
prompt_cost_usd: costEstimate.promptCostUsd,
|
|
109
|
-
completion_cost_usd: costEstimate.completionCostUsd,
|
|
110
|
-
model: costEstimate.model,
|
|
111
|
-
pricing: {
|
|
112
|
-
prompt_per_1k: pricing.promptPer1K,
|
|
113
|
-
completion_per_1k: pricing.completionPer1K,
|
|
114
|
-
},
|
|
115
|
-
};
|
|
116
|
-
}
|
|
106
|
+
// Token counts are not provider billing records. Never turn a generic price
|
|
107
|
+
// table into assurance cost evidence.
|
|
108
|
+
const cost_provenance: CostProvenance = costProvenance ?? {
|
|
109
|
+
schema_version: '1',
|
|
110
|
+
status: 'unavailable',
|
|
111
|
+
unavailable_reason: 'provider_billing_not_recorded',
|
|
112
|
+
};
|
|
117
113
|
|
|
118
114
|
return {
|
|
119
115
|
success_rate: validCases.length > 0 ? passedCases.length / validCases.length : 0,
|
|
@@ -129,7 +125,7 @@ function calculateMetrics(cases: CaseResult[], model?: string): RunMetrics {
|
|
|
129
125
|
total_tokens: totalPromptTokens + totalCompletionTokens,
|
|
130
126
|
total_prompt_tokens: totalPromptTokens,
|
|
131
127
|
total_completion_tokens: totalCompletionTokens,
|
|
132
|
-
|
|
128
|
+
cost_provenance,
|
|
133
129
|
};
|
|
134
130
|
}
|
|
135
131
|
|