@artemiskit/core 0.5.0 → 0.5.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,11 @@
1
+ /** Bounded execution provenance derived from declared and observed run evidence. */
2
+ import type { CaseResult, ExecutionProvenance } from '../artifacts/types';
3
+ export declare function createExecutionProvenance(options: {
4
+ provider: string;
5
+ requestedModel?: string;
6
+ temperature?: number;
7
+ maxTokens?: number;
8
+ seed?: number;
9
+ cases: CaseResult[];
10
+ }): ExecutionProvenance;
11
+ //# sourceMappingURL=execution-provenance.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"execution-provenance.d.ts","sourceRoot":"","sources":["../../src/provenance/execution-provenance.ts"],"names":[],"mappings":"AAAA,oFAAoF;AAEpF,OAAO,KAAK,EAAE,UAAU,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAE1E,wBAAgB,yBAAyB,CAAC,OAAO,EAAE;IACjD,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,UAAU,EAAE,CAAC;CACrB,GAAG,mBAAmB,CA2BtB"}
@@ -4,4 +4,5 @@
4
4
  export { getGitInfo } from './git';
5
5
  export { getEnvironmentInfo } from './environment';
6
6
  export { createWorkloadIdentity } from './workload-identity';
7
+ export { createExecutionProvenance } from './execution-provenance';
7
8
  //# sourceMappingURL=index.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/provenance/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,OAAO,CAAC;AACnC,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/provenance/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,OAAO,CAAC;AACnC,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAC7D,OAAO,EAAE,yBAAyB,EAAE,MAAM,wBAAwB,CAAC"}
@@ -1 +1 @@
1
- {"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../src/runner/executor.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAIV,UAAU,EACX,MAAM,oBAAoB,CAAC;AAI5B,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAQnD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAwE/C;;GAEG;AACH,wBAAsB,WAAW,CAC/B,QAAQ,EAAE,QAAQ,EAClB,OAAO,EAAE,eAAe,GACvB,OAAO,CAAC,UAAU,CAAC,CAuCrB"}
1
+ {"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../src/runner/executor.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAKV,UAAU,EACX,MAAM,oBAAoB,CAAC;AAI5B,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAQnD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAwE/C;;GAEG;AACH,wBAAsB,WAAW,CAC/B,QAAQ,EAAE,QAAQ,EAClB,OAAO,EAAE,eAAe,GACvB,OAAO,CAAC,UAAU,CAAC,CA6ErB"}
@@ -1 +1 @@
1
- {"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/runner/runner.ts"],"names":[],"mappings":"AAAA;;GAEG;AAOH,OAAO,KAAK,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAErD;;GAEG;AACH,wBAAsB,WAAW,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,CA4HzE;AAED;;GAEG;AACH,wBAAsB,YAAY,CAAC,WAAW,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CASlF"}
1
+ {"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/runner/runner.ts"],"names":[],"mappings":"AAAA;;GAEG;AAQH,OAAO,KAAK,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAErD;;GAEG;AACH,wBAAsB,WAAW,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,CAyJzE;AAED;;GAEG;AACH,wBAAsB,YAAY,CAAC,WAAW,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CASlF"}
@@ -2,7 +2,7 @@
2
2
  * Runner types and interfaces
3
3
  */
4
4
  import type { ModelClient } from '../adapters/types';
5
- import type { CaseResult, ResolvedConfig, RunManifest } from '../artifacts/types';
5
+ import type { CaseResult, CostProvenance, ResolvedConfig, RunManifest } from '../artifacts/types';
6
6
  import type { RedactionConfig } from '../redaction/types';
7
7
  import type { Scenario } from '../scenario/schema';
8
8
  import type { ToolExecutor } from '../tools';
@@ -26,6 +26,13 @@ export interface RunOptions {
26
26
  timeout?: number;
27
27
  /** Number of retries per case */
28
28
  retries?: number;
29
+ /** One-based coordinate for an independently planned repetition. */
30
+ repetition?: {
31
+ index: number;
32
+ total: number;
33
+ };
34
+ /** Attested or operator-supplied monetary evidence; omitted means unavailable. */
35
+ costProvenance?: CostProvenance;
29
36
  /** Redaction configuration (CLI overrides scenario) */
30
37
  redaction?: RedactionConfig;
31
38
  /** SDK-only executor for explicitly supplied real tools. */
@@ -52,8 +59,15 @@ export interface RunResult {
52
59
  export interface ExecutorContext {
53
60
  client: ModelClient;
54
61
  scenario: Scenario;
62
+ /** Effective model selected outside the scenario, such as a CLI override. */
63
+ requestedModel?: string;
55
64
  timeout?: number;
56
65
  retries?: number;
66
+ runId?: string;
67
+ repetition?: {
68
+ index: number;
69
+ total: number;
70
+ };
57
71
  /** Redaction configuration for this execution */
58
72
  redaction?: RedactionConfig;
59
73
  toolExecutor?: ToolExecutor;
@@ -1 +1 @@
1
- {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/runner/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAAE,UAAU,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAClF,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAE7C;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,0BAA0B;IAC1B,QAAQ,EAAE,QAAQ,CAAC;IACnB,0BAA0B;IAC1B,MAAM,EAAE,WAAW,CAAC;IACpB,oCAAoC;IACpC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,kDAAkD;IAClD,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,2BAA2B;IAC3B,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,oCAAoC;IACpC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iCAAiC;IACjC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,uDAAuD;IACvD,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,4DAA4D;IAC5D,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,oCAAoC;IACpC,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IAC5E,oCAAoC;IACpC,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;CACxC;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,6BAA6B;IAC7B,QAAQ,EAAE,WAAW,CAAC;IACtB,8BAA8B;IAC9B,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,+BAA+B;IAC/B,OAAO,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,WAAW,CAAC;IACpB,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,YAAY,CAAC,EAAE,YAAY,CAAC;CAC7B"}
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/runner/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAAE,UAAU,EAAE,cAAc,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAClG,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAE7C;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,0BAA0B;IAC1B,QAAQ,EAAE,QAAQ,CAAC;IACnB,0BAA0B;IAC1B,MAAM,EAAE,WAAW,CAAC;IACpB,oCAAoC;IACpC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,kDAAkD;IAClD,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,2BAA2B;IAC3B,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,oCAAoC;IACpC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iCAAiC;IACjC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,oEAAoE;IACpE,UAAU,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IAC9C,kFAAkF;IAClF,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,uDAAuD;IACvD,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,4DAA4D;IAC5D,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,oCAAoC;IACpC,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IAC5E,oCAAoC;IACpC,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;CACxC;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,6BAA6B;IAC7B,QAAQ,EAAE,WAAW,CAAC;IACtB,8BAA8B;IAC9B,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,+BAA+B;IAC/B,OAAO,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,WAAW,CAAC;IACpB,QAAQ,EAAE,QAAQ,CAAC;IACnB,6EAA6E;IAC7E,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IAC9C,iDAAiD;IACjD,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,YAAY,CAAC,EAAE,YAAY,CAAC;CAC7B"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@artemiskit/core",
3
- "version": "0.5.0",
3
+ "version": "0.5.2",
4
4
  "description": "Core runner, evaluators, and storage for ArtemisKit LLM evaluation toolkit",
5
5
  "type": "module",
6
6
  "license": "Apache-2.0",
@@ -55,13 +55,19 @@ describe('createRunManifest', () => {
55
55
  endTime,
56
56
  });
57
57
 
58
- expect(manifest.version).toBe('1.2');
58
+ expect(manifest.version).toBe('1.4');
59
59
  expect(manifest.project).toBe('test-project');
60
60
  expect(manifest.run_id).toBeTruthy();
61
61
  expect(manifest.run_id.length).toBe(12);
62
62
  expect(manifest.config.scenario).toBe('test-scenario');
63
63
  expect(manifest.config.provider).toBe('openai');
64
64
  expect(manifest.config.model).toBe('gpt-4');
65
+ expect(manifest.metrics.cost).toBeUndefined();
66
+ expect(manifest.metrics.cost_provenance).toEqual({
67
+ schema_version: '1',
68
+ status: 'unavailable',
69
+ unavailable_reason: 'provider_billing_not_recorded',
70
+ });
65
71
  });
66
72
 
67
73
  test('calculates metrics correctly', () => {
@@ -189,6 +195,30 @@ describe('createRunManifest', () => {
189
195
  expect(() => assertRunManifestIntegrity(manifest)).not.toThrow();
190
196
  });
191
197
 
198
+ test('retains target and evaluator execution provenance separately', () => {
199
+ const manifest = createRunManifest({
200
+ project: 'test-project',
201
+ config: { scenario: 'test-scenario', provider: 'openai' },
202
+ executionProvenance: {
203
+ schema_version: '1',
204
+ target: {
205
+ provider: 'openai',
206
+ requested_models: ['gpt-requested'],
207
+ observed_models: ['gpt-observed'],
208
+ generation: { temperature: 0, max_tokens: 100, seed: 42 },
209
+ },
210
+ evaluator: { models: ['judge-model'] },
211
+ },
212
+ cases: mockCases,
213
+ startTime: new Date(),
214
+ endTime: new Date(),
215
+ });
216
+
217
+ expect(manifest.execution_provenance?.target.observed_models).toEqual(['gpt-observed']);
218
+ expect(manifest.execution_provenance?.evaluator?.models).toEqual(['judge-model']);
219
+ expect(() => assertRunManifestIntegrity(manifest)).not.toThrow();
220
+ });
221
+
192
222
  test('includes provenance information', () => {
193
223
  const manifest = createRunManifest({
194
224
  project: 'test-project',
@@ -303,5 +333,97 @@ describe('createRunManifest', () => {
303
333
  expect(() =>
304
334
  assertRunManifestIntegrity({ ...historical, workload_identity: { schema_version: '1' } })
305
335
  ).toThrow('malformed workload identity');
336
+ expect(() =>
337
+ assertRunManifestIntegrity({
338
+ ...historical,
339
+ execution_provenance: { schema_version: '1', target: { provider: '' } },
340
+ })
341
+ ).toThrow('malformed execution provenance');
342
+ expect(() =>
343
+ assertRunManifestIntegrity({
344
+ ...historical,
345
+ cases: [{ ...historical.cases[0], target: { provider: '', observed_models: ['x'] } }],
346
+ })
347
+ ).toThrow('malformed target evidence');
348
+ });
349
+
350
+ test('retains attested cost evidence but rejects malformed monetary claims', () => {
351
+ const manifest = createRunManifest({
352
+ project: 'test-project',
353
+ config: { scenario: 'cost', provider: 'openai' },
354
+ costProvenance: {
355
+ schema_version: '1',
356
+ status: 'known',
357
+ amount: 1.25,
358
+ currency: 'USD',
359
+ source: 'provider_billing',
360
+ recorded_at: '2026-09-10T00:00:00.000Z',
361
+ },
362
+ cases: mockCases,
363
+ startTime: new Date(),
364
+ endTime: new Date(),
365
+ });
366
+
367
+ expect(manifest.metrics.cost_provenance?.status).toBe('known');
368
+ expect(() => assertRunManifestIntegrity(manifest)).not.toThrow();
369
+ expect(() =>
370
+ assertRunManifestIntegrity({
371
+ ...manifest,
372
+ metrics: {
373
+ ...manifest.metrics,
374
+ cost_provenance: { schema_version: '1', status: 'known', amount: 1 },
375
+ },
376
+ })
377
+ ).toThrow('malformed cost provenance');
378
+ });
379
+
380
+ test('validates bounded retry-chain evidence without requiring it from historical manifests', () => {
381
+ const manifest = createRunManifest({
382
+ project: 'test-project',
383
+ config: { scenario: 'retry', provider: 'fixture' },
384
+ attemptEvidence: {
385
+ schema_version: '1',
386
+ repetition: { index: 1, total: 2 },
387
+ retry_policy: { default_max_retries: 1, backoff: 'exponential', initial_delay_ms: 1000 },
388
+ timeout: { default_ms: 5000 },
389
+ },
390
+ cases: [
391
+ {
392
+ ...mockCases[0],
393
+ attempts: 2,
394
+ attempt_evidence: [
395
+ {
396
+ attempt_id: 'run:case-1:1',
397
+ retry_chain_id: 'run:case-1',
398
+ repetition_index: 1,
399
+ attempt_number: 1,
400
+ status: 'error',
401
+ included_in_outcome: false,
402
+ latency_ms: 3,
403
+ error_code: 'target_error',
404
+ },
405
+ {
406
+ attempt_id: 'run:case-1:2',
407
+ retry_chain_id: 'run:case-1',
408
+ repetition_index: 1,
409
+ attempt_number: 2,
410
+ status: 'passed',
411
+ included_in_outcome: true,
412
+ latency_ms: 4,
413
+ },
414
+ ],
415
+ },
416
+ ],
417
+ startTime: new Date(),
418
+ endTime: new Date(),
419
+ });
420
+
421
+ expect(() => assertRunManifestIntegrity(manifest)).not.toThrow();
422
+ expect(() =>
423
+ assertRunManifestIntegrity({
424
+ ...manifest,
425
+ cases: [{ ...manifest.cases[0], attempt_evidence: [] }],
426
+ })
427
+ ).toThrow('malformed attempt evidence');
306
428
  });
307
429
  });
@@ -3,14 +3,15 @@
3
3
  */
4
4
 
5
5
  import { nanoid } from 'nanoid';
6
- import { estimateCost, getModelPricing } from '../cost/pricing';
7
6
  import { getEnvironmentInfo } from '../provenance/environment';
8
7
  import { getGitInfo } from '../provenance/git';
9
8
  import type {
10
9
  CaseResult,
11
- CostEstimateInfo,
10
+ CostProvenance,
11
+ ExecutionProvenance,
12
12
  ManifestRedactionInfo,
13
13
  ResolvedConfig,
14
+ RunAttemptEvidence,
14
15
  RunConfig,
15
16
  RunManifest,
16
17
  RunMetrics,
@@ -26,6 +27,10 @@ export function createRunManifest(options: {
26
27
  config: RunConfig;
27
28
  resolvedConfig?: ResolvedConfig;
28
29
  workloadIdentity?: WorkloadIdentity;
30
+ executionProvenance?: ExecutionProvenance;
31
+ attemptEvidence?: RunAttemptEvidence;
32
+ costProvenance?: CostProvenance;
33
+ runId?: string;
29
34
  cases: CaseResult[];
30
35
  startTime: Date;
31
36
  endTime: Date;
@@ -38,6 +43,10 @@ export function createRunManifest(options: {
38
43
  config,
39
44
  resolvedConfig,
40
45
  workloadIdentity,
46
+ executionProvenance,
47
+ attemptEvidence,
48
+ costProvenance,
49
+ runId,
41
50
  cases,
42
51
  startTime,
43
52
  endTime,
@@ -46,15 +55,13 @@ export function createRunManifest(options: {
46
55
  redaction,
47
56
  } = options;
48
57
 
49
- // Get model for cost calculation - prefer resolvedConfig, then config
50
- const modelForCost = resolvedConfig?.model || config.model;
51
- const metrics = calculateMetrics(cases, modelForCost);
58
+ const metrics = calculateMetrics(cases, costProvenance);
52
59
  const git = getGitInfo();
53
60
  const environment = getEnvironmentInfo();
54
61
 
55
62
  return {
56
- version: '1.2',
57
- run_id: nanoid(12),
63
+ version: '1.4',
64
+ run_id: runId ?? nanoid(12),
58
65
  project,
59
66
  start_time: startTime.toISOString(),
60
67
  end_time: endTime.toISOString(),
@@ -62,6 +69,8 @@ export function createRunManifest(options: {
62
69
  config,
63
70
  resolved_config: resolvedConfig,
64
71
  workload_identity: workloadIdentity,
72
+ execution_provenance: executionProvenance,
73
+ attempt_evidence: attemptEvidence,
65
74
  metrics,
66
75
  git,
67
76
  provenance: {
@@ -78,7 +87,7 @@ export function createRunManifest(options: {
78
87
  /**
79
88
  * Calculate metrics from case results
80
89
  */
81
- function calculateMetrics(cases: CaseResult[], model?: string): RunMetrics {
90
+ function calculateMetrics(cases: CaseResult[], costProvenance?: CostProvenance): RunMetrics {
82
91
  const passedCases = cases.filter((c) => getCaseEvaluationStatus(c) === 'passed');
83
92
  const validCases = cases.filter((c) => {
84
93
  const status = getCaseEvaluationStatus(c);
@@ -94,26 +103,13 @@ function calculateMetrics(cases: CaseResult[], model?: string): RunMetrics {
94
103
  const totalPromptTokens = cases.reduce((sum, c) => sum + c.tokens.prompt, 0);
95
104
  const totalCompletionTokens = cases.reduce((sum, c) => sum + c.tokens.completion, 0);
96
105
 
97
- // Calculate cost if model is provided
98
- let cost: CostEstimateInfo | undefined;
99
- if (
100
- model &&
101
- !model.toLowerCase().includes('ling-') &&
102
- (totalPromptTokens > 0 || totalCompletionTokens > 0)
103
- ) {
104
- const costEstimate = estimateCost(totalPromptTokens, totalCompletionTokens, model);
105
- const pricing = getModelPricing(model);
106
- cost = {
107
- total_usd: costEstimate.totalUsd,
108
- prompt_cost_usd: costEstimate.promptCostUsd,
109
- completion_cost_usd: costEstimate.completionCostUsd,
110
- model: costEstimate.model,
111
- pricing: {
112
- prompt_per_1k: pricing.promptPer1K,
113
- completion_per_1k: pricing.completionPer1K,
114
- },
115
- };
116
- }
106
+ // Token counts are not provider billing records. Never turn a generic price
107
+ // table into assurance cost evidence.
108
+ const cost_provenance: CostProvenance = costProvenance ?? {
109
+ schema_version: '1',
110
+ status: 'unavailable',
111
+ unavailable_reason: 'provider_billing_not_recorded',
112
+ };
117
113
 
118
114
  return {
119
115
  success_rate: validCases.length > 0 ? passedCases.length / validCases.length : 0,
@@ -129,7 +125,7 @@ function calculateMetrics(cases: CaseResult[], model?: string): RunMetrics {
129
125
  total_tokens: totalPromptTokens + totalCompletionTokens,
130
126
  total_prompt_tokens: totalPromptTokens,
131
127
  total_completion_tokens: totalCompletionTokens,
132
- cost,
128
+ cost_provenance,
133
129
  };
134
130
  }
135
131