@artemiskit/core 0.5.0 → 0.5.2

package/src/artifacts/types.ts

@@ -65,6 +65,33 @@ export interface WorkloadIdentity {
   rubric: ContentIdentity;
 }
 
+/** Bounded target identity captured from a case execution. */
+export interface CaseTargetEvidence {
+  provider: string;
+  requested_model?: string;
+  /** Model identifiers returned by the target provider during this case. */
+  observed_models?: string[];
+}
+
+/** Requested and observed execution configuration for a complete run. */
+export interface ExecutionProvenance {
+  schema_version: '1';
+  target: {
+    provider: string;
+    requested_models?: string[];
+    observed_models?: string[];
+    generation?: {
+      temperature?: number;
+      max_tokens?: number;
+      seed?: number;
+    };
+  };
+  /** Judge/evaluator model identities, never combined with target identity. */
+  evaluator?: {
+    models?: string[];
+  };
+}
+
 // ============================================================================
 // Case Result Types
 // ============================================================================
@@ -97,6 +124,39 @@ export interface CaseEvaluationEvidence {
   };
 }
 
+/** A bounded record of one execution in a retry chain. */
+export interface CaseAttemptEvidence {
+  attempt_id: string;
+  retry_chain_id: string;
+  /** One-based coordinate within a deliberately independent repetition. */
+  repetition_index: number;
+  /** One-based coordinate within this retry chain. */
+  attempt_number: number;
+  status: CaseEvaluationStatus;
+  /** Only the terminal measurement can contribute to an outcome rate. */
+  included_in_outcome: boolean;
+  latency_ms: number;
+  /** Sanitized classification, never arbitrary provider error text. */
+  error_code?: 'timeout' | 'target_error' | 'tool_error';
+}
+
+/** Declared retry and repetition context for a run. */
+export interface RunAttemptEvidence {
+  schema_version: '1';
+  repetition: {
+    index: number;
+    total: number;
+  };
+  retry_policy: {
+    default_max_retries: number;
+    backoff: 'exponential';
+    initial_delay_ms: number;
+  };
+  timeout?: {
+    default_ms: number;
+  };
+}
+
 /**
  * Individual test case result
  */
@@ -108,6 +168,8 @@ export interface CaseResult {
   status?: CaseEvaluationStatus;
   /** Number of execution attempts represented by this terminal result. */
   attempts?: number;
+  /** Bounded retry-chain evidence for this terminal case result. */
+  attempt_evidence?: CaseAttemptEvidence[];
   score: number;
   matcherType: string;
   reason?: string;
@@ -124,6 +186,8 @@ export interface CaseResult {
   error?: string;
   /** Sanitized evaluator evidence; arbitrary evaluator details are never stored here. */
   evidence?: CaseEvaluationEvidence;
+  /** Requested and observed target identity for this case. */
+  target?: CaseTargetEvidence;
   /** Redaction information for this case */
   redaction?: CaseRedactionInfo;
   /** Ordered tool activity captured for an enabled tool loop. */
@@ -151,6 +215,28 @@ export interface CostEstimateInfo {
   };
 }
 
+/** Whether a monetary value is attested, supplied by an operator, or unavailable. */
+export type CostProvenanceStatus = 'known' | 'user_supplied' | 'unavailable';
+
+/**
+ * Cost evidence suitable for assurance reporting. Generic token-price estimates
+ * are intentionally not cost evidence.
+ */
+export interface CostProvenance {
+  schema_version: '1';
+  status: CostProvenanceStatus;
+  /** Required for known and user-supplied amounts. */
+  amount?: number;
+  /** ISO 4217 currency required with an amount. */
+  currency?: string;
+  /** Origin of a recorded monetary value. */
+  source?: 'provider_billing' | 'operator_input';
+  /** ISO timestamp for a recorded monetary value. */
+  recorded_at?: string;
+  /** Stable reason code when no attested amount is available. */
+  unavailable_reason?: 'provider_billing_not_recorded' | 'unsupported_provider' | 'not_requested';
+}
+
 /**
  * Run metrics
  */
@@ -173,7 +259,10 @@ export interface RunMetrics {
   total_prompt_tokens: number;
   total_completion_tokens: number;
   /** Estimated cost information */
+  /** @deprecated Generic pricing estimates are not assurance cost evidence. */
   cost?: CostEstimateInfo;
+  /** Explicit monetary evidence for assurance reporting. */
+  cost_provenance?: CostProvenance;
 }
 
 /**
@@ -302,6 +391,10 @@ export interface RunManifest {
   resolved_config?: ResolvedConfig;
   /** Versioned identities for the declared workload and evaluation rubric. */
   workload_identity?: WorkloadIdentity;
+  /** Requested and observed target/evaluator configuration for this run. */
+  execution_provenance?: ExecutionProvenance;
+  /** Retry-chain and repetition context. Present in manifest v1.4+. */
+  attempt_evidence?: RunAttemptEvidence;
   metrics: RunMetrics;
   git: GitInfo;
   provenance: ProvenanceInfo;
@@ -351,6 +444,15 @@ export function assertRunManifestIntegrity(manifest: unknown): asserts manifest
   if (manifest.workload_identity !== undefined) {
     assertWorkloadIdentity(manifest.workload_identity);
   }
+  if (manifest.execution_provenance !== undefined) {
+    assertExecutionProvenance(manifest.execution_provenance);
+  }
+  if (manifest.attempt_evidence !== undefined) {
+    assertRunAttemptEvidence(manifest.attempt_evidence);
+  }
+  if (isRecord(manifest.metrics) && manifest.metrics.cost_provenance !== undefined) {
+    assertCostProvenance(manifest.metrics.cost_provenance);
+  }
 
   for (const [index, caseResult] of manifest.cases.entries()) {
     if (!isRecord(caseResult)) {
@@ -370,9 +472,169 @@ export function assertRunManifestIntegrity(manifest: unknown): asserts manifest
     if (caseResult.evidence !== undefined) {
       assertCaseEvaluationEvidence(caseResult.evidence, index);
     }
+    if (caseResult.target !== undefined) {
+      assertCaseTargetEvidence(caseResult.target);
+    }
+    if (caseResult.attempt_evidence !== undefined) {
+      assertCaseAttemptEvidence(caseResult.attempt_evidence, index);
+    }
   }
 }
 
+function assertRunAttemptEvidence(evidence: unknown): void {
+  if (
+    !isRecord(evidence) ||
+    evidence.schema_version !== '1' ||
+    !isRecord(evidence.repetition) ||
+    !isPositiveSafeInteger(evidence.repetition.index) ||
+    !isPositiveSafeInteger(evidence.repetition.total) ||
+    evidence.repetition.index > evidence.repetition.total ||
+    !isRecord(evidence.retry_policy) ||
+    !isNonnegativeSafeInteger(evidence.retry_policy.default_max_retries) ||
+    evidence.retry_policy.backoff !== 'exponential' ||
+    !isNonnegativeFiniteNumber(evidence.retry_policy.initial_delay_ms) ||
+    (evidence.timeout !== undefined &&
+      (!isRecord(evidence.timeout) || !isPositiveFiniteNumber(evidence.timeout.default_ms)))
+  ) {
+    throw new Error('Invalid run manifest: malformed attempt evidence');
+  }
+}
+
+function assertCaseAttemptEvidence(evidence: unknown, index: number): void {
+  if (!Array.isArray(evidence) || evidence.length === 0 || evidence.length > 100) {
+    throw new Error(`Invalid run manifest: case ${index} has malformed attempt evidence`);
+  }
+  for (const attempt of evidence) {
+    if (
+      !isRecord(attempt) ||
+      !isBoundedNonemptyString(attempt.attempt_id, 200) ||
+      !isBoundedNonemptyString(attempt.retry_chain_id, 200) ||
+      !isPositiveSafeInteger(attempt.repetition_index) ||
+      !isPositiveSafeInteger(attempt.attempt_number) ||
+      !isCaseEvaluationStatus(attempt.status) ||
+      typeof attempt.included_in_outcome !== 'boolean' ||
+      !isNonnegativeFiniteNumber(attempt.latency_ms) ||
+      (attempt.error_code !== undefined &&
+        attempt.error_code !== 'timeout' &&
+        attempt.error_code !== 'target_error' &&
+        attempt.error_code !== 'tool_error')
+    ) {
+      throw new Error(`Invalid run manifest: case ${index} has malformed attempt evidence`);
+    }
+  }
+}
+
+function assertCostProvenance(cost: unknown): void {
+  if (!isRecord(cost) || cost.schema_version !== '1') {
+    throw new Error('Invalid run manifest: malformed cost provenance');
+  }
+  if (cost.status === 'unavailable') {
+    if (
+      cost.amount !== undefined ||
+      cost.currency !== undefined ||
+      cost.source !== undefined ||
+      cost.recorded_at !== undefined ||
+      (cost.unavailable_reason !== 'provider_billing_not_recorded' &&
+        cost.unavailable_reason !== 'unsupported_provider' &&
+        cost.unavailable_reason !== 'not_requested')
+    ) {
+      throw new Error('Invalid run manifest: malformed cost provenance');
+    }
+    return;
+  }
+  if (
+    (cost.status !== 'known' && cost.status !== 'user_supplied') ||
+    !isNonnegativeFiniteNumber(cost.amount) ||
+    !isBoundedNonemptyString(cost.currency, 3) ||
+    (cost.status === 'known' && cost.source !== 'provider_billing') ||
+    (cost.status === 'user_supplied' && cost.source !== 'operator_input') ||
+    !isIsoTimestamp(cost.recorded_at) ||
+    cost.unavailable_reason !== undefined
+  ) {
+    throw new Error('Invalid run manifest: malformed cost provenance');
+  }
+}
+
+function isCaseEvaluationStatus(value: unknown): value is CaseEvaluationStatus {
+  return value === 'passed' || value === 'failed' || value === 'invalid' || value === 'error';
+}
+
+function isBoundedNonemptyString(value: unknown, maxLength: number): value is string {
+  return typeof value === 'string' && value.length > 0 && value.length <= maxLength;
+}
+
+function isNonnegativeFiniteNumber(value: unknown): value is number {
+  return typeof value === 'number' && Number.isFinite(value) && value >= 0;
+}
+
+function isPositiveFiniteNumber(value: unknown): value is number {
+  return isNonnegativeFiniteNumber(value) && value > 0;
+}
+
+function isNonnegativeSafeInteger(value: unknown): value is number {
+  return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0;
+}
+
+function isPositiveSafeInteger(value: unknown): value is number {
+  return isNonnegativeSafeInteger(value) && value > 0;
+}
+
+function isIsoTimestamp(value: unknown): value is string {
+  return typeof value === 'string' && Number.isFinite(Date.parse(value));
+}
+
+function assertCaseTargetEvidence(target: unknown): void {
+  if (
+    !isRecord(target) ||
+    typeof target.provider !== 'string' ||
+    target.provider.length === 0 ||
+    target.provider.length > 100 ||
+    (target.requested_model !== undefined &&
+      (typeof target.requested_model !== 'string' || target.requested_model.length > 200)) ||
+    !isBoundedStringList(target.observed_models)
+  ) {
+    throw new Error('Invalid run manifest: malformed target evidence');
+  }
+}
+
+function assertExecutionProvenance(provenance: unknown): void {
+  if (!isRecord(provenance) || provenance.schema_version !== '1' || !isRecord(provenance.target)) {
+    throw new Error('Invalid run manifest: malformed execution provenance');
+  }
+  const target = provenance.target;
+  if (
+    typeof target.provider !== 'string' ||
+    target.provider.length === 0 ||
+    target.provider.length > 100 ||
+    !isBoundedStringList(target.requested_models) ||
+    !isBoundedStringList(target.observed_models) ||
+    (target.generation !== undefined && !isGenerationConfig(target.generation))
+  ) {
+    throw new Error('Invalid run manifest: malformed execution provenance');
+  }
+  if (provenance.evaluator !== undefined) {
+    if (!isRecord(provenance.evaluator) || !isBoundedStringList(provenance.evaluator.models)) {
+      throw new Error('Invalid run manifest: malformed execution provenance');
+    }
+  }
+}
+
+function isBoundedStringList(value: unknown): boolean {
+  return (
+    value === undefined ||
+    (Array.isArray(value) &&
+      value.length <= 100 &&
+      value.every((item) => typeof item === 'string' && item.length > 0 && item.length <= 200))
+  );
+}
+
+function isGenerationConfig(value: unknown): boolean {
+  if (!isRecord(value)) return false;
+  return [value.temperature, value.max_tokens, value.seed].every(
+    (item) => item === undefined || (typeof item === 'number' && Number.isFinite(item))
+  );
+}
+
 function assertWorkloadIdentity(identity: unknown): void {
   if (
     !isRecord(identity) ||

package/src/provenance/execution-provenance.test.ts

@@ -0,0 +1,59 @@
+import { describe, expect, test } from 'bun:test';
+import { createExecutionProvenance } from './execution-provenance';
+
+describe('createExecutionProvenance', () => {
+  test('keeps requested, observed target, and evaluator model identities separate', () => {
+    const provenance = createExecutionProvenance({
+      provider: 'openai',
+      requestedModel: 'gpt-requested',
+      temperature: 0.2,
+      maxTokens: 100,
+      seed: 7,
+      cases: [
+        {
+          id: 'one',
+          ok: true,
+          score: 1,
+          matcherType: 'exact',
+          latencyMs: 1,
+          tokens: { prompt: 1, completion: 1, total: 2 },
+          prompt: 'prompt',
+          response: 'response',
+          expected: { type: 'exact', value: 'response', caseSensitive: true },
+          tags: [],
+          target: {
+            provider: 'openai',
+            requested_model: 'gpt-requested',
+            observed_models: ['gpt-observed'],
+          },
+          evidence: { evaluator: 'llm_grader', model: 'judge-model' },
+        },
+      ],
+    });
+
+    expect(provenance).toEqual({
+      schema_version: '1',
+      target: {
+        provider: 'openai',
+        requested_models: ['gpt-requested'],
+        observed_models: ['gpt-observed'],
+        generation: { temperature: 0.2, max_tokens: 100, seed: 7 },
+      },
+      evaluator: { models: ['judge-model'] },
+    });
+  });
+
+  test('records an unavailable observed identity without inventing one', () => {
+    const provenance = createExecutionProvenance({
+      provider: 'custom',
+      requestedModel: 'requested-model',
+      cases: [],
+    });
+
+    expect(provenance.target).toEqual({
+      provider: 'custom',
+      requested_models: ['requested-model'],
+    });
+    expect(provenance.evaluator).toBeUndefined();
+  });
+});

package/src/provenance/execution-provenance.ts

@@ -0,0 +1,53 @@
+/** Bounded execution provenance derived from declared and observed run evidence. */
+
+import type { CaseResult, ExecutionProvenance } from '../artifacts/types';
+
+export function createExecutionProvenance(options: {
+  provider: string;
+  requestedModel?: string;
+  temperature?: number;
+  maxTokens?: number;
+  seed?: number;
+  cases: CaseResult[];
+}): ExecutionProvenance {
+  const requestedModels = uniqueStrings([
+    options.requestedModel,
+    ...options.cases.map((caseResult) => caseResult.target?.requested_model),
+  ]);
+  const observedModels = uniqueStrings(
+    options.cases.flatMap((caseResult) => caseResult.target?.observed_models ?? [])
+  );
+  const evaluatorModels = uniqueStrings(
+    options.cases.map((caseResult) => caseResult.evidence?.model)
+  );
+  const generation = omitUndefined({
+    temperature: options.temperature,
+    max_tokens: options.maxTokens,
+    seed: options.seed,
+  });
+
+  return {
+    schema_version: '1',
+    target: {
+      provider: boundedString(options.provider, 100) ?? 'unknown',
+      ...(requestedModels.length ? { requested_models: requestedModels } : {}),
+      ...(observedModels.length ? { observed_models: observedModels } : {}),
+      ...(Object.keys(generation).length ? { generation } : {}),
+    },
+    ...(evaluatorModels.length ? { evaluator: { models: evaluatorModels } } : {}),
+  };
+}
+
+function uniqueStrings(values: unknown[]): string[] {
+  return [...new Set(values.map((value) => boundedString(value, 200)).filter(Boolean))] as string[];
+}
+
+function boundedString(value: unknown, maxLength: number): string | undefined {
+  return typeof value === 'string' && value.length > 0 ? value.slice(0, maxLength) : undefined;
+}
+
+function omitUndefined<T extends Record<string, number | undefined>>(value: T): Partial<T> {
+  return Object.fromEntries(
+    Object.entries(value).filter(([, item]) => item !== undefined)
+  ) as Partial<T>;
+}

package/src/provenance/index.ts

@@ -5,3 +5,4 @@
 export { getGitInfo } from './git';
 export { getEnvironmentInfo } from './environment';
 export { createWorkloadIdentity } from './workload-identity';
+export { createExecutionProvenance } from './execution-provenance';

package/src/runner/executor.test.ts

@@ -170,6 +170,11 @@ describe('executeCase tool loop', () => {
     expect(result.error).toBe('TOOL_EXECUTOR_REQUIRED');
     expect(result.latencyMs).toBe(4);
     expect(result.tokens).toEqual({ prompt: 7, completion: 2, total: 9 });
+    expect(result.target).toEqual({
+      provider: 'ling',
+      requested_model: 'Ling-3.0-flash',
+      observed_models: ['Ling-3.0-flash'],
+    });
   });
 
   it('retains prior generation metrics when a later generation rejects', async () => {
@@ -201,6 +206,7 @@ describe('executeCase tool loop', () => {
       steps: 1,
       terminationReason: 'tool_error',
     });
+    expect(result.target?.observed_models).toEqual(['Ling-3.0-flash']);
   });
 
   it('retains prior generation metrics when a later generation times out', async () => {
@@ -304,7 +310,61 @@ describe('executeCase measurement integrity', () => {
       status: 'error',
       response: '',
       error: 'provider unavailable',
+      target: {
+        provider: 'test',
+      },
     });
+    expect(result.target?.observed_models).toBeUndefined();
+  });
+
+  it('records retried target failures as excluded retry-chain attempts', async () => {
+    let calls = 0;
+    const retryClient: ModelClient = {
+      ...client,
+      generate: async () => {
+        calls++;
+        if (calls === 1) throw new Error('temporary provider failure');
+        return {
+          id: 'response',
+          model: 'target-model',
+          text: 'target response',
+          tokens: { prompt: 1, completion: 1, total: 2 },
+          latencyMs: 1,
+          finishReason: 'stop',
+        };
+      },
+    };
+    registerEvaluator('custom', {
+      type: 'custom',
+      evaluate: async () => ({ passed: true, score: 1 }),
+    });
+
+    const result = await executeCase(scenario.cases[0], {
+      client: retryClient,
+      scenario,
+      retries: 1,
+      runId: 'assurance-run',
+      repetition: { index: 2, total: 3 },
+    });
+
+    expect(result.attempts).toBe(2);
+    expect(result.attempt_evidence).toEqual([
+      expect.objectContaining({
+        attempt_id: 'assurance-run:custom-evaluation:1',
+        retry_chain_id: 'assurance-run:custom-evaluation',
+        repetition_index: 2,
+        attempt_number: 1,
+        status: 'error',
+        included_in_outcome: false,
+        error_code: 'target_error',
+      }),
+      expect.objectContaining({
+        attempt_id: 'assurance-run:custom-evaluation:2',
+        attempt_number: 2,
+        status: 'passed',
+        included_in_outcome: true,
+      }),
+    ]);
   });
 
   it('retains only the bounded evidence contract rather than evaluator details', async () => {