@artemiskit/core 0.5.1 → 0.5.3

package/src/comparison/eligibility.test.ts

@@ -0,0 +1,101 @@
+import { describe, expect, test } from 'bun:test';
+import type { RunManifest } from '../artifacts';
+import { assessComparisonEligibility, isComparisonAvailable } from './eligibility';
+
+function manifest(overrides: Partial<RunManifest> = {}): RunManifest {
+  return {
+    version: '1.3',
+    run_id: 'run-id',
+    project: 'project',
+    start_time: '2026-09-10T00:00:00.000Z',
+    end_time: '2026-09-10T00:00:01.000Z',
+    duration_ms: 1000,
+    config: { scenario: 'customer-service', provider: 'openai', model: 'model-a' },
+    workload_identity: {
+      schema_version: '1',
+      workload: { schema_version: '1', algorithm: 'sha256', digest: 'a'.repeat(64) },
+      rubric: { schema_version: '1', algorithm: 'sha256', digest: 'b'.repeat(64) },
+    },
+    execution_provenance: {
+      schema_version: '1',
+      target: { provider: 'openai', requested_models: ['model-a'], generation: { temperature: 0 } },
+    },
+    metrics: {
+      success_rate: 1,
+      total_cases: 1,
+      passed_cases: 1,
+      failed_cases: 0,
+      median_latency_ms: 1,
+      p95_latency_ms: 1,
+      total_tokens: 1,
+      total_prompt_tokens: 1,
+      total_completion_tokens: 0,
+    },
+    git: { commit: 'commit', branch: 'main', dirty: false },
+    provenance: { run_by: 'test' },
+    cases: [],
+    environment: { node_version: 'test', platform: 'test', arch: 'test' },
+    ...overrides,
+  };
+}
+
+describe('assessComparisonEligibility', () => {
+  test('accepts matching declared workload, rubric, and execution configuration', () => {
+    const eligibility = assessComparisonEligibility(manifest(), manifest({ run_id: 'current' }));
+
+    expect(eligibility).toEqual({ schema_version: '1', status: 'compatible', reasons: [] });
+    expect(isComparisonAvailable(eligibility)).toBe(true);
+  });
+
+  test('qualifies legacy artifacts instead of presenting them as fully compatible', () => {
+    const legacy = manifest({ workload_identity: undefined, execution_provenance: undefined });
+    const eligibility = assessComparisonEligibility(legacy, manifest({ run_id: 'current' }));
+
+    expect(eligibility.status).toBe('qualified');
+    expect(eligibility.reasons.map((reason) => reason.code)).toEqual([
+      'workload_identity_missing',
+      'rubric_identity_missing',
+      'execution_provenance_missing',
+    ]);
+    expect(isComparisonAvailable(eligibility)).toBe(true);
+  });
+
+  test('qualifies a deliberate target-model change without treating it as the same execution', () => {
+    const current = manifest({
+      run_id: 'current',
+      execution_provenance: {
+        schema_version: '1',
+        target: {
+          provider: 'openai',
+          requested_models: ['model-b'],
+          generation: { temperature: 0 },
+        },
+      },
+    });
+
+    const eligibility = assessComparisonEligibility(manifest(), current);
+
+    expect(eligibility.status).toBe('qualified');
+    expect(eligibility.reasons).toEqual([{ code: 'target_model_changed' }]);
+  });
+
+  test('refuses deltas when workload or rubric evidence differs', () => {
+    const current = manifest({
+      run_id: 'current',
+      workload_identity: {
+        schema_version: '1',
+        workload: { schema_version: '1', algorithm: 'sha256', digest: 'c'.repeat(64) },
+        rubric: { schema_version: '1', algorithm: 'sha256', digest: 'd'.repeat(64) },
+      },
+    });
+
+    const eligibility = assessComparisonEligibility(manifest(), current);
+
+    expect(eligibility.status).toBe('incomparable');
+    expect(eligibility.reasons.map((reason) => reason.code)).toEqual([
+      'workload_mismatch',
+      'rubric_mismatch',
+    ]);
+    expect(isComparisonAvailable(eligibility)).toBe(false);
+  });
+});

package/src/comparison/eligibility.ts

@@ -0,0 +1,116 @@
+/**
+ * Compatibility decisions for comparisons of saved scenario-evaluation runs.
+ *
+ * This contract compares declared evidence only. A matching digest proves the
+ * same canonical workload/rubric was declared; it does not attest to provider
+ * behaviour or replace a future assessment-profile compatibility decision.
+ */
+
+import type { RunManifest } from '../artifacts/types';
+
+export type ComparisonEligibilityStatus = 'compatible' | 'qualified' | 'incomparable';
+
+export type ComparisonEligibilityReasonCode =
+  | 'scenario_mismatch'
+  | 'workload_identity_missing'
+  | 'rubric_identity_missing'
+  | 'workload_mismatch'
+  | 'rubric_mismatch'
+  | 'execution_provenance_missing'
+  | 'target_provider_changed'
+  | 'target_model_changed'
+  | 'generation_settings_changed';
+
+/** A bounded, machine-readable reason for a comparison decision. */
+export interface ComparisonEligibilityReason {
+  code: ComparisonEligibilityReasonCode;
+}
+
+/**
+ * Versioned decision that accompanies every comparison. `qualified` permits
+ * a visibly-qualified delta; `incomparable` prohibits a delta entirely.
+ */
+export interface ComparisonEligibility {
+  schema_version: '1';
+  status: ComparisonEligibilityStatus;
+  reasons: ComparisonEligibilityReason[];
+}
+
+export function assessComparisonEligibility(
+  baseline: RunManifest,
+  current: RunManifest
+): ComparisonEligibility {
+  const reasons: ComparisonEligibilityReason[] = [];
+
+  if (baseline.config.scenario !== current.config.scenario) {
+    reasons.push({ code: 'scenario_mismatch' });
+  }
+
+  const baselineIdentity = baseline.workload_identity;
+  const currentIdentity = current.workload_identity;
+  if (!baselineIdentity || !currentIdentity) {
+    if (!baselineIdentity || !currentIdentity) reasons.push({ code: 'workload_identity_missing' });
+    if (!baselineIdentity || !currentIdentity) reasons.push({ code: 'rubric_identity_missing' });
+  } else {
+    if (baselineIdentity.workload.digest !== currentIdentity.workload.digest) {
+      reasons.push({ code: 'workload_mismatch' });
+    }
+    if (baselineIdentity.rubric.digest !== currentIdentity.rubric.digest) {
+      reasons.push({ code: 'rubric_mismatch' });
+    }
+  }
+
+  if (reasons.some((reason) => isIncomparableReason(reason.code))) {
+    return { schema_version: '1', status: 'incomparable', reasons };
+  }
+
+  const baselineExecution = baseline.execution_provenance;
+  const currentExecution = current.execution_provenance;
+  if (!baselineExecution || !currentExecution) {
+    reasons.push({ code: 'execution_provenance_missing' });
+  } else {
+    if (baselineExecution.target.provider !== currentExecution.target.provider) {
+      reasons.push({ code: 'target_provider_changed' });
+    }
+    if (
+      !sameStringSet(
+        baselineExecution.target.requested_models,
+        currentExecution.target.requested_models
+      )
+    ) {
+      reasons.push({ code: 'target_model_changed' });
+    }
+    if (!sameGeneration(baselineExecution.target.generation, currentExecution.target.generation)) {
+      reasons.push({ code: 'generation_settings_changed' });
+    }
+  }
+
+  return {
+    schema_version: '1',
+    status: reasons.length === 0 ? 'compatible' : 'qualified',
+    reasons,
+  };
+}
+
+export function isComparisonAvailable(eligibility: ComparisonEligibility): boolean {
+  return eligibility.status !== 'incomparable';
+}
+
+function isIncomparableReason(code: ComparisonEligibilityReasonCode): boolean {
+  return code === 'scenario_mismatch' || code === 'workload_mismatch' || code === 'rubric_mismatch';
+}
+
+function sameStringSet(left?: string[], right?: string[]): boolean {
+  return JSON.stringify([...(left ?? [])].sort()) === JSON.stringify([...(right ?? [])].sort());
+}
+
+function sameGeneration(
+  left?: { temperature?: number; max_tokens?: number; seed?: number },
+  right?: { temperature?: number; max_tokens?: number; seed?: number }
+): boolean {
+  return (
+    left?.temperature === right?.temperature &&
+    left?.max_tokens === right?.max_tokens &&
+    left?.seed === right?.seed
+  );
+}

package/src/comparison/index.ts

@@ -0,0 +1,8 @@
+export {
+  assessComparisonEligibility,
+  isComparisonAvailable,
+  type ComparisonEligibility,
+  type ComparisonEligibilityReason,
+  type ComparisonEligibilityReasonCode,
+  type ComparisonEligibilityStatus,
+} from './eligibility';

package/src/index.ts

@@ -24,6 +24,9 @@ export * from './artifacts';
 // Provenance
 export * from './provenance';
 
+// Comparison eligibility
+export * from './comparison';
+
 // Utilities
 export * from './utils';
 

package/src/runner/executor.test.ts

@@ -317,6 +317,56 @@ describe('executeCase measurement integrity', () => {
     expect(result.target?.observed_models).toBeUndefined();
   });
 
+  it('records retried target failures as excluded retry-chain attempts', async () => {
+    let calls = 0;
+    const retryClient: ModelClient = {
+      ...client,
+      generate: async () => {
+        calls++;
+        if (calls === 1) throw new Error('temporary provider failure');
+        return {
+          id: 'response',
+          model: 'target-model',
+          text: 'target response',
+          tokens: { prompt: 1, completion: 1, total: 2 },
+          latencyMs: 1,
+          finishReason: 'stop',
+        };
+      },
+    };
+    registerEvaluator('custom', {
+      type: 'custom',
+      evaluate: async () => ({ passed: true, score: 1 }),
+    });
+
+    const result = await executeCase(scenario.cases[0], {
+      client: retryClient,
+      scenario,
+      retries: 1,
+      runId: 'assurance-run',
+      repetition: { index: 2, total: 3 },
+    });
+
+    expect(result.attempts).toBe(2);
+    expect(result.attempt_evidence).toEqual([
+      expect.objectContaining({
+        attempt_id: 'assurance-run:custom-evaluation:1',
+        retry_chain_id: 'assurance-run:custom-evaluation',
+        repetition_index: 2,
+        attempt_number: 1,
+        status: 'error',
+        included_in_outcome: false,
+        error_code: 'target_error',
+      }),
+      expect.objectContaining({
+        attempt_id: 'assurance-run:custom-evaluation:2',
+        attempt_number: 2,
+        status: 'passed',
+        included_in_outcome: true,
+      }),
+    ]);
+  });
+
   it('retains only the bounded evidence contract rather than evaluator details', async () => {
     const evaluator: Evaluator = {
       type: 'custom',

package/src/runner/executor.ts

@@ -3,6 +3,7 @@
  */
 
 import type {
+  CaseAttemptEvidence,
   CaseEvaluationEvidence,
   CaseEvaluationStatus,
   CaseRedactionInfo,
@@ -99,18 +100,50 @@ export async function executeCase(
   context: ExecutorContext
 ): Promise<CaseResult> {
   const { timeout, retries = 0 } = context;
+  if (!Number.isSafeInteger(retries) || retries < 0 || retries > 99) {
+    throw new RangeError('retries must be a whole number between 0 and 99');
+  }
   const caseStartTime = Date.now();
   const requestedModel = testCase.model || context.requestedModel || context.scenario.model;
+  const retryChainId = `${context.runId ?? 'untracked'}:${testCase.id}`;
+  const repetitionIndex = context.repetition?.index ?? 1;
+  const attemptEvidence: CaseAttemptEvidence[] = [];
 
   let lastError: Error | null = null;
 
   for (let attempt = 0; attempt <= retries; attempt++) {
+    const attemptStartTime = Date.now();
     try {
       const result = await executeCaseAttempt(testCase, context, timeout);
-      return { ...result, attempts: attempt + 1 };
+      return withAttemptEvidence(result, attemptEvidence, {
+        retryChainId,
+        repetitionIndex,
+        attemptNumber: attempt + 1,
+        includedInOutcome: true,
+        latencyMs: result.latencyMs,
+      });
     } catch (error) {
       lastError = error as Error;
-      if (error instanceof ToolLoopError) return { ...error.caseResult, attempts: attempt + 1 };
+      if (error instanceof ToolLoopError) {
+        return withAttemptEvidence(error.caseResult, attemptEvidence, {
+          retryChainId,
+          repetitionIndex,
+          attemptNumber: attempt + 1,
+          includedInOutcome: true,
+          latencyMs: error.caseResult.latencyMs,
+          errorCode: 'tool_error',
+        });
+      }
+      attemptEvidence.push({
+        attempt_id: `${retryChainId}:${attempt + 1}`,
+        retry_chain_id: retryChainId,
+        repetition_index: repetitionIndex,
+        attempt_number: attempt + 1,
+        status: 'error',
+        included_in_outcome: false,
+        latency_ms: Date.now() - attemptStartTime,
+        error_code: error instanceof TimeoutError ? 'timeout' : 'target_error',
+      });
       if (attempt < retries) {
         // Wait before retry with exponential backoff
         await sleep(2 ** attempt * 1000);
@@ -126,6 +159,10 @@ export async function executeCase(
     ok: false,
     status: 'error',
     attempts: retries + 1,
+    attempt_evidence: attemptEvidence.map((entry, index) => ({
+      ...entry,
+      included_in_outcome: index === attemptEvidence.length - 1,
+    })),
     score: 0,
     matcherType: testCase.expected.type,
     reason: `Failed after ${retries + 1} attempts: ${lastError?.message}`,
@@ -140,6 +177,41 @@ export async function executeCase(
   };
 }
 
+function withAttemptEvidence(
+  result: CaseResult,
+  priorAttempts: CaseAttemptEvidence[],
+  input: {
+    retryChainId: string;
+    repetitionIndex: number;
+    attemptNumber: number;
+    includedInOutcome: boolean;
+    latencyMs: number;
+    errorCode?: CaseAttemptEvidence['error_code'];
+  }
+): CaseResult {
+  return {
+    ...result,
+    attempts: input.attemptNumber,
+    attempt_evidence: [
+      ...priorAttempts,
+      {
+        attempt_id: `${input.retryChainId}:${input.attemptNumber}`,
+        retry_chain_id: input.retryChainId,
+        repetition_index: input.repetitionIndex,
+        attempt_number: input.attemptNumber,
+        status: getTerminalStatus(result),
+        included_in_outcome: input.includedInOutcome,
+        latency_ms: input.latencyMs,
+        ...(input.errorCode ? { error_code: input.errorCode } : {}),
+      },
+    ],
+  };
+}
+
+function getTerminalStatus(result: CaseResult): CaseEvaluationStatus {
+  return result.status ?? (result.ok ? 'passed' : result.error ? 'error' : 'failed');
+}
+
 async function executeCaseAttempt(
   testCase: TestCase,
   context: ExecutorContext,

package/src/runner/runner.ts

@@ -2,6 +2,7 @@
  * Scenario runner - main entry point for running test scenarios
  */
 
+import { nanoid } from 'nanoid';
 import { createRunManifest } from '../artifacts/manifest';
 import type { CaseResult, ManifestRedactionInfo } from '../artifacts/types';
 import { createExecutionProvenance, createWorkloadIdentity } from '../provenance';
@@ -22,6 +23,8 @@ export async function runScenario(options: RunOptions): Promise<RunResult> {
     concurrency = 1,
     timeout,
     retries,
+    repetition = { index: 1, total: 1 },
+    costProvenance,
     redaction,
     toolExecutor,
     onCaseComplete,
@@ -42,6 +45,7 @@ export async function runScenario(options: RunOptions): Promise<RunResult> {
   onProgress?.(`Running ${cases.length} test cases...`);
 
   const startTime = new Date();
+  const runId = nanoid(12);
   const results: CaseResult[] = [];
 
   if (concurrency === 1) {
@@ -54,6 +58,8 @@ export async function runScenario(options: RunOptions): Promise<RunResult> {
         requestedModel: resolvedConfig?.model,
         timeout: testCase.timeout || timeout,
         retries: testCase.retries ?? retries,
+        runId,
+        repetition,
         redaction,
         toolExecutor,
       });
@@ -74,6 +80,8 @@ export async function runScenario(options: RunOptions): Promise<RunResult> {
             requestedModel: resolvedConfig?.model,
             timeout: testCase.timeout || timeout,
             retries: testCase.retries ?? retries,
+            runId,
+            repetition,
             redaction,
             toolExecutor,
           });
@@ -132,9 +140,21 @@ export async function runScenario(options: RunOptions): Promise<RunResult> {
       seed: scenario.seed,
       cases: results,
     }),
+    attemptEvidence: {
+      schema_version: '1',
+      repetition,
+      retry_policy: {
+        default_max_retries: retries ?? 0,
+        backoff: 'exponential',
+        initial_delay_ms: 1000,
+      },
+      ...(timeout ? { timeout: { default_ms: timeout } } : {}),
+    },
+    costProvenance,
     cases: results,
     startTime,
     endTime,
+    runId,
     redaction: redactionInfo,
   });
 

package/src/runner/types.ts

@@ -3,7 +3,7 @@
  */
 
 import type { ModelClient } from '../adapters/types';
-import type { CaseResult, ResolvedConfig, RunManifest } from '../artifacts/types';
+import type { CaseResult, CostProvenance, ResolvedConfig, RunManifest } from '../artifacts/types';
 import type { RedactionConfig } from '../redaction/types';
 import type { Scenario } from '../scenario/schema';
 import type { ToolExecutor } from '../tools';
@@ -28,6 +28,10 @@ export interface RunOptions {
   timeout?: number;
   /** Number of retries per case */
   retries?: number;
+  /** One-based coordinate for an independently planned repetition. */
+  repetition?: { index: number; total: number };
+  /** Attested or operator-supplied monetary evidence; omitted means unavailable. */
+  costProvenance?: CostProvenance;
   /** Redaction configuration (CLI overrides scenario) */
   redaction?: RedactionConfig;
   /** SDK-only executor for explicitly supplied real tools. */
@@ -60,6 +64,8 @@ export interface ExecutorContext {
   requestedModel?: string;
   timeout?: number;
   retries?: number;
+  runId?: string;
+  repetition?: { index: number; total: number };
   /** Redaction configuration for this execution */
   redaction?: RedactionConfig;
   toolExecutor?: ToolExecutor;

package/src/scenario/schema.ts

@@ -225,7 +225,7 @@ export const TestCaseSchema = z.object({
   tags: z.array(z.string()).optional().default([]),
   metadata: z.record(z.unknown()).optional().default({}),
   timeout: z.number().optional(),
-  retries: z.number().optional().default(0),
+  retries: z.number().int().min(0).max(99).optional().default(0),
   provider: ProviderSchema.optional(),
   model: z.string().optional(),
   variables: VariablesSchema.optional(),

package/src/storage/local.test.ts

@@ -179,6 +179,45 @@ describe('LocalStorageAdapter', () => {
     expect(comparison.delta.successRate).toBeCloseTo(0.1, 2);
     expect(comparison.delta.latency).toBe(-30);
     expect(comparison.delta.tokens).toBe(100);
+    expect(comparison.eligibility.status).toBe('qualified');
+  });
+
+  test('withholds deltas for incompatible workload evidence', async () => {
+    const baseline = {
+      ...mockManifest,
+      run_id: 'incompatible-baseline',
+      workload_identity: {
+        schema_version: '1' as const,
+        workload: {
+          schema_version: '1' as const,
+          algorithm: 'sha256' as const,
+          digest: 'a'.repeat(64),
+        },
+        rubric: {
+          schema_version: '1' as const,
+          algorithm: 'sha256' as const,
+          digest: 'b'.repeat(64),
+        },
+      },
+      execution_provenance: { schema_version: '1' as const, target: { provider: 'openai' } },
+    };
+    const current = {
+      ...baseline,
+      run_id: 'incompatible-current',
+      workload_identity: {
+        ...baseline.workload_identity,
+        workload: { ...baseline.workload_identity.workload, digest: 'c'.repeat(64) },
+      },
+    };
+
+    await storage.save(baseline);
+    await storage.save(current);
+
+    const comparison = await storage.compare('incompatible-baseline', 'incompatible-current');
+
+    expect(comparison.eligibility.status).toBe('incomparable');
+    expect(comparison.eligibility.reasons).toEqual([{ code: 'workload_mismatch' }]);
+    expect(comparison.delta).toBeUndefined();
   });
 
   test('handles empty storage gracefully', async () => {

package/src/storage/local.ts

@@ -12,6 +12,7 @@ import {
   assertRunManifestIntegrity,
   isRunManifest,
 } from '../artifacts/types';
+import { assessComparisonEligibility, isComparisonAvailable } from '../comparison';
 import type {
   BaselineMetadata,
   BaselineStorageAdapter,
@@ -212,9 +213,15 @@ export class LocalStorageAdapter implements BaselineStorageAdapter {
       this.loadRun(currentId),
     ]);
 
+    const eligibility = assessComparisonEligibility(baseline, current);
+    if (!isComparisonAvailable(eligibility)) {
+      return { baseline, current, eligibility };
+    }
+
     return {
       baseline,
       current,
+      eligibility,
       delta: {
         successRate: current.metrics.success_rate - baseline.metrics.success_rate,
         latency: current.metrics.median_latency_ms - baseline.metrics.median_latency_ms,
@@ -375,7 +382,8 @@ export class LocalStorageAdapter implements BaselineStorageAdapter {
     const comparison = await this.compare(baseline.runId, runId);
 
     // Check for regression (negative delta in success rate)
-    const hasRegression = comparison.delta.successRate < -regressionThreshold;
+    const hasRegression =
+      comparison.delta !== undefined && comparison.delta.successRate < -regressionThreshold;
 
     return {
       baseline,

package/src/storage/supabase.ts

@@ -9,6 +9,7 @@ import {
   assertRunManifestIntegrity,
   getCaseEvaluationStatus,
 } from '../artifacts/types';
+import { assessComparisonEligibility, isComparisonAvailable } from '../comparison';
 import type {
   AnalyticsStorageAdapter,
   BaselineMetadata,
@@ -199,9 +200,15 @@ export class SupabaseStorageAdapter implements AnalyticsStorageAdapter {
   async compare(baselineId: string, currentId: string): Promise<ComparisonResult> {
     const [baseline, current] = await Promise.all([this.load(baselineId), this.load(currentId)]);
 
+    const eligibility = assessComparisonEligibility(baseline, current);
+    if (!isComparisonAvailable(eligibility)) {
+      return { baseline, current, eligibility };
+    }
+
     return {
       baseline,
       current,
+      eligibility,
       delta: {
         successRate: current.metrics.success_rate - baseline.metrics.success_rate,
         latency: current.metrics.median_latency_ms - baseline.metrics.median_latency_ms,
@@ -400,7 +407,8 @@ export class SupabaseStorageAdapter implements AnalyticsStorageAdapter {
     const comparison = await this.compare(baseline.runId, runId);
 
     // Check for regression (success rate dropped by more than threshold)
-    const hasRegression = comparison.delta.successRate < -regressionThreshold;
+    const hasRegression =
+      comparison.delta !== undefined && comparison.delta.successRate < -regressionThreshold;
 
     return {
       baseline,

package/src/storage/types.ts

@@ -9,6 +9,7 @@ import type {
   RunManifest,
   StressManifest,
 } from '../artifacts/types';
+import type { ComparisonEligibility } from '../comparison';
 
 /**
  * Run listing item
@@ -30,7 +31,10 @@ export interface RunListItem {
 export interface ComparisonResult {
   baseline: RunManifest;
   current: RunManifest;
-  delta: {
+  /** Compatibility decision made before any metric delta is calculated. */
+  eligibility: ComparisonEligibility;
+  /** Absent when workloads or rubrics are incomparable. */
+  delta?: {
     successRate: number;
     latency: number;
     tokens: number;