@artemiskit/core 0.5.0 → 0.5.2

package/src/runner/executor.ts

@@ -3,6 +3,7 @@
  */
 
 import type {
+  CaseAttemptEvidence,
   CaseEvaluationEvidence,
   CaseEvaluationStatus,
   CaseRedactionInfo,
@@ -99,17 +100,50 @@ export async function executeCase(
   context: ExecutorContext
 ): Promise<CaseResult> {
   const { timeout, retries = 0 } = context;
+  if (!Number.isSafeInteger(retries) || retries < 0 || retries > 99) {
+    throw new RangeError('retries must be a whole number between 0 and 99');
+  }
   const caseStartTime = Date.now();
+  const requestedModel = testCase.model || context.requestedModel || context.scenario.model;
+  const retryChainId = `${context.runId ?? 'untracked'}:${testCase.id}`;
+  const repetitionIndex = context.repetition?.index ?? 1;
+  const attemptEvidence: CaseAttemptEvidence[] = [];
 
   let lastError: Error | null = null;
 
   for (let attempt = 0; attempt <= retries; attempt++) {
+    const attemptStartTime = Date.now();
     try {
       const result = await executeCaseAttempt(testCase, context, timeout);
-      return { ...result, attempts: attempt + 1 };
+      return withAttemptEvidence(result, attemptEvidence, {
+        retryChainId,
+        repetitionIndex,
+        attemptNumber: attempt + 1,
+        includedInOutcome: true,
+        latencyMs: result.latencyMs,
+      });
     } catch (error) {
       lastError = error as Error;
-      if (error instanceof ToolLoopError) return { ...error.caseResult, attempts: attempt + 1 };
+      if (error instanceof ToolLoopError) {
+        return withAttemptEvidence(error.caseResult, attemptEvidence, {
+          retryChainId,
+          repetitionIndex,
+          attemptNumber: attempt + 1,
+          includedInOutcome: true,
+          latencyMs: error.caseResult.latencyMs,
+          errorCode: 'tool_error',
+        });
+      }
+      attemptEvidence.push({
+        attempt_id: `${retryChainId}:${attempt + 1}`,
+        retry_chain_id: retryChainId,
+        repetition_index: repetitionIndex,
+        attempt_number: attempt + 1,
+        status: 'error',
+        included_in_outcome: false,
+        latency_ms: Date.now() - attemptStartTime,
+        error_code: error instanceof TimeoutError ? 'timeout' : 'target_error',
+      });
       if (attempt < retries) {
         // Wait before retry with exponential backoff
         await sleep(2 ** attempt * 1000);
@@ -125,6 +159,10 @@ export async function executeCase(
     ok: false,
     status: 'error',
     attempts: retries + 1,
+    attempt_evidence: attemptEvidence.map((entry, index) => ({
+      ...entry,
+      included_in_outcome: index === attemptEvidence.length - 1,
+    })),
     score: 0,
     matcherType: testCase.expected.type,
     reason: `Failed after ${retries + 1} attempts: ${lastError?.message}`,
@@ -135,15 +173,51 @@ export async function executeCase(
     expected: testCase.expected,
     tags: testCase.tags,
     error: lastError?.message,
+    target: targetEvidence(context.client.provider, requestedModel),
   };
 }
 
+function withAttemptEvidence(
+  result: CaseResult,
+  priorAttempts: CaseAttemptEvidence[],
+  input: {
+    retryChainId: string;
+    repetitionIndex: number;
+    attemptNumber: number;
+    includedInOutcome: boolean;
+    latencyMs: number;
+    errorCode?: CaseAttemptEvidence['error_code'];
+  }
+): CaseResult {
+  return {
+    ...result,
+    attempts: input.attemptNumber,
+    attempt_evidence: [
+      ...priorAttempts,
+      {
+        attempt_id: `${input.retryChainId}:${input.attemptNumber}`,
+        retry_chain_id: input.retryChainId,
+        repetition_index: input.repetitionIndex,
+        attempt_number: input.attemptNumber,
+        status: getTerminalStatus(result),
+        included_in_outcome: input.includedInOutcome,
+        latency_ms: input.latencyMs,
+        ...(input.errorCode ? { error_code: input.errorCode } : {}),
+      },
+    ],
+  };
+}
+
+function getTerminalStatus(result: CaseResult): CaseEvaluationStatus {
+  return result.status ?? (result.ok ? 'passed' : result.error ? 'error' : 'failed');
+}
+
 async function executeCaseAttempt(
   testCase: TestCase,
   context: ExecutorContext,
   timeout?: number
 ): Promise<CaseResult> {
-  const { client, scenario, redaction: cliRedaction, toolExecutor } = context;
+  const { client, scenario, requestedModel, redaction: cliRedaction, toolExecutor } = context;
 
   // Merge scenario-level and case-level variables (case overrides scenario)
   const variables = mergeVariables(scenario.variables, testCase.variables);
@@ -171,7 +245,7 @@ async function executeCaseAttempt(
   const generate = () =>
     client.generate({
       prompt: loopPrompt,
-      model: testCase.model || scenario.model,
+      model: testCase.model || requestedModel || scenario.model,
       temperature: scenario.temperature,
       maxTokens: scenario.maxTokens,
       seed: scenario.seed,
@@ -182,6 +256,7 @@ async function executeCaseAttempt(
   let result = timeout
     ? await Promise.race([generatePromise, createTimeout(timeout)])
     : await generatePromise;
+  const observedModels = [result.model];
   const generationMetrics = {
     latencyMs: result.latencyMs,
     tokens: { ...result.tokens },
@@ -198,7 +273,12 @@ async function executeCaseAttempt(
           terminationReason: 'tool_error',
         },
         generationMetrics,
-        'TOOL_EXECUTOR_REQUIRED'
+        'TOOL_EXECUTOR_REQUIRED',
+        targetEvidence(
+          client.provider,
+          testCase.model || requestedModel || scenario.model,
+          observedModels
+        )
       );
     }
     const executor =
@@ -225,7 +305,12 @@ async function executeCaseAttempt(
               terminationReason: 'duplicate_call',
             },
             generationMetrics,
-            'TOOL_DUPLICATE_CALL'
+            'TOOL_DUPLICATE_CALL',
+            targetEvidence(
+              client.provider,
+              testCase.model || requestedModel || scenario.model,
+              observedModels
+            )
           );
         }
         seenCalls.add(fingerprint);
@@ -263,7 +348,12 @@ async function executeCaseAttempt(
                     : 'tool_error',
             },
             generationMetrics,
-            execution.error?.code ?? 'TOOL_EXECUTION_FAILED'
+            execution.error?.code ?? 'TOOL_EXECUTION_FAILED',
+            targetEvidence(
+              client.provider,
+              testCase.model || requestedModel || scenario.model,
+              observedModels
+            )
           );
         }
         const content = JSON.stringify(execution.result ?? {});
@@ -281,7 +371,12 @@ async function executeCaseAttempt(
           toolTrace,
           { status: 'error', steps: step + 1, terminationReason: 'timeout' },
           generationMetrics,
-          'TOOL_LOOP_TIMEOUT'
+          'TOOL_LOOP_TIMEOUT',
+          targetEvidence(
+            client.provider,
+            testCase.model || requestedModel || scenario.model,
+            observedModels
+          )
         );
       }
       const requestTimeout = timeout ? Math.min(timeout, remainingLoopTime) : remainingLoopTime;
@@ -298,9 +393,15 @@ async function executeCaseAttempt(
             terminationReason: timedOut ? 'timeout' : 'tool_error',
           },
           generationMetrics,
-          timedOut ? 'TOOL_LOOP_TIMEOUT' : 'TOOL_GENERATION_FAILED'
+          timedOut ? 'TOOL_LOOP_TIMEOUT' : 'TOOL_GENERATION_FAILED',
+          targetEvidence(
+            client.provider,
+            testCase.model || requestedModel || scenario.model,
+            observedModels
+          )
         );
       }
+      observedModels.push(result.model);
       generationMetrics.latencyMs += result.latencyMs;
       generationMetrics.tokens.prompt += result.tokens.prompt;
       generationMetrics.tokens.completion += result.tokens.completion;
@@ -316,7 +417,12 @@ async function executeCaseAttempt(
           terminationReason: 'max_steps',
         },
         generationMetrics,
-        'TOOL_LOOP_MAX_STEPS'
+        'TOOL_LOOP_MAX_STEPS',
+        targetEvidence(
+          client.provider,
+          testCase.model || requestedModel || scenario.model,
+          observedModels
+        )
       );
     }
     toolLoop = { status: 'completed', steps: toolTrace.length, terminationReason: 'completed' };
@@ -435,17 +541,44 @@ async function executeCaseAttempt(
     tags: testCase.tags,
     redaction: redactionInfo,
     evidence: finalEvidence,
+    target: targetEvidence(
+      client.provider,
+      testCase.model || requestedModel || scenario.model,
+      observedModels
+    ),
     toolTrace: toolTrace.length ? toolTrace : undefined,
     toolLoop,
   };
 }
 
+function targetEvidence(
+  provider: string,
+  requestedModel?: string,
+  observedModels?: unknown[]
+): NonNullable<CaseResult['target']> {
+  const observed = [
+    ...new Set(
+      (observedModels ?? []).filter(
+        (model): model is string => typeof model === 'string' && model.length > 0
+      )
+    ),
+  ]
+    .map((model) => sanitizeArtifactText(model, 200))
+    .filter((model): model is string => Boolean(model));
+  return {
+    provider: sanitizeArtifactText(provider, 100) ?? 'unknown',
+    ...(requestedModel ? { requested_model: sanitizeArtifactText(requestedModel, 200) } : {}),
+    ...(observed.length ? { observed_models: observed } : {}),
+  };
+}
+
 function createToolLoopError(
   testCase: TestCase,
   toolTrace: ToolTraceEntry[],
   toolLoop: ToolLoopSummary,
   generationMetrics: Pick<CaseResult, 'latencyMs' | 'tokens'>,
-  code: string
+  code: string,
+  target?: CaseResult['target']
 ): ToolLoopError {
   return new ToolLoopError(code, {
     id: testCase.id,
@@ -462,6 +595,7 @@ function createToolLoopError(
     expected: testCase.expected,
     tags: testCase.tags,
     error: code,
+    target,
     toolTrace,
     toolLoop,
   });

package/src/runner/release-validation.test.ts

@@ -96,6 +96,13 @@ describe('release validation: fixture-backed workflow cases', () => {
       workload: { algorithm: 'sha256' },
       rubric: { algorithm: 'sha256' },
     });
+    expect(result.manifest.execution_provenance).toMatchObject({
+      schema_version: '1',
+      target: {
+        provider: 'fixture',
+        observed_models: ['fixture-model'],
+      },
+    });
   });
 
   test('RV-02 retains independent logistics tool evidence through a fixture-backed workflow', async () => {
@@ -171,4 +178,33 @@ describe('release validation: fixture-backed workflow cases', () => {
       toolTrace: [{ toolCall: { id: 'capacity-1' }, result: { available: true } }],
     });
   });
+
+  test('records a CLI-resolved model for each case when the scenario does not declare one', async () => {
+    const scenario = ScenarioSchema.parse({
+      name: 'resolved model evidence',
+      cases: [
+        {
+          id: 'cli-model-override',
+          prompt: 'Respond with assured',
+          expected: { type: 'exact', value: 'assured' },
+        },
+      ],
+    });
+
+    const result = await runScenario({
+      scenario,
+      client: fixtureClient(['assured']),
+      resolvedConfig: {
+        provider: 'fixture',
+        model: 'fixture-cli-model',
+        source: { provider: 'cli', model: 'cli' },
+      },
+    });
+
+    expect(result.cases[0].target).toEqual({
+      provider: 'fixture',
+      requested_model: 'fixture-cli-model',
+      observed_models: ['fixture-model'],
+    });
+  });
 });

package/src/runner/runner.ts

@@ -2,9 +2,10 @@
  * Scenario runner - main entry point for running test scenarios
  */
 
+import { nanoid } from 'nanoid';
 import { createRunManifest } from '../artifacts/manifest';
 import type { CaseResult, ManifestRedactionInfo } from '../artifacts/types';
-import { createWorkloadIdentity } from '../provenance';
+import { createExecutionProvenance, createWorkloadIdentity } from '../provenance';
 import { Redactor } from '../redaction';
 import { executeCase } from './executor';
 import type { RunOptions, RunResult } from './types';
@@ -22,6 +23,8 @@ export async function runScenario(options: RunOptions): Promise<RunResult> {
     concurrency = 1,
     timeout,
     retries,
+    repetition = { index: 1, total: 1 },
+    costProvenance,
     redaction,
     toolExecutor,
     onCaseComplete,
@@ -42,6 +45,7 @@ export async function runScenario(options: RunOptions): Promise<RunResult> {
   onProgress?.(`Running ${cases.length} test cases...`);
 
   const startTime = new Date();
+  const runId = nanoid(12);
   const results: CaseResult[] = [];
 
   if (concurrency === 1) {
@@ -51,8 +55,11 @@ export async function runScenario(options: RunOptions): Promise<RunResult> {
       const result = await executeCase(testCase, {
         client,
         scenario,
+        requestedModel: resolvedConfig?.model,
         timeout: testCase.timeout || timeout,
         retries: testCase.retries ?? retries,
+        runId,
+        repetition,
         redaction,
         toolExecutor,
       });
@@ -70,8 +77,11 @@ export async function runScenario(options: RunOptions): Promise<RunResult> {
           const result = await executeCase(testCase, {
             client,
             scenario,
+            requestedModel: resolvedConfig?.model,
             timeout: testCase.timeout || timeout,
             retries: testCase.retries ?? retries,
+            runId,
+            repetition,
             redaction,
             toolExecutor,
           });
@@ -122,9 +132,29 @@ export async function runScenario(options: RunOptions): Promise<RunResult> {
     },
     resolvedConfig,
     workloadIdentity: createWorkloadIdentity(scenario),
+    executionProvenance: createExecutionProvenance({
+      provider: client.provider,
+      requestedModel: resolvedConfig?.model || scenario.model,
+      temperature: resolvedConfig?.temperature ?? scenario.temperature,
+      maxTokens: resolvedConfig?.max_tokens ?? scenario.maxTokens,
+      seed: scenario.seed,
+      cases: results,
+    }),
+    attemptEvidence: {
+      schema_version: '1',
+      repetition,
+      retry_policy: {
+        default_max_retries: retries ?? 0,
+        backoff: 'exponential',
+        initial_delay_ms: 1000,
+      },
+      ...(timeout ? { timeout: { default_ms: timeout } } : {}),
+    },
+    costProvenance,
     cases: results,
     startTime,
     endTime,
+    runId,
     redaction: redactionInfo,
   });
 

package/src/runner/types.ts

@@ -3,7 +3,7 @@
  */
 
 import type { ModelClient } from '../adapters/types';
-import type { CaseResult, ResolvedConfig, RunManifest } from '../artifacts/types';
+import type { CaseResult, CostProvenance, ResolvedConfig, RunManifest } from '../artifacts/types';
 import type { RedactionConfig } from '../redaction/types';
 import type { Scenario } from '../scenario/schema';
 import type { ToolExecutor } from '../tools';
@@ -28,6 +28,10 @@ export interface RunOptions {
   timeout?: number;
   /** Number of retries per case */
   retries?: number;
+  /** One-based coordinate for an independently planned repetition. */
+  repetition?: { index: number; total: number };
+  /** Attested or operator-supplied monetary evidence; omitted means unavailable. */
+  costProvenance?: CostProvenance;
   /** Redaction configuration (CLI overrides scenario) */
   redaction?: RedactionConfig;
   /** SDK-only executor for explicitly supplied real tools. */
@@ -56,8 +60,12 @@ export interface RunResult {
 export interface ExecutorContext {
   client: ModelClient;
   scenario: Scenario;
+  /** Effective model selected outside the scenario, such as a CLI override. */
+  requestedModel?: string;
   timeout?: number;
   retries?: number;
+  runId?: string;
+  repetition?: { index: number; total: number };
   /** Redaction configuration for this execution */
   redaction?: RedactionConfig;
   toolExecutor?: ToolExecutor;

package/src/scenario/schema.ts

@@ -225,7 +225,7 @@ export const TestCaseSchema = z.object({
   tags: z.array(z.string()).optional().default([]),
   metadata: z.record(z.unknown()).optional().default({}),
   timeout: z.number().optional(),
-  retries: z.number().optional().default(0),
+  retries: z.number().int().min(0).max(99).optional().default(0),
   provider: ProviderSchema.optional(),
   model: z.string().optional(),
   variables: VariablesSchema.optional(),