@artemiskit/core 0.4.2 → 0.5.1

package/src/runner/executor.ts

@@ -100,6 +100,7 @@ export async function executeCase(
 ): Promise<CaseResult> {
   const { timeout, retries = 0 } = context;
   const caseStartTime = Date.now();
+  const requestedModel = testCase.model || context.requestedModel || context.scenario.model;
 
   let lastError: Error | null = null;
 
@@ -135,6 +136,7 @@ export async function executeCase(
     expected: testCase.expected,
     tags: testCase.tags,
     error: lastError?.message,
+    target: targetEvidence(context.client.provider, requestedModel),
   };
 }
 
@@ -143,7 +145,7 @@ async function executeCaseAttempt(
   context: ExecutorContext,
   timeout?: number
 ): Promise<CaseResult> {
-  const { client, scenario, redaction: cliRedaction, toolExecutor } = context;
+  const { client, scenario, requestedModel, redaction: cliRedaction, toolExecutor } = context;
 
   // Merge scenario-level and case-level variables (case overrides scenario)
   const variables = mergeVariables(scenario.variables, testCase.variables);
@@ -171,7 +173,7 @@ async function executeCaseAttempt(
   const generate = () =>
     client.generate({
       prompt: loopPrompt,
-      model: testCase.model || scenario.model,
+      model: testCase.model || requestedModel || scenario.model,
       temperature: scenario.temperature,
       maxTokens: scenario.maxTokens,
       seed: scenario.seed,
@@ -182,6 +184,7 @@ async function executeCaseAttempt(
   let result = timeout
     ? await Promise.race([generatePromise, createTimeout(timeout)])
     : await generatePromise;
+  const observedModels = [result.model];
   const generationMetrics = {
     latencyMs: result.latencyMs,
     tokens: { ...result.tokens },
@@ -198,7 +201,12 @@ async function executeCaseAttempt(
           terminationReason: 'tool_error',
         },
         generationMetrics,
-        'TOOL_EXECUTOR_REQUIRED'
+        'TOOL_EXECUTOR_REQUIRED',
+        targetEvidence(
+          client.provider,
+          testCase.model || requestedModel || scenario.model,
+          observedModels
+        )
       );
     }
     const executor =
@@ -225,7 +233,12 @@ async function executeCaseAttempt(
               terminationReason: 'duplicate_call',
             },
             generationMetrics,
-            'TOOL_DUPLICATE_CALL'
+            'TOOL_DUPLICATE_CALL',
+            targetEvidence(
+              client.provider,
+              testCase.model || requestedModel || scenario.model,
+              observedModels
+            )
           );
         }
         seenCalls.add(fingerprint);
@@ -263,7 +276,12 @@ async function executeCaseAttempt(
                     : 'tool_error',
             },
             generationMetrics,
-            execution.error?.code ?? 'TOOL_EXECUTION_FAILED'
+            execution.error?.code ?? 'TOOL_EXECUTION_FAILED',
+            targetEvidence(
+              client.provider,
+              testCase.model || requestedModel || scenario.model,
+              observedModels
+            )
           );
         }
         const content = JSON.stringify(execution.result ?? {});
@@ -281,7 +299,12 @@ async function executeCaseAttempt(
           toolTrace,
           { status: 'error', steps: step + 1, terminationReason: 'timeout' },
           generationMetrics,
-          'TOOL_LOOP_TIMEOUT'
+          'TOOL_LOOP_TIMEOUT',
+          targetEvidence(
+            client.provider,
+            testCase.model || requestedModel || scenario.model,
+            observedModels
+          )
         );
       }
       const requestTimeout = timeout ? Math.min(timeout, remainingLoopTime) : remainingLoopTime;
@@ -298,9 +321,15 @@ async function executeCaseAttempt(
             terminationReason: timedOut ? 'timeout' : 'tool_error',
           },
           generationMetrics,
-          timedOut ? 'TOOL_LOOP_TIMEOUT' : 'TOOL_GENERATION_FAILED'
+          timedOut ? 'TOOL_LOOP_TIMEOUT' : 'TOOL_GENERATION_FAILED',
+          targetEvidence(
+            client.provider,
+            testCase.model || requestedModel || scenario.model,
+            observedModels
+          )
         );
       }
+      observedModels.push(result.model);
       generationMetrics.latencyMs += result.latencyMs;
       generationMetrics.tokens.prompt += result.tokens.prompt;
       generationMetrics.tokens.completion += result.tokens.completion;
@@ -316,7 +345,12 @@ async function executeCaseAttempt(
           terminationReason: 'max_steps',
         },
         generationMetrics,
-        'TOOL_LOOP_MAX_STEPS'
+        'TOOL_LOOP_MAX_STEPS',
+        targetEvidence(
+          client.provider,
+          testCase.model || requestedModel || scenario.model,
+          observedModels
+        )
       );
     }
     toolLoop = { status: 'completed', steps: toolTrace.length, terminationReason: 'completed' };
@@ -435,17 +469,44 @@ async function executeCaseAttempt(
     tags: testCase.tags,
     redaction: redactionInfo,
     evidence: finalEvidence,
+    target: targetEvidence(
+      client.provider,
+      testCase.model || requestedModel || scenario.model,
+      observedModels
+    ),
     toolTrace: toolTrace.length ? toolTrace : undefined,
     toolLoop,
   };
 }
 
+function targetEvidence(
+  provider: string,
+  requestedModel?: string,
+  observedModels?: unknown[]
+): NonNullable<CaseResult['target']> {
+  const observed = [
+    ...new Set(
+      (observedModels ?? []).filter(
+        (model): model is string => typeof model === 'string' && model.length > 0
+      )
+    ),
+  ]
+    .map((model) => sanitizeArtifactText(model, 200))
+    .filter((model): model is string => Boolean(model));
+  return {
+    provider: sanitizeArtifactText(provider, 100) ?? 'unknown',
+    ...(requestedModel ? { requested_model: sanitizeArtifactText(requestedModel, 200) } : {}),
+    ...(observed.length ? { observed_models: observed } : {}),
+  };
+}
+
 function createToolLoopError(
   testCase: TestCase,
   toolTrace: ToolTraceEntry[],
   toolLoop: ToolLoopSummary,
   generationMetrics: Pick<CaseResult, 'latencyMs' | 'tokens'>,
-  code: string
+  code: string,
+  target?: CaseResult['target']
 ): ToolLoopError {
   return new ToolLoopError(code, {
     id: testCase.id,
@@ -462,6 +523,7 @@ function createToolLoopError(
     expected: testCase.expected,
     tags: testCase.tags,
     error: code,
+    target,
     toolTrace,
     toolLoop,
   });

package/src/runner/release-validation.test.ts

@@ -91,6 +91,18 @@ describe('release validation: fixture-backed workflow cases', () => {
       failed_cases: 2,
       success_rate: 0.6,
     });
+    expect(result.manifest.workload_identity).toMatchObject({
+      schema_version: '1',
+      workload: { algorithm: 'sha256' },
+      rubric: { algorithm: 'sha256' },
+    });
+    expect(result.manifest.execution_provenance).toMatchObject({
+      schema_version: '1',
+      target: {
+        provider: 'fixture',
+        observed_models: ['fixture-model'],
+      },
+    });
   });
 
   test('RV-02 retains independent logistics tool evidence through a fixture-backed workflow', async () => {
@@ -166,4 +178,33 @@ describe('release validation: fixture-backed workflow cases', () => {
       toolTrace: [{ toolCall: { id: 'capacity-1' }, result: { available: true } }],
     });
   });
+
+  test('records a CLI-resolved model for each case when the scenario does not declare one', async () => {
+    const scenario = ScenarioSchema.parse({
+      name: 'resolved model evidence',
+      cases: [
+        {
+          id: 'cli-model-override',
+          prompt: 'Respond with assured',
+          expected: { type: 'exact', value: 'assured' },
+        },
+      ],
+    });
+
+    const result = await runScenario({
+      scenario,
+      client: fixtureClient(['assured']),
+      resolvedConfig: {
+        provider: 'fixture',
+        model: 'fixture-cli-model',
+        source: { provider: 'cli', model: 'cli' },
+      },
+    });
+
+    expect(result.cases[0].target).toEqual({
+      provider: 'fixture',
+      requested_model: 'fixture-cli-model',
+      observed_models: ['fixture-model'],
+    });
+  });
 });

package/src/runner/runner.ts

@@ -4,6 +4,7 @@
 
 import { createRunManifest } from '../artifacts/manifest';
 import type { CaseResult, ManifestRedactionInfo } from '../artifacts/types';
+import { createExecutionProvenance, createWorkloadIdentity } from '../provenance';
 import { Redactor } from '../redaction';
 import { executeCase } from './executor';
 import type { RunOptions, RunResult } from './types';
@@ -50,6 +51,7 @@ export async function runScenario(options: RunOptions): Promise<RunResult> {
       const result = await executeCase(testCase, {
         client,
         scenario,
+        requestedModel: resolvedConfig?.model,
         timeout: testCase.timeout || timeout,
         retries: testCase.retries ?? retries,
         redaction,
@@ -69,6 +71,7 @@ export async function runScenario(options: RunOptions): Promise<RunResult> {
           const result = await executeCase(testCase, {
             client,
             scenario,
+            requestedModel: resolvedConfig?.model,
             timeout: testCase.timeout || timeout,
             retries: testCase.retries ?? retries,
             redaction,
@@ -120,6 +123,15 @@ export async function runScenario(options: RunOptions): Promise<RunResult> {
       seed: scenario.seed,
     },
     resolvedConfig,
+    workloadIdentity: createWorkloadIdentity(scenario),
+    executionProvenance: createExecutionProvenance({
+      provider: client.provider,
+      requestedModel: resolvedConfig?.model || scenario.model,
+      temperature: resolvedConfig?.temperature ?? scenario.temperature,
+      maxTokens: resolvedConfig?.max_tokens ?? scenario.maxTokens,
+      seed: scenario.seed,
+      cases: results,
+    }),
     cases: results,
     startTime,
     endTime,

package/src/runner/types.ts

@@ -56,6 +56,8 @@ export interface RunResult {
 export interface ExecutorContext {
   client: ModelClient;
   scenario: Scenario;
+  /** Effective model selected outside the scenario, such as a CLI override. */
+  requestedModel?: string;
   timeout?: number;
   retries?: number;
   /** Redaction configuration for this execution */