@artemiskit/core 0.4.2 → 0.5.1

package/src/artifacts/manifest.test.ts

@@ -55,7 +55,7 @@ describe('createRunManifest', () => {
       endTime,
     });
 
-    expect(manifest.version).toBe('1.1');
+    expect(manifest.version).toBe('1.3');
     expect(manifest.project).toBe('test-project');
     expect(manifest.run_id).toBeTruthy();
     expect(manifest.run_id.length).toBe(12);
@@ -163,6 +163,56 @@ describe('createRunManifest', () => {
     expect(manifest.resolved_config?.source.model).toBe('config');
   });
 
+  test('retains a validated workload identity when provided', () => {
+    const manifest = createRunManifest({
+      project: 'test-project',
+      config: { scenario: 'test-scenario', provider: 'openai' },
+      workloadIdentity: {
+        schema_version: '1',
+        workload: {
+          schema_version: '1',
+          algorithm: 'sha256',
+          digest: 'a'.repeat(64),
+        },
+        rubric: {
+          schema_version: '1',
+          algorithm: 'sha256',
+          digest: 'b'.repeat(64),
+        },
+      },
+      cases: mockCases,
+      startTime: new Date(),
+      endTime: new Date(),
+    });
+
+    expect(manifest.workload_identity?.workload.digest).toBe('a'.repeat(64));
+    expect(() => assertRunManifestIntegrity(manifest)).not.toThrow();
+  });
+
+  test('retains target and evaluator execution provenance separately', () => {
+    const manifest = createRunManifest({
+      project: 'test-project',
+      config: { scenario: 'test-scenario', provider: 'openai' },
+      executionProvenance: {
+        schema_version: '1',
+        target: {
+          provider: 'openai',
+          requested_models: ['gpt-requested'],
+          observed_models: ['gpt-observed'],
+          generation: { temperature: 0, max_tokens: 100, seed: 42 },
+        },
+        evaluator: { models: ['judge-model'] },
+      },
+      cases: mockCases,
+      startTime: new Date(),
+      endTime: new Date(),
+    });
+
+    expect(manifest.execution_provenance?.target.observed_models).toEqual(['gpt-observed']);
+    expect(manifest.execution_provenance?.evaluator?.models).toEqual(['judge-model']);
+    expect(() => assertRunManifestIntegrity(manifest)).not.toThrow();
+  });
+
   test('includes provenance information', () => {
     const manifest = createRunManifest({
       project: 'test-project',
@@ -273,5 +323,21 @@ describe('createRunManifest', () => {
       ],
     };
     expect(() => assertRunManifestIntegrity(malformed)).toThrow('invalid evidence validation');
+
+    expect(() =>
+      assertRunManifestIntegrity({ ...historical, workload_identity: { schema_version: '1' } })
+    ).toThrow('malformed workload identity');
+    expect(() =>
+      assertRunManifestIntegrity({
+        ...historical,
+        execution_provenance: { schema_version: '1', target: { provider: '' } },
+      })
+    ).toThrow('malformed execution provenance');
+    expect(() =>
+      assertRunManifestIntegrity({
+        ...historical,
+        cases: [{ ...historical.cases[0], target: { provider: '', observed_models: ['x'] } }],
+      })
+    ).toThrow('malformed target evidence');
   });
 });

package/src/artifacts/manifest.ts

@@ -9,11 +9,13 @@ import { getGitInfo } from '../provenance/git';
 import type {
   CaseResult,
   CostEstimateInfo,
+  ExecutionProvenance,
   ManifestRedactionInfo,
   ResolvedConfig,
   RunConfig,
   RunManifest,
   RunMetrics,
+  WorkloadIdentity,
 } from './types';
 import { getCaseEvaluationStatus } from './types';
 
@@ -24,6 +26,8 @@ export function createRunManifest(options: {
   project: string;
   config: RunConfig;
   resolvedConfig?: ResolvedConfig;
+  workloadIdentity?: WorkloadIdentity;
+  executionProvenance?: ExecutionProvenance;
   cases: CaseResult[];
   startTime: Date;
   endTime: Date;
@@ -35,6 +39,8 @@ export function createRunManifest(options: {
     project,
     config,
     resolvedConfig,
+    workloadIdentity,
+    executionProvenance,
     cases,
     startTime,
     endTime,
@@ -50,7 +56,7 @@ export function createRunManifest(options: {
   const environment = getEnvironmentInfo();
 
   return {
-    version: '1.1',
+    version: '1.3',
     run_id: nanoid(12),
     project,
     start_time: startTime.toISOString(),
@@ -58,6 +64,8 @@ export function createRunManifest(options: {
     duration_ms: endTime.getTime() - startTime.getTime(),
     config,
     resolved_config: resolvedConfig,
+    workload_identity: workloadIdentity,
+    execution_provenance: executionProvenance,
     metrics,
     git,
     provenance: {

package/src/artifacts/types.ts

@@ -42,6 +42,56 @@ export interface ManifestRedactionInfo {
   };
 }
 
+// ============================================================================
+// Reproducible Evidence Types
+// ============================================================================
+
+/** A versioned SHA-256 digest of canonical, redacted assessment material. */
+export interface ContentIdentity {
+  schema_version: '1';
+  algorithm: 'sha256';
+  digest: string;
+}
+
+/**
+ * Separates the scenario workload from the criteria used to judge it.
+ *
+ * The digests prove matching declared, sanitized inputs. They are not a
+ * signature or an attestation of provider behaviour.
+ */
+export interface WorkloadIdentity {
+  schema_version: '1';
+  workload: ContentIdentity;
+  rubric: ContentIdentity;
+}
+
+/** Bounded target identity captured from a case execution. */
+export interface CaseTargetEvidence {
+  provider: string;
+  requested_model?: string;
+  /** Model identifiers returned by the target provider during this case. */
+  observed_models?: string[];
+}
+
+/** Requested and observed execution configuration for a complete run. */
+export interface ExecutionProvenance {
+  schema_version: '1';
+  target: {
+    provider: string;
+    requested_models?: string[];
+    observed_models?: string[];
+    generation?: {
+      temperature?: number;
+      max_tokens?: number;
+      seed?: number;
+    };
+  };
+  /** Judge/evaluator model identities, never combined with target identity. */
+  evaluator?: {
+    models?: string[];
+  };
+}
+
 // ============================================================================
 // Case Result Types
 // ============================================================================
@@ -101,6 +151,8 @@ export interface CaseResult {
   error?: string;
   /** Sanitized evaluator evidence; arbitrary evaluator details are never stored here. */
   evidence?: CaseEvaluationEvidence;
+  /** Requested and observed target identity for this case. */
+  target?: CaseTargetEvidence;
   /** Redaction information for this case */
   redaction?: CaseRedactionInfo;
   /** Ordered tool activity captured for an enabled tool loop. */
@@ -277,6 +329,10 @@ export interface RunManifest {
   config: RunConfig;
   /** Resolved configuration with full provider details and source tracking */
   resolved_config?: ResolvedConfig;
+  /** Versioned identities for the declared workload and evaluation rubric. */
+  workload_identity?: WorkloadIdentity;
+  /** Requested and observed target/evaluator configuration for this run. */
+  execution_provenance?: ExecutionProvenance;
   metrics: RunMetrics;
   git: GitInfo;
   provenance: ProvenanceInfo;
@@ -323,6 +379,13 @@ export function assertRunManifestIntegrity(manifest: unknown): asserts manifest
     throw new Error('Invalid run manifest: expected an object with a cases array');
   }
 
+  if (manifest.workload_identity !== undefined) {
+    assertWorkloadIdentity(manifest.workload_identity);
+  }
+  if (manifest.execution_provenance !== undefined) {
+    assertExecutionProvenance(manifest.execution_provenance);
+  }
+
   for (const [index, caseResult] of manifest.cases.entries()) {
     if (!isRecord(caseResult)) {
       throw new Error(`Invalid run manifest: case ${index} is not an object`);
@@ -341,9 +404,85 @@ export function assertRunManifestIntegrity(manifest: unknown): asserts manifest
     if (caseResult.evidence !== undefined) {
       assertCaseEvaluationEvidence(caseResult.evidence, index);
     }
+    if (caseResult.target !== undefined) {
+      assertCaseTargetEvidence(caseResult.target);
+    }
   }
 }
 
+function assertCaseTargetEvidence(target: unknown): void {
+  if (
+    !isRecord(target) ||
+    typeof target.provider !== 'string' ||
+    target.provider.length === 0 ||
+    target.provider.length > 100 ||
+    (target.requested_model !== undefined &&
+      (typeof target.requested_model !== 'string' || target.requested_model.length > 200)) ||
+    !isBoundedStringList(target.observed_models)
+  ) {
+    throw new Error('Invalid run manifest: malformed target evidence');
+  }
+}
+
+function assertExecutionProvenance(provenance: unknown): void {
+  if (!isRecord(provenance) || provenance.schema_version !== '1' || !isRecord(provenance.target)) {
+    throw new Error('Invalid run manifest: malformed execution provenance');
+  }
+  const target = provenance.target;
+  if (
+    typeof target.provider !== 'string' ||
+    target.provider.length === 0 ||
+    target.provider.length > 100 ||
+    !isBoundedStringList(target.requested_models) ||
+    !isBoundedStringList(target.observed_models) ||
+    (target.generation !== undefined && !isGenerationConfig(target.generation))
+  ) {
+    throw new Error('Invalid run manifest: malformed execution provenance');
+  }
+  if (provenance.evaluator !== undefined) {
+    if (!isRecord(provenance.evaluator) || !isBoundedStringList(provenance.evaluator.models)) {
+      throw new Error('Invalid run manifest: malformed execution provenance');
+    }
+  }
+}
+
+function isBoundedStringList(value: unknown): boolean {
+  return (
+    value === undefined ||
+    (Array.isArray(value) &&
+      value.length <= 100 &&
+      value.every((item) => typeof item === 'string' && item.length > 0 && item.length <= 200))
+  );
+}
+
+function isGenerationConfig(value: unknown): boolean {
+  if (!isRecord(value)) return false;
+  return [value.temperature, value.max_tokens, value.seed].every(
+    (item) => item === undefined || (typeof item === 'number' && Number.isFinite(item))
+  );
+}
+
+function assertWorkloadIdentity(identity: unknown): void {
+  if (
+    !isRecord(identity) ||
+    identity.schema_version !== '1' ||
+    !isContentIdentity(identity.workload) ||
+    !isContentIdentity(identity.rubric)
+  ) {
+    throw new Error('Invalid run manifest: malformed workload identity');
+  }
+}
+
+function isContentIdentity(value: unknown): boolean {
+  return (
+    isRecord(value) &&
+    value.schema_version === '1' &&
+    value.algorithm === 'sha256' &&
+    typeof value.digest === 'string' &&
+    /^[a-f0-9]{64}$/.test(value.digest)
+  );
+}
+
 function assertCaseEvaluationEvidence(evidence: unknown, caseIndex: number): void {
   if (
     !isRecord(evidence) ||

package/src/provenance/execution-provenance.test.ts

@@ -0,0 +1,59 @@
+import { describe, expect, test } from 'bun:test';
+import { createExecutionProvenance } from './execution-provenance';
+
+describe('createExecutionProvenance', () => {
+  test('keeps requested, observed target, and evaluator model identities separate', () => {
+    const provenance = createExecutionProvenance({
+      provider: 'openai',
+      requestedModel: 'gpt-requested',
+      temperature: 0.2,
+      maxTokens: 100,
+      seed: 7,
+      cases: [
+        {
+          id: 'one',
+          ok: true,
+          score: 1,
+          matcherType: 'exact',
+          latencyMs: 1,
+          tokens: { prompt: 1, completion: 1, total: 2 },
+          prompt: 'prompt',
+          response: 'response',
+          expected: { type: 'exact', value: 'response', caseSensitive: true },
+          tags: [],
+          target: {
+            provider: 'openai',
+            requested_model: 'gpt-requested',
+            observed_models: ['gpt-observed'],
+          },
+          evidence: { evaluator: 'llm_grader', model: 'judge-model' },
+        },
+      ],
+    });
+
+    expect(provenance).toEqual({
+      schema_version: '1',
+      target: {
+        provider: 'openai',
+        requested_models: ['gpt-requested'],
+        observed_models: ['gpt-observed'],
+        generation: { temperature: 0.2, max_tokens: 100, seed: 7 },
+      },
+      evaluator: { models: ['judge-model'] },
+    });
+  });
+
+  test('records an unavailable observed identity without inventing one', () => {
+    const provenance = createExecutionProvenance({
+      provider: 'custom',
+      requestedModel: 'requested-model',
+      cases: [],
+    });
+
+    expect(provenance.target).toEqual({
+      provider: 'custom',
+      requested_models: ['requested-model'],
+    });
+    expect(provenance.evaluator).toBeUndefined();
+  });
+});

package/src/provenance/execution-provenance.ts

@@ -0,0 +1,53 @@
+/** Bounded execution provenance derived from declared and observed run evidence. */
+
+import type { CaseResult, ExecutionProvenance } from '../artifacts/types';
+
+export function createExecutionProvenance(options: {
+  provider: string;
+  requestedModel?: string;
+  temperature?: number;
+  maxTokens?: number;
+  seed?: number;
+  cases: CaseResult[];
+}): ExecutionProvenance {
+  const requestedModels = uniqueStrings([
+    options.requestedModel,
+    ...options.cases.map((caseResult) => caseResult.target?.requested_model),
+  ]);
+  const observedModels = uniqueStrings(
+    options.cases.flatMap((caseResult) => caseResult.target?.observed_models ?? [])
+  );
+  const evaluatorModels = uniqueStrings(
+    options.cases.map((caseResult) => caseResult.evidence?.model)
+  );
+  const generation = omitUndefined({
+    temperature: options.temperature,
+    max_tokens: options.maxTokens,
+    seed: options.seed,
+  });
+
+  return {
+    schema_version: '1',
+    target: {
+      provider: boundedString(options.provider, 100) ?? 'unknown',
+      ...(requestedModels.length ? { requested_models: requestedModels } : {}),
+      ...(observedModels.length ? { observed_models: observedModels } : {}),
+      ...(Object.keys(generation).length ? { generation } : {}),
+    },
+    ...(evaluatorModels.length ? { evaluator: { models: evaluatorModels } } : {}),
+  };
+}
+
+function uniqueStrings(values: unknown[]): string[] {
+  return [...new Set(values.map((value) => boundedString(value, 200)).filter(Boolean))] as string[];
+}
+
+function boundedString(value: unknown, maxLength: number): string | undefined {
+  return typeof value === 'string' && value.length > 0 ? value.slice(0, maxLength) : undefined;
+}
+
+function omitUndefined<T extends Record<string, number | undefined>>(value: T): Partial<T> {
+  return Object.fromEntries(
+    Object.entries(value).filter(([, item]) => item !== undefined)
+  ) as Partial<T>;
+}

package/src/provenance/index.ts

@@ -4,3 +4,5 @@
 
 export { getGitInfo } from './git';
 export { getEnvironmentInfo } from './environment';
+export { createWorkloadIdentity } from './workload-identity';
+export { createExecutionProvenance } from './execution-provenance';

package/src/provenance/workload-identity.test.ts

@@ -0,0 +1,69 @@
+import { describe, expect, test } from 'bun:test';
+import { validateScenario } from '../scenario';
+import { createWorkloadIdentity } from './workload-identity';
+
+const baseScenario = () =>
+  validateScenario({
+    name: 'Customer-service policy test',
+    version: '2026-09',
+    providerConfig: { apiKey: 'sk-live-very-secret-value' },
+    setup: {
+      systemPrompt: 'Use token=internal-secret only for this fixture.',
+    },
+    cases: [
+      {
+        id: 'refund-policy',
+        prompt: 'Can I receive a refund?',
+        expected: {
+          type: 'llm_grader',
+          rubric: 'Approve only eligible refunds.',
+          strict: true,
+          threshold: 0.8,
+        },
+      },
+    ],
+  });
+
+describe('createWorkloadIdentity', () => {
+  test('is deterministic for equivalent declared scenario material', () => {
+    const first = createWorkloadIdentity(baseScenario());
+    const second = createWorkloadIdentity(baseScenario());
+
+    expect(first).toEqual(second);
+    expect(first.workload.digest).toHaveLength(64);
+    expect(first.rubric.digest).toHaveLength(64);
+  });
+
+  test('separates workload changes from rubric changes', () => {
+    const base = baseScenario();
+    const changedPrompt = structuredClone(base);
+    changedPrompt.cases[0].prompt = 'Can I receive a replacement?';
+    const changedRubric = structuredClone(base);
+    changedRubric.cases[0].expected = {
+      type: 'llm_grader',
+      rubric: 'Approve only eligible replacements.',
+      strict: true,
+      threshold: 0.8,
+    };
+
+    const baseIdentity = createWorkloadIdentity(base);
+    const promptIdentity = createWorkloadIdentity(changedPrompt);
+    const rubricIdentity = createWorkloadIdentity(changedRubric);
+
+    expect(promptIdentity.workload.digest).not.toBe(baseIdentity.workload.digest);
+    expect(promptIdentity.rubric.digest).toBe(baseIdentity.rubric.digest);
+    expect(rubricIdentity.workload.digest).toBe(baseIdentity.workload.digest);
+    expect(rubricIdentity.rubric.digest).not.toBe(baseIdentity.rubric.digest);
+  });
+
+  test('excludes sensitive configuration and recognized secret text before hashing', () => {
+    const first = baseScenario();
+    first.cases[0].metadata = { author: 'ArtemisKit', authToken: 'fixture-token-one' };
+    const second = structuredClone(first);
+    second.providerConfig = { apiKey: 'sk-live-another-secret-value' };
+    second.setup = { systemPrompt: 'Use token=different-secret only for this fixture.' };
+    second.cases[0].metadata = { author: 'ArtemisKit', authToken: 'fixture-token-two' };
+
+    expect(createWorkloadIdentity(second)).toEqual(createWorkloadIdentity(first));
+  });
+});

package/src/provenance/workload-identity.ts

@@ -0,0 +1,92 @@
+/**
+ * Canonical, redacted workload identities for reproducible assessment evidence.
+ */
+
+import { createHash } from 'node:crypto';
+import type { WorkloadIdentity } from '../artifacts/types';
+import { redactText, resolvePatterns } from '../redaction/redactor';
+import { DEFAULT_REDACTION_PATTERNS } from '../redaction/types';
+import type { Scenario } from '../scenario/schema';
+
+const SENSITIVE_KEY = /(?:api[-_]?key|authorization|auth|credential|password|secret|token)$/i;
+const REDACTED_SECRET = '[REDACTED_SECRET]';
+
+/**
+ * Create independent identities for the declared workload and its evaluation
+ * criteria. Input is sorted, redacted, and hashed locally; raw material is
+ * never retained in the identity artifact.
+ */
+export function createWorkloadIdentity(scenario: Scenario): WorkloadIdentity {
+  const patterns = resolvePatterns([
+    ...DEFAULT_REDACTION_PATTERNS,
+    ...(scenario.redaction?.patterns ?? []),
+  ]).map(({ regex }) => regex);
+
+  const workload = {
+    name: scenario.name,
+    version: scenario.version,
+    description: scenario.description,
+    tags: scenario.tags,
+    variables: scenario.variables,
+    setup: scenario.setup,
+    cases: scenario.cases.map((testCase) => ({
+      id: testCase.id,
+      name: testCase.name,
+      description: testCase.description,
+      prompt: testCase.prompt,
+      tags: testCase.tags,
+      metadata: testCase.metadata,
+      timeout: testCase.timeout,
+      retries: testCase.retries,
+      variables: testCase.variables,
+    })),
+  };
+
+  const rubric = {
+    cases: scenario.cases.map((testCase) => ({
+      id: testCase.id,
+      expected: testCase.expected,
+    })),
+  };
+
+  return {
+    schema_version: '1',
+    workload: createContentIdentity(workload, patterns),
+    rubric: createContentIdentity(rubric, patterns),
+  };
+}
+
+function createContentIdentity(value: unknown, patterns: RegExp[]) {
+  const canonical = JSON.stringify(canonicalize(value, patterns));
+  return {
+    schema_version: '1' as const,
+    algorithm: 'sha256' as const,
+    digest: createHash('sha256').update(canonical).digest('hex'),
+  };
+}
+
+function canonicalize(value: unknown, patterns: RegExp[], key?: string): unknown {
+  if (key && SENSITIVE_KEY.test(key)) return REDACTED_SECRET;
+
+  if (typeof value === 'string') {
+    return redactText(value, patterns, REDACTED_SECRET).text;
+  }
+
+  if (Array.isArray(value)) {
+    return value.map((item) => canonicalize(item, patterns));
+  }
+
+  if (value && typeof value === 'object') {
+    return Object.fromEntries(
+      Object.entries(value)
+        .filter(([, nestedValue]) => nestedValue !== undefined)
+        .sort(([left], [right]) => left.localeCompare(right))
+        .map(([nestedKey, nestedValue]) => [
+          nestedKey,
+          canonicalize(nestedValue, patterns, nestedKey),
+        ])
+    );
+  }
+
+  return value;
+}

package/src/runner/executor.test.ts

@@ -170,6 +170,11 @@ describe('executeCase tool loop', () => {
     expect(result.error).toBe('TOOL_EXECUTOR_REQUIRED');
     expect(result.latencyMs).toBe(4);
     expect(result.tokens).toEqual({ prompt: 7, completion: 2, total: 9 });
+    expect(result.target).toEqual({
+      provider: 'ling',
+      requested_model: 'Ling-3.0-flash',
+      observed_models: ['Ling-3.0-flash'],
+    });
   });
 
   it('retains prior generation metrics when a later generation rejects', async () => {
@@ -201,6 +206,7 @@ describe('executeCase tool loop', () => {
       steps: 1,
       terminationReason: 'tool_error',
     });
+    expect(result.target?.observed_models).toEqual(['Ling-3.0-flash']);
   });
 
   it('retains prior generation metrics when a later generation times out', async () => {
@@ -304,7 +310,11 @@ describe('executeCase measurement integrity', () => {
       status: 'error',
       response: '',
       error: 'provider unavailable',
+      target: {
+        provider: 'test',
+      },
     });
+    expect(result.target?.observed_models).toBeUndefined();
   });
 
   it('retains only the bounded evidence contract rather than evaluator details', async () => {