@artemiskit/core 0.3.0 → 0.4.1

package/src/artifacts/manifest.test.ts

@@ -4,6 +4,7 @@
 
 import { describe, expect, test } from 'bun:test';
 import { createRunManifest } from './manifest';
+import { assertRunManifestIntegrity } from './types';
 import type { CaseResult } from './types';
 
 describe('createRunManifest', () => {
@@ -12,35 +13,29 @@ describe('createRunManifest', () => {
       id: 'case-1',
       name: 'Test Case 1',
       ok: true,
+      status: 'passed',
       latencyMs: 100,
       prompt: 'Hello',
       response: 'Hi there!',
-      tokens: { prompt: 10, completion: 5 },
-      evaluations: [
-        {
-          type: 'contains',
-          passed: true,
-          score: 1,
-          reason: 'Contains expected value',
-        },
-      ],
+      tokens: { prompt: 10, completion: 5, total: 15 },
+      score: 1,
+      matcherType: 'contains',
+      expected: { type: 'contains', values: ['Hi'], mode: 'any' },
+      tags: [],
     },
     {
       id: 'case-2',
       name: 'Test Case 2',
       ok: false,
+      status: 'failed',
       latencyMs: 200,
       prompt: 'Goodbye',
       response: 'See you!',
-      tokens: { prompt: 8, completion: 4 },
-      evaluations: [
-        {
-          type: 'contains',
-          passed: false,
-          score: 0,
-          reason: 'Missing expected value',
-        },
-      ],
+      tokens: { prompt: 8, completion: 4, total: 12 },
+      score: 0,
+      matcherType: 'contains',
+      expected: { type: 'contains', values: ['Bye'], mode: 'any' },
+      tags: [],
     },
   ];
 
@@ -60,7 +55,7 @@ describe('createRunManifest', () => {
       endTime,
     });
 
-    expect(manifest.version).toBe('1.0');
+    expect(manifest.version).toBe('1.1');
     expect(manifest.project).toBe('test-project');
     expect(manifest.run_id).toBeTruthy();
     expect(manifest.run_id.length).toBe(12);
@@ -86,6 +81,10 @@ describe('createRunManifest', () => {
     expect(manifest.metrics.passed_cases).toBe(1);
     expect(manifest.metrics.failed_cases).toBe(1);
     expect(manifest.metrics.success_rate).toBe(0.5);
+    expect(manifest.metrics.total_attempts).toBe(2);
+    expect(manifest.metrics.valid_evaluations).toBe(2);
+    expect(manifest.metrics.invalid_evaluations).toBe(0);
+    expect(manifest.metrics.outcome_rate_denominator).toBe(2);
     expect(manifest.metrics.total_tokens).toBe(27); // (10+5) + (8+4)
     expect(manifest.metrics.total_prompt_tokens).toBe(18); // 10 + 8
     expect(manifest.metrics.total_completion_tokens).toBe(9); // 5 + 4
@@ -129,6 +128,10 @@ describe('createRunManifest', () => {
     expect(manifest.metrics.passed_cases).toBe(0);
     expect(manifest.metrics.failed_cases).toBe(0);
     expect(manifest.metrics.success_rate).toBe(0);
+    expect(manifest.metrics.total_attempts).toBe(0);
+    expect(manifest.metrics.valid_evaluations).toBe(0);
+    expect(manifest.metrics.invalid_evaluations).toBe(0);
+    expect(manifest.metrics.outcome_rate_denominator).toBe(0);
     expect(manifest.metrics.median_latency_ms).toBe(0);
     expect(manifest.metrics.p95_latency_ms).toBe(0);
   });
@@ -203,4 +206,72 @@ describe('createRunManifest', () => {
     expect(manifest.metrics.median_latency_ms).toBe(300);
     expect(manifest.metrics.p95_latency_ms).toBe(500);
   });
+
+  test('excludes invalid and target-error measurements from the success-rate denominator', () => {
+    const manifest = createRunManifest({
+      project: 'test-project',
+      config: { scenario: 'test-scenario', provider: 'openai' },
+      cases: [
+        { ...mockCases[0], attempts: 2 },
+        { ...mockCases[1], status: 'invalid', error: undefined },
+        { ...mockCases[1], id: 'case-3', status: 'error', error: 'provider unavailable' },
+      ],
+      startTime: new Date(),
+      endTime: new Date(),
+    });
+
+    expect(manifest.metrics).toMatchObject({
+      total_attempts: 4,
+      total_cases: 3,
+      valid_evaluations: 1,
+      invalid_evaluations: 2,
+      outcome_rate_denominator: 1,
+      passed_cases: 1,
+      failed_cases: 0,
+      success_rate: 1,
+    });
+  });
+
+  test('maps historical manifests without a status using the documented legacy mapping', () => {
+    const manifest = createRunManifest({
+      project: 'test-project',
+      config: { scenario: 'legacy', provider: 'openai' },
+      cases: [
+        { ...mockCases[0], status: undefined },
+        { ...mockCases[1], status: undefined, error: 'legacy execution failure' },
+      ],
+      startTime: new Date(),
+      endTime: new Date(),
+    });
+
+    expect(manifest.metrics).toMatchObject({
+      valid_evaluations: 1,
+      invalid_evaluations: 1,
+      outcome_rate_denominator: 1,
+      success_rate: 1,
+    });
+  });
+
+  test('accepts historical cases without integrity fields but rejects malformed new evidence', () => {
+    const historical = createRunManifest({
+      project: 'test-project',
+      config: { scenario: 'legacy', provider: 'openai' },
+      cases: [{ ...mockCases[0], status: undefined, evidence: undefined }],
+      startTime: new Date(),
+      endTime: new Date(),
+    });
+    expect(() => assertRunManifestIntegrity(historical)).not.toThrow();
+
+    const malformed = {
+      ...historical,
+      cases: [
+        {
+          ...mockCases[0],
+          status: 'passed',
+          evidence: { evaluator: 'custom', validation: { status: 'unknown' } },
+        },
+      ],
+    };
+    expect(() => assertRunManifestIntegrity(malformed)).toThrow('invalid evidence validation');
+  });
 });

package/src/artifacts/manifest.ts

@@ -15,6 +15,7 @@ import type {
   RunManifest,
   RunMetrics,
 } from './types';
+import { getCaseEvaluationStatus } from './types';
 
 /**
  * Create a new run manifest
@@ -49,7 +50,7 @@ export function createRunManifest(options: {
   const environment = getEnvironmentInfo();
 
   return {
-    version: '1.0',
+    version: '1.1',
     run_id: nanoid(12),
     project,
     start_time: startTime.toISOString(),
@@ -74,7 +75,11 @@ export function createRunManifest(options: {
  * Calculate metrics from case results
  */
 function calculateMetrics(cases: CaseResult[], model?: string): RunMetrics {
-  const passedCases = cases.filter((c) => c.ok);
+  const passedCases = cases.filter((c) => getCaseEvaluationStatus(c) === 'passed');
+  const validCases = cases.filter((c) => {
+    const status = getCaseEvaluationStatus(c);
+    return status === 'passed' || status === 'failed';
+  });
   const latencies = cases.map((c) => c.latencyMs).sort((a, b) => a - b);
 
   const medianLatency = latencies.length > 0 ? latencies[Math.floor(latencies.length / 2)] : 0;
@@ -87,7 +92,11 @@ function calculateMetrics(cases: CaseResult[], model?: string): RunMetrics {
 
   // Calculate cost if model is provided
   let cost: CostEstimateInfo | undefined;
-  if (model && (totalPromptTokens > 0 || totalCompletionTokens > 0)) {
+  if (
+    model &&
+    !model.toLowerCase().includes('ling-') &&
+    (totalPromptTokens > 0 || totalCompletionTokens > 0)
+  ) {
     const costEstimate = estimateCost(totalPromptTokens, totalCompletionTokens, model);
     const pricing = getModelPricing(model);
     cost = {
@@ -103,10 +112,14 @@ function calculateMetrics(cases: CaseResult[], model?: string): RunMetrics {
   }
 
   return {
-    success_rate: cases.length > 0 ? passedCases.length / cases.length : 0,
+    success_rate: validCases.length > 0 ? passedCases.length / validCases.length : 0,
+    total_attempts: cases.reduce((sum, c) => sum + (c.attempts ?? 1), 0),
     total_cases: cases.length,
+    valid_evaluations: validCases.length,
+    invalid_evaluations: cases.length - validCases.length,
+    outcome_rate_denominator: validCases.length,
     passed_cases: passedCases.length,
-    failed_cases: cases.length - passedCases.length,
+    failed_cases: validCases.length - passedCases.length,
     median_latency_ms: medianLatency,
     p95_latency_ms: p95Latency,
     total_tokens: totalPromptTokens + totalCompletionTokens,

package/src/artifacts/types.ts

@@ -16,6 +16,8 @@ export interface CaseRedactionInfo {
   promptRedacted: boolean;
   /** Whether response was redacted */
   responseRedacted: boolean;
+  /** Whether evaluator reason text was redacted */
+  reasonRedacted?: boolean;
   /** Number of redactions in this case */
   redactionCount: number;
 }
@@ -34,6 +36,8 @@ export interface ManifestRedactionInfo {
   summary: {
     promptsRedacted: number;
     responsesRedacted: number;
+    /** Cases whose evaluator reason text was redacted. */
+    reasonsRedacted?: number;
     totalRedactions: number;
   };
 }
@@ -42,6 +46,26 @@ export interface ManifestRedactionInfo {
 // Case Result Types
 // ============================================================================
 
+/**
+ * Terminal status of a case measurement.
+ *
+ * `invalid` means a target response was available but could not be evaluated
+ * reliably. `error` means execution did not produce a usable target response.
+ */
+export type CaseEvaluationStatus = 'passed' | 'failed' | 'invalid' | 'error';
+
+/** Reviewed, bounded evaluator evidence retained in a run artifact. */
+export interface CaseEvaluationEvidence {
+  evaluator: string;
+  score?: number;
+  threshold?: number;
+  model?: string;
+  validation?: {
+    status: 'valid' | 'invalid';
+    code?: string;
+  };
+}
+
 /**
  * Individual test case result
  */
@@ -49,6 +73,10 @@ export interface CaseResult {
   id: string;
   name?: string;
   ok: boolean;
+  /** Present in manifest v1.1+. Missing values use the documented legacy mapping. */
+  status?: CaseEvaluationStatus;
+  /** Number of execution attempts represented by this terminal result. */
+  attempts?: number;
   score: number;
   matcherType: string;
   reason?: string;
@@ -63,8 +91,14 @@ export interface CaseResult {
   expected: object;
   tags: string[];
   error?: string;
+  /** Sanitized evaluator evidence; arbitrary evaluator details are never stored here. */
+  evidence?: CaseEvaluationEvidence;
   /** Redaction information for this case */
   redaction?: CaseRedactionInfo;
+  /** Ordered tool activity captured for an enabled tool loop. */
+  toolTrace?: import('../tools').ToolTraceEntry[];
+  /** Terminal status for an enabled tool loop. */
+  toolLoop?: import('../tools').ToolLoopSummary;
 }
 
 /**
@@ -91,7 +125,15 @@ export interface CostEstimateInfo {
  */
 export interface RunMetrics {
   success_rate: number;
+  /** Number of execution attempts, including retries. */
+  total_attempts?: number;
   total_cases: number;
+  /** Case results with a valid evaluator outcome (passed or failed). */
+  valid_evaluations?: number;
+  /** Case results excluded from outcome rates (invalid or error). */
+  invalid_evaluations?: number;
+  /** Denominator used for success_rate; zero produces a success_rate of zero. */
+  outcome_rate_denominator?: number;
   passed_cases: number;
   failed_cases: number;
   median_latency_ms: number;
@@ -240,6 +282,97 @@ export interface RunManifest {
   redaction?: ManifestRedactionInfo;
 }
 
+/**
+ * Read a case status from both v1.1 artifacts and historical v1.0 artifacts.
+ * Historical records cannot distinguish evaluator failures from ordinary failed
+ * criteria unless they set the legacy `error` field.
+ */
+export function getCaseEvaluationStatus(caseResult: CaseResult): CaseEvaluationStatus {
+  if (
+    caseResult.status === 'passed' ||
+    caseResult.status === 'failed' ||
+    caseResult.status === 'invalid' ||
+    caseResult.status === 'error'
+  ) {
+    return caseResult.status;
+  }
+  if (caseResult.ok) return 'passed';
+  return caseResult.error ? 'error' : 'failed';
+}
+
+/**
+ * Reject untrusted integrity-bearing fields before a run manifest is persisted
+ * or treated as a standard run. Historical manifests remain supported because
+ * status and evidence are optional in the v1.0 contract.
+ */
+export function assertRunManifestIntegrity(manifest: unknown): asserts manifest is RunManifest {
+  if (!isRecord(manifest) || !Array.isArray(manifest.cases)) {
+    throw new Error('Invalid run manifest: expected an object with a cases array');
+  }
+
+  for (const [index, caseResult] of manifest.cases.entries()) {
+    if (!isRecord(caseResult)) {
+      throw new Error(`Invalid run manifest: case ${index} is not an object`);
+    }
+
+    if (
+      caseResult.status !== undefined &&
+      caseResult.status !== 'passed' &&
+      caseResult.status !== 'failed' &&
+      caseResult.status !== 'invalid' &&
+      caseResult.status !== 'error'
+    ) {
+      throw new Error(`Invalid run manifest: case ${index} has an unknown status`);
+    }
+
+    if (caseResult.evidence !== undefined) {
+      assertCaseEvaluationEvidence(caseResult.evidence, index);
+    }
+  }
+}
+
+function assertCaseEvaluationEvidence(evidence: unknown, caseIndex: number): void {
+  if (
+    !isRecord(evidence) ||
+    typeof evidence.evaluator !== 'string' ||
+    evidence.evaluator.length > 100
+  ) {
+    throw new Error(`Invalid run manifest: case ${caseIndex} has malformed evaluator evidence`);
+  }
+
+  if (evidence.score !== undefined && !isUnitIntervalNumber(evidence.score)) {
+    throw new Error(`Invalid run manifest: case ${caseIndex} has an invalid evidence score`);
+  }
+  if (evidence.threshold !== undefined && !isUnitIntervalNumber(evidence.threshold)) {
+    throw new Error(`Invalid run manifest: case ${caseIndex} has an invalid evidence threshold`);
+  }
+  if (
+    evidence.model !== undefined &&
+    (typeof evidence.model !== 'string' || evidence.model.length > 200)
+  ) {
+    throw new Error(`Invalid run manifest: case ${caseIndex} has an invalid evidence model`);
+  }
+
+  if (evidence.validation !== undefined) {
+    if (
+      !isRecord(evidence.validation) ||
+      (evidence.validation.status !== 'valid' && evidence.validation.status !== 'invalid') ||
+      (evidence.validation.code !== undefined &&
+        (typeof evidence.validation.code !== 'string' || evidence.validation.code.length > 100))
+    ) {
+      throw new Error(`Invalid run manifest: case ${caseIndex} has invalid evidence validation`);
+    }
+  }
+}
+
+function isUnitIntervalNumber(value: unknown): value is number {
+  return typeof value === 'number' && Number.isFinite(value) && value >= 0 && value <= 1;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+
 // ============================================================================
 // Red Team Manifest Types
 // ============================================================================

package/src/evaluators/index.ts

@@ -12,6 +12,7 @@ import { LLMGraderEvaluator } from './llm-grader';
 import { NotContainsEvaluator } from './not-contains';
 import { RegexEvaluator } from './regex';
 import { SimilarityEvaluator } from './similarity';
+import { ToolTraceEvaluator } from './tool-trace';
 import type { Evaluator } from './types';
 
 const evaluators = new Map<string, Evaluator>();
@@ -25,6 +26,7 @@ evaluators.set('json_schema', new JsonSchemaEvaluator());
 evaluators.set('llm_grader', new LLMGraderEvaluator());
 evaluators.set('similarity', new SimilarityEvaluator());
 evaluators.set('inline', new InlineEvaluator());
+evaluators.set('tool_trace', new ToolTraceEvaluator());
 
 /**
  * Get an evaluator by type
@@ -63,3 +65,4 @@ export { JsonSchemaEvaluator } from './json-schema';
 export { LLMGraderEvaluator } from './llm-grader';
 export { SimilarityEvaluator } from './similarity';
 export { InlineEvaluator, SUPPORTED_EXPRESSIONS } from './inline';
+export { ToolTraceEvaluator } from './tool-trace';

package/src/evaluators/json-schema.test.ts

@@ -0,0 +1,130 @@
+import { describe, expect, it } from 'bun:test';
+import type { Expected } from '../scenario/schema';
+import { JsonSchemaEvaluator } from './json-schema';
+
+const expected: Expected = {
+  type: 'json_schema',
+  schema: {
+    type: 'object',
+    required: ['profile', 'status', 'version'],
+    additionalProperties: false,
+    properties: {
+      profile: {
+        type: 'object',
+        required: ['roles'],
+        additionalProperties: false,
+        properties: {
+          roles: {
+            type: 'array',
+            items: { type: 'string', enum: ['admin', 'member'] },
+          },
+        },
+      },
+      status: { enum: ['active', 'disabled'] },
+      version: { const: 'v1' },
+    },
+  },
+};
+
+describe('JsonSchemaEvaluator', () => {
+  const evaluator = new JsonSchemaEvaluator();
+
+  it('accepts a response matching a nested JSON schema', async () => {
+    const result = await evaluator.evaluate(
+      JSON.stringify({ profile: { roles: ['admin'] }, status: 'active', version: 'v1' }),
+      expected
+    );
+
+    expect(result.passed).toBe(true);
+    expect(result.score).toBe(1);
+  });
+
+  it('rejects a missing nested required property', async () => {
+    const result = await evaluator.evaluate(
+      JSON.stringify({ profile: {}, status: 'active', version: 'v1' }),
+      expected
+    );
+
+    expect(result.passed).toBe(false);
+    expect(result.score).toBe(0);
+  });
+
+  it('rejects values outside enum constraints', async () => {
+    const result = await evaluator.evaluate(
+      JSON.stringify({ profile: { roles: ['owner'] }, status: 'pending', version: 'v1' }),
+      expected
+    );
+
+    expect(result.passed).toBe(false);
+    expect(result.reason).toContain('allowed values');
+  });
+
+  it('rejects values that violate const constraints', async () => {
+    const result = await evaluator.evaluate(
+      JSON.stringify({ profile: { roles: ['member'] }, status: 'active', version: 'v2' }),
+      expected
+    );
+
+    expect(result.passed).toBe(false);
+    expect(result.reason).toContain('constant');
+  });
+
+  it('rejects additional properties in closed objects', async () => {
+    const result = await evaluator.evaluate(
+      JSON.stringify({
+        profile: { roles: ['admin'], internal: true },
+        status: 'active',
+        version: 'v1',
+      }),
+      expected
+    );
+
+    expect(result.passed).toBe(false);
+    expect(result.reason).toContain('additional properties');
+  });
+
+  it('reports invalid schemas without exposing response content', async () => {
+    const sensitiveResponse = 'sensitive-response-value';
+    const result = await evaluator.evaluate(sensitiveResponse, {
+      type: 'json_schema',
+      schema: { type: 'not-a-json-schema-type' },
+    });
+
+    expect(result).toEqual({
+      passed: false,
+      score: 0,
+      reason: 'Invalid JSON schema',
+      details: { error: 'Invalid JSON schema' },
+    });
+    expect(JSON.stringify(result)).not.toContain('sensitive-response-value');
+  });
+
+  it('validates distinct schema objects with the same $id independently', async () => {
+    const firstExpected: Expected = {
+      type: 'json_schema',
+      schema: {
+        $id: 'https://schemas.example.test/shared-result',
+        type: 'object',
+        required: ['first'],
+        properties: { first: { const: true } },
+      },
+    };
+    const secondExpected: Expected = {
+      type: 'json_schema',
+      schema: {
+        $id: 'https://schemas.example.test/shared-result',
+        type: 'object',
+        required: ['second'],
+        properties: { second: { const: true } },
+      },
+    };
+
+    const firstResult = await evaluator.evaluate('{"first":true}', firstExpected);
+    const cachedResult = await evaluator.evaluate('{"first":true}', firstExpected);
+    const distinctResult = await evaluator.evaluate('{"second":true}', secondExpected);
+
+    expect(firstResult.passed).toBe(true);
+    expect(cachedResult.passed).toBe(true);
+    expect(distinctResult.passed).toBe(true);
+  });
+});

package/src/evaluators/json-schema.ts

@@ -2,10 +2,21 @@
  * JSON Schema evaluator - validates response against a JSON schema
  */
 
-import { z } from 'zod';
+import Ajv, { type ValidateFunction } from 'ajv';
 import type { Expected } from '../scenario/schema';
 import type { Evaluator, EvaluatorResult } from './types';
 
+const validators = new WeakMap<object, ValidateFunction>();
+
+function getValidator(schema: Record<string, unknown>): ValidateFunction {
+  const cached = validators.get(schema);
+  if (cached) return cached;
+
+  const validator = new Ajv({ allErrors: true, strict: false }).compile(schema);
+  validators.set(schema, validator);
+  return validator;
+}
+
 export class JsonSchemaEvaluator implements Evaluator {
   readonly type = 'json_schema';
 
@@ -14,6 +25,18 @@ export class JsonSchemaEvaluator implements Evaluator {
       throw new Error('Invalid expected type for JsonSchemaEvaluator');
     }
 
+    let validator: ValidateFunction;
+    try {
+      validator = getValidator(expected.schema);
+    } catch {
+      return {
+        passed: false,
+        score: 0,
+        reason: 'Invalid JSON schema',
+        details: { error: 'Invalid JSON schema' },
+      };
+    }
+
     let parsed: unknown;
     try {
       const jsonMatch = response.match(/```(?:json)?\s*([\s\S]*?)```/);
@@ -28,71 +51,23 @@ export class JsonSchemaEvaluator implements Evaluator {
       };
     }
 
-    try {
-      const zodSchema = this.jsonSchemaToZod(expected.schema);
-      const result = zodSchema.safeParse(parsed);
-
-      if (result.success) {
-        return {
-          passed: true,
-          score: 1,
-          reason: 'Response matches JSON schema',
-          details: { parsed },
-        };
-      }
-      const issues = result.error.issues.map((i) => `${i.path.join('.')}: ${i.message}`);
-      return {
-        passed: false,
-        score: 0,
-        reason: `Schema validation failed: ${issues.join(', ')}`,
-        details: {
-          parsed,
-          errors: issues,
-        },
-      };
-    } catch (error) {
+    if (validator(parsed)) {
       return {
-        passed: false,
-        score: 0,
-        reason: `Schema error: ${(error as Error).message}`,
-        details: { error: (error as Error).message },
+        passed: true,
+        score: 1,
+        reason: 'Response matches JSON schema',
+        details: { parsed },
       };
     }
-  }
 
-  private jsonSchemaToZod(schema: Record<string, unknown>): z.ZodTypeAny {
-    const type = schema.type as string;
-
-    switch (type) {
-      case 'string':
-        return z.string();
-      case 'number':
-        return z.number();
-      case 'integer':
-        return z.number().int();
-      case 'boolean':
-        return z.boolean();
-      case 'null':
-        return z.null();
-      case 'array':
-        if (schema.items) {
-          return z.array(this.jsonSchemaToZod(schema.items as Record<string, unknown>));
-        }
-        return z.array(z.unknown());
-      case 'object':
-        if (schema.properties) {
-          const shape: Record<string, z.ZodTypeAny> = {};
-          const required = (schema.required as string[]) || [];
-
-          for (const [key, value] of Object.entries(schema.properties as Record<string, unknown>)) {
-            const fieldSchema = this.jsonSchemaToZod(value as Record<string, unknown>);
-            shape[key] = required.includes(key) ? fieldSchema : fieldSchema.optional();
-          }
-          return z.object(shape);
-        }
-        return z.record(z.unknown());
-      default:
-        return z.unknown();
-    }
+    const errors = (validator.errors ?? []).map(
+      (error) => `${error.instancePath || '/'} ${error.message ?? 'is invalid'}`
+    );
+    return {
+      passed: false,
+      score: 0,
+      reason: `Schema validation failed: ${errors.join('; ')}`,
+      details: { parsed, errors },
+    };
   }
 }