@artemiskit/core 0.3.0 → 0.4.0

package/src/artifacts/manifest.test.ts

@@ -12,35 +12,29 @@ describe('createRunManifest', () => {
       id: 'case-1',
       name: 'Test Case 1',
       ok: true,
+      status: 'passed',
       latencyMs: 100,
       prompt: 'Hello',
       response: 'Hi there!',
-      tokens: { prompt: 10, completion: 5 },
-      evaluations: [
-        {
-          type: 'contains',
-          passed: true,
-          score: 1,
-          reason: 'Contains expected value',
-        },
-      ],
+      tokens: { prompt: 10, completion: 5, total: 15 },
+      score: 1,
+      matcherType: 'contains',
+      expected: { type: 'contains', values: ['Hi'], mode: 'any' },
+      tags: [],
     },
     {
       id: 'case-2',
       name: 'Test Case 2',
       ok: false,
+      status: 'failed',
       latencyMs: 200,
       prompt: 'Goodbye',
       response: 'See you!',
-      tokens: { prompt: 8, completion: 4 },
-      evaluations: [
-        {
-          type: 'contains',
-          passed: false,
-          score: 0,
-          reason: 'Missing expected value',
-        },
-      ],
+      tokens: { prompt: 8, completion: 4, total: 12 },
+      score: 0,
+      matcherType: 'contains',
+      expected: { type: 'contains', values: ['Bye'], mode: 'any' },
+      tags: [],
     },
   ];
 
@@ -60,7 +54,7 @@ describe('createRunManifest', () => {
       endTime,
     });
 
-    expect(manifest.version).toBe('1.0');
+    expect(manifest.version).toBe('1.1');
     expect(manifest.project).toBe('test-project');
     expect(manifest.run_id).toBeTruthy();
     expect(manifest.run_id.length).toBe(12);
@@ -86,6 +80,10 @@ describe('createRunManifest', () => {
     expect(manifest.metrics.passed_cases).toBe(1);
     expect(manifest.metrics.failed_cases).toBe(1);
     expect(manifest.metrics.success_rate).toBe(0.5);
+    expect(manifest.metrics.total_attempts).toBe(2);
+    expect(manifest.metrics.valid_evaluations).toBe(2);
+    expect(manifest.metrics.invalid_evaluations).toBe(0);
+    expect(manifest.metrics.outcome_rate_denominator).toBe(2);
     expect(manifest.metrics.total_tokens).toBe(27); // (10+5) + (8+4)
     expect(manifest.metrics.total_prompt_tokens).toBe(18); // 10 + 8
     expect(manifest.metrics.total_completion_tokens).toBe(9); // 5 + 4
@@ -129,6 +127,10 @@ describe('createRunManifest', () => {
     expect(manifest.metrics.passed_cases).toBe(0);
     expect(manifest.metrics.failed_cases).toBe(0);
     expect(manifest.metrics.success_rate).toBe(0);
+    expect(manifest.metrics.total_attempts).toBe(0);
+    expect(manifest.metrics.valid_evaluations).toBe(0);
+    expect(manifest.metrics.invalid_evaluations).toBe(0);
+    expect(manifest.metrics.outcome_rate_denominator).toBe(0);
     expect(manifest.metrics.median_latency_ms).toBe(0);
     expect(manifest.metrics.p95_latency_ms).toBe(0);
   });
@@ -203,4 +205,49 @@ describe('createRunManifest', () => {
     expect(manifest.metrics.median_latency_ms).toBe(300);
     expect(manifest.metrics.p95_latency_ms).toBe(500);
   });
+
+  test('excludes invalid and target-error measurements from the success-rate denominator', () => {
+    const manifest = createRunManifest({
+      project: 'test-project',
+      config: { scenario: 'test-scenario', provider: 'openai' },
+      cases: [
+        { ...mockCases[0], attempts: 2 },
+        { ...mockCases[1], status: 'invalid', error: undefined },
+        { ...mockCases[1], id: 'case-3', status: 'error', error: 'provider unavailable' },
+      ],
+      startTime: new Date(),
+      endTime: new Date(),
+    });
+
+    expect(manifest.metrics).toMatchObject({
+      total_attempts: 4,
+      total_cases: 3,
+      valid_evaluations: 1,
+      invalid_evaluations: 2,
+      outcome_rate_denominator: 1,
+      passed_cases: 1,
+      failed_cases: 0,
+      success_rate: 1,
+    });
+  });
+
+  test('maps historical manifests without a status using the documented legacy mapping', () => {
+    const manifest = createRunManifest({
+      project: 'test-project',
+      config: { scenario: 'legacy', provider: 'openai' },
+      cases: [
+        { ...mockCases[0], status: undefined },
+        { ...mockCases[1], status: undefined, error: 'legacy execution failure' },
+      ],
+      startTime: new Date(),
+      endTime: new Date(),
+    });
+
+    expect(manifest.metrics).toMatchObject({
+      valid_evaluations: 1,
+      invalid_evaluations: 1,
+      outcome_rate_denominator: 1,
+      success_rate: 1,
+    });
+  });
 });

package/src/artifacts/manifest.ts

@@ -15,6 +15,7 @@ import type {
   RunManifest,
   RunMetrics,
 } from './types';
+import { getCaseEvaluationStatus } from './types';
 
 /**
  * Create a new run manifest
@@ -49,7 +50,7 @@ export function createRunManifest(options: {
   const environment = getEnvironmentInfo();
 
   return {
-    version: '1.0',
+    version: '1.1',
     run_id: nanoid(12),
     project,
     start_time: startTime.toISOString(),
@@ -74,7 +75,11 @@ export function createRunManifest(options: {
  * Calculate metrics from case results
  */
 function calculateMetrics(cases: CaseResult[], model?: string): RunMetrics {
-  const passedCases = cases.filter((c) => c.ok);
+  const passedCases = cases.filter((c) => getCaseEvaluationStatus(c) === 'passed');
+  const validCases = cases.filter((c) => {
+    const status = getCaseEvaluationStatus(c);
+    return status === 'passed' || status === 'failed';
+  });
   const latencies = cases.map((c) => c.latencyMs).sort((a, b) => a - b);
 
   const medianLatency = latencies.length > 0 ? latencies[Math.floor(latencies.length / 2)] : 0;
@@ -87,7 +92,11 @@ function calculateMetrics(cases: CaseResult[], model?: string): RunMetrics {
 
   // Calculate cost if model is provided
   let cost: CostEstimateInfo | undefined;
-  if (model && (totalPromptTokens > 0 || totalCompletionTokens > 0)) {
+  if (
+    model &&
+    !model.toLowerCase().includes('ling-') &&
+    (totalPromptTokens > 0 || totalCompletionTokens > 0)
+  ) {
     const costEstimate = estimateCost(totalPromptTokens, totalCompletionTokens, model);
     const pricing = getModelPricing(model);
     cost = {
@@ -103,10 +112,14 @@ function calculateMetrics(cases: CaseResult[], model?: string): RunMetrics {
   }
 
   return {
-    success_rate: cases.length > 0 ? passedCases.length / cases.length : 0,
+    success_rate: validCases.length > 0 ? passedCases.length / validCases.length : 0,
+    total_attempts: cases.reduce((sum, c) => sum + (c.attempts ?? 1), 0),
     total_cases: cases.length,
+    valid_evaluations: validCases.length,
+    invalid_evaluations: cases.length - validCases.length,
+    outcome_rate_denominator: validCases.length,
     passed_cases: passedCases.length,
-    failed_cases: cases.length - passedCases.length,
+    failed_cases: validCases.length - passedCases.length,
     median_latency_ms: medianLatency,
     p95_latency_ms: p95Latency,
     total_tokens: totalPromptTokens + totalCompletionTokens,

package/src/artifacts/types.ts

@@ -42,6 +42,26 @@ export interface ManifestRedactionInfo {
 // Case Result Types
 // ============================================================================
 
+/**
+ * Terminal status of a case measurement.
+ *
+ * `invalid` means a target response was available but could not be evaluated
+ * reliably. `error` means execution did not produce a usable target response.
+ */
+export type CaseEvaluationStatus = 'passed' | 'failed' | 'invalid' | 'error';
+
+/** Reviewed, bounded evaluator evidence retained in a run artifact. */
+export interface CaseEvaluationEvidence {
+  evaluator: string;
+  score?: number;
+  threshold?: number;
+  model?: string;
+  validation?: {
+    status: 'valid' | 'invalid';
+    code?: string;
+  };
+}
+
 /**
  * Individual test case result
  */
@@ -49,6 +69,10 @@ export interface CaseResult {
   id: string;
   name?: string;
   ok: boolean;
+  /** Present in manifest v1.1+. Missing values use the documented legacy mapping. */
+  status?: CaseEvaluationStatus;
+  /** Number of execution attempts represented by this terminal result. */
+  attempts?: number;
   score: number;
   matcherType: string;
   reason?: string;
@@ -63,8 +87,14 @@ export interface CaseResult {
   expected: object;
   tags: string[];
   error?: string;
+  /** Sanitized evaluator evidence; arbitrary evaluator details are never stored here. */
+  evidence?: CaseEvaluationEvidence;
   /** Redaction information for this case */
   redaction?: CaseRedactionInfo;
+  /** Ordered tool activity captured for an enabled tool loop. */
+  toolTrace?: import('../tools').ToolTraceEntry[];
+  /** Terminal status for an enabled tool loop. */
+  toolLoop?: import('../tools').ToolLoopSummary;
 }
 
 /**
@@ -91,7 +121,15 @@ export interface CostEstimateInfo {
  */
 export interface RunMetrics {
   success_rate: number;
+  /** Number of execution attempts, including retries. */
+  total_attempts?: number;
   total_cases: number;
+  /** Case results with a valid evaluator outcome (passed or failed). */
+  valid_evaluations?: number;
+  /** Case results excluded from outcome rates (invalid or error). */
+  invalid_evaluations?: number;
+  /** Denominator used for success_rate; zero produces a success_rate of zero. */
+  outcome_rate_denominator?: number;
   passed_cases: number;
   failed_cases: number;
   median_latency_ms: number;
@@ -240,6 +278,17 @@ export interface RunManifest {
   redaction?: ManifestRedactionInfo;
 }
 
+/**
+ * Read a case status from both v1.1 artifacts and historical v1.0 artifacts.
+ * Historical records cannot distinguish evaluator failures from ordinary failed
+ * criteria unless they set the legacy `error` field.
+ */
+export function getCaseEvaluationStatus(caseResult: CaseResult): CaseEvaluationStatus {
+  if (caseResult.status) return caseResult.status;
+  if (caseResult.ok) return 'passed';
+  return caseResult.error ? 'error' : 'failed';
+}
+
 // ============================================================================
 // Red Team Manifest Types
 // ============================================================================

package/src/evaluators/index.ts

@@ -12,6 +12,7 @@ import { LLMGraderEvaluator } from './llm-grader';
 import { NotContainsEvaluator } from './not-contains';
 import { RegexEvaluator } from './regex';
 import { SimilarityEvaluator } from './similarity';
+import { ToolTraceEvaluator } from './tool-trace';
 import type { Evaluator } from './types';
 
 const evaluators = new Map<string, Evaluator>();
@@ -25,6 +26,7 @@ evaluators.set('json_schema', new JsonSchemaEvaluator());
 evaluators.set('llm_grader', new LLMGraderEvaluator());
 evaluators.set('similarity', new SimilarityEvaluator());
 evaluators.set('inline', new InlineEvaluator());
+evaluators.set('tool_trace', new ToolTraceEvaluator());
 
 /**
  * Get an evaluator by type
@@ -63,3 +65,4 @@ export { JsonSchemaEvaluator } from './json-schema';
 export { LLMGraderEvaluator } from './llm-grader';
 export { SimilarityEvaluator } from './similarity';
 export { InlineEvaluator, SUPPORTED_EXPRESSIONS } from './inline';
+export { ToolTraceEvaluator } from './tool-trace';

package/src/evaluators/json-schema.test.ts

@@ -0,0 +1,130 @@
+import { describe, expect, it } from 'bun:test';
+import type { Expected } from '../scenario/schema';
+import { JsonSchemaEvaluator } from './json-schema';
+
+const expected: Expected = {
+  type: 'json_schema',
+  schema: {
+    type: 'object',
+    required: ['profile', 'status', 'version'],
+    additionalProperties: false,
+    properties: {
+      profile: {
+        type: 'object',
+        required: ['roles'],
+        additionalProperties: false,
+        properties: {
+          roles: {
+            type: 'array',
+            items: { type: 'string', enum: ['admin', 'member'] },
+          },
+        },
+      },
+      status: { enum: ['active', 'disabled'] },
+      version: { const: 'v1' },
+    },
+  },
+};
+
+describe('JsonSchemaEvaluator', () => {
+  const evaluator = new JsonSchemaEvaluator();
+
+  it('accepts a response matching a nested JSON schema', async () => {
+    const result = await evaluator.evaluate(
+      JSON.stringify({ profile: { roles: ['admin'] }, status: 'active', version: 'v1' }),
+      expected
+    );
+
+    expect(result.passed).toBe(true);
+    expect(result.score).toBe(1);
+  });
+
+  it('rejects a missing nested required property', async () => {
+    const result = await evaluator.evaluate(
+      JSON.stringify({ profile: {}, status: 'active', version: 'v1' }),
+      expected
+    );
+
+    expect(result.passed).toBe(false);
+    expect(result.score).toBe(0);
+  });
+
+  it('rejects values outside enum constraints', async () => {
+    const result = await evaluator.evaluate(
+      JSON.stringify({ profile: { roles: ['owner'] }, status: 'pending', version: 'v1' }),
+      expected
+    );
+
+    expect(result.passed).toBe(false);
+    expect(result.reason).toContain('allowed values');
+  });
+
+  it('rejects values that violate const constraints', async () => {
+    const result = await evaluator.evaluate(
+      JSON.stringify({ profile: { roles: ['member'] }, status: 'active', version: 'v2' }),
+      expected
+    );
+
+    expect(result.passed).toBe(false);
+    expect(result.reason).toContain('constant');
+  });
+
+  it('rejects additional properties in closed objects', async () => {
+    const result = await evaluator.evaluate(
+      JSON.stringify({
+        profile: { roles: ['admin'], internal: true },
+        status: 'active',
+        version: 'v1',
+      }),
+      expected
+    );
+
+    expect(result.passed).toBe(false);
+    expect(result.reason).toContain('additional properties');
+  });
+
+  it('reports invalid schemas without exposing response content', async () => {
+    const sensitiveResponse = 'sensitive-response-value';
+    const result = await evaluator.evaluate(sensitiveResponse, {
+      type: 'json_schema',
+      schema: { type: 'not-a-json-schema-type' },
+    });
+
+    expect(result).toEqual({
+      passed: false,
+      score: 0,
+      reason: 'Invalid JSON schema',
+      details: { error: 'Invalid JSON schema' },
+    });
+    expect(JSON.stringify(result)).not.toContain('sensitive-response-value');
+  });
+
+  it('validates distinct schema objects with the same $id independently', async () => {
+    const firstExpected: Expected = {
+      type: 'json_schema',
+      schema: {
+        $id: 'https://schemas.example.test/shared-result',
+        type: 'object',
+        required: ['first'],
+        properties: { first: { const: true } },
+      },
+    };
+    const secondExpected: Expected = {
+      type: 'json_schema',
+      schema: {
+        $id: 'https://schemas.example.test/shared-result',
+        type: 'object',
+        required: ['second'],
+        properties: { second: { const: true } },
+      },
+    };
+
+    const firstResult = await evaluator.evaluate('{"first":true}', firstExpected);
+    const cachedResult = await evaluator.evaluate('{"first":true}', firstExpected);
+    const distinctResult = await evaluator.evaluate('{"second":true}', secondExpected);
+
+    expect(firstResult.passed).toBe(true);
+    expect(cachedResult.passed).toBe(true);
+    expect(distinctResult.passed).toBe(true);
+  });
+});

package/src/evaluators/json-schema.ts

@@ -2,10 +2,21 @@
  * JSON Schema evaluator - validates response against a JSON schema
  */
 
-import { z } from 'zod';
+import Ajv, { type ValidateFunction } from 'ajv';
 import type { Expected } from '../scenario/schema';
 import type { Evaluator, EvaluatorResult } from './types';
 
+const validators = new WeakMap<object, ValidateFunction>();
+
+function getValidator(schema: Record<string, unknown>): ValidateFunction {
+  const cached = validators.get(schema);
+  if (cached) return cached;
+
+  const validator = new Ajv({ allErrors: true, strict: false }).compile(schema);
+  validators.set(schema, validator);
+  return validator;
+}
+
 export class JsonSchemaEvaluator implements Evaluator {
   readonly type = 'json_schema';
 
@@ -14,6 +25,18 @@ export class JsonSchemaEvaluator implements Evaluator {
       throw new Error('Invalid expected type for JsonSchemaEvaluator');
     }
 
+    let validator: ValidateFunction;
+    try {
+      validator = getValidator(expected.schema);
+    } catch {
+      return {
+        passed: false,
+        score: 0,
+        reason: 'Invalid JSON schema',
+        details: { error: 'Invalid JSON schema' },
+      };
+    }
+
     let parsed: unknown;
     try {
       const jsonMatch = response.match(/```(?:json)?\s*([\s\S]*?)```/);
@@ -28,71 +51,23 @@ export class JsonSchemaEvaluator implements Evaluator {
       };
     }
 
-    try {
-      const zodSchema = this.jsonSchemaToZod(expected.schema);
-      const result = zodSchema.safeParse(parsed);
-
-      if (result.success) {
-        return {
-          passed: true,
-          score: 1,
-          reason: 'Response matches JSON schema',
-          details: { parsed },
-        };
-      }
-      const issues = result.error.issues.map((i) => `${i.path.join('.')}: ${i.message}`);
-      return {
-        passed: false,
-        score: 0,
-        reason: `Schema validation failed: ${issues.join(', ')}`,
-        details: {
-          parsed,
-          errors: issues,
-        },
-      };
-    } catch (error) {
+    if (validator(parsed)) {
       return {
-        passed: false,
-        score: 0,
-        reason: `Schema error: ${(error as Error).message}`,
-        details: { error: (error as Error).message },
+        passed: true,
+        score: 1,
+        reason: 'Response matches JSON schema',
+        details: { parsed },
       };
     }
-  }
 
-  private jsonSchemaToZod(schema: Record<string, unknown>): z.ZodTypeAny {
-    const type = schema.type as string;
-
-    switch (type) {
-      case 'string':
-        return z.string();
-      case 'number':
-        return z.number();
-      case 'integer':
-        return z.number().int();
-      case 'boolean':
-        return z.boolean();
-      case 'null':
-        return z.null();
-      case 'array':
-        if (schema.items) {
-          return z.array(this.jsonSchemaToZod(schema.items as Record<string, unknown>));
-        }
-        return z.array(z.unknown());
-      case 'object':
-        if (schema.properties) {
-          const shape: Record<string, z.ZodTypeAny> = {};
-          const required = (schema.required as string[]) || [];
-
-          for (const [key, value] of Object.entries(schema.properties as Record<string, unknown>)) {
-            const fieldSchema = this.jsonSchemaToZod(value as Record<string, unknown>);
-            shape[key] = required.includes(key) ? fieldSchema : fieldSchema.optional();
-          }
-          return z.object(shape);
-        }
-        return z.record(z.unknown());
-      default:
-        return z.unknown();
-    }
+    const errors = (validator.errors ?? []).map(
+      (error) => `${error.instancePath || '/'} ${error.message ?? 'is invalid'}`
+    );
+    return {
+      passed: false,
+      score: 0,
+      reason: `Schema validation failed: ${errors.join('; ')}`,
+      details: { parsed, errors },
+    };
   }
 }

package/src/evaluators/llm-grader.test.ts

@@ -0,0 +1,80 @@
+import { describe, expect, test } from 'bun:test';
+import type { ModelClient } from '../adapters/types';
+import { ScenarioSchema } from '../scenario/schema';
+import { LLMGraderEvaluator } from './llm-grader';
+
+function graderCase(strict: boolean) {
+  return ScenarioSchema.parse({
+    name: 'grader parsing',
+    cases: [
+      {
+        id: 'grade',
+        prompt: 'grade this',
+        expected: { type: 'llm_grader', rubric: 'Be correct', threshold: 0.7, strict },
+      },
+    ],
+  }).cases[0].expected;
+}
+
+function clientWith(text: string): ModelClient {
+  return {
+    provider: 'test',
+    generate: async () => ({
+      id: 'judge-response',
+      model: 'judge-model',
+      text,
+      tokens: { prompt: 1, completion: 1, total: 2 },
+      latencyMs: 1,
+      finishReason: 'stop',
+    }),
+    capabilities: async () => ({
+      streaming: false,
+      functionCalling: false,
+      toolUse: false,
+      maxContext: 1,
+    }),
+  };
+}
+
+describe('LLMGraderEvaluator strict parsing', () => {
+  test('accepts finite numeric boundary scores in exact JSON', async () => {
+    const evaluator = new LLMGraderEvaluator();
+    const passing = await evaluator.evaluate('response', graderCase(true), {
+      client: clientWith('{"score":1,"reason":"complete"}'),
+    });
+    const failing = await evaluator.evaluate('response', graderCase(true), {
+      client: clientWith('{"score":0,"reason":"missing requirement"}'),
+    });
+
+    expect(passing).toMatchObject({ passed: true, score: 1, status: 'passed' });
+    expect(failing).toMatchObject({ passed: false, score: 0, status: 'failed' });
+  });
+
+  test('marks malformed, coerced, and out-of-range strict judge scores invalid', async () => {
+    const evaluator = new LLMGraderEvaluator();
+    for (const text of [
+      'Score: 1',
+      '{"score":"1","reason":"coerced"}',
+      '{"score":1.01,"reason":"out of range"}',
+      '```json\n{"score":1,"reason":"wrapped"}\n```',
+    ]) {
+      const result = await evaluator.evaluate('response', graderCase(true), {
+        client: clientWith(text),
+      });
+      expect(result).toMatchObject({
+        passed: false,
+        score: 0,
+        status: 'invalid',
+        evidence: { validation: { status: 'invalid', code: 'grader_failure' } },
+      });
+    }
+  });
+
+  test('keeps permissive parsing for non-strict legacy graders', async () => {
+    const result = await new LLMGraderEvaluator().evaluate('response', graderCase(false), {
+      client: clientWith('Score: 0.8'),
+    });
+
+    expect(result).toMatchObject({ passed: true, score: 0.8, status: 'passed' });
+  });
+});