@artemiskit/core 0.3.0 → 0.4.1

package/src/evaluators/llm-grader.test.ts

@@ -0,0 +1,80 @@
+import { describe, expect, test } from 'bun:test';
+import type { ModelClient } from '../adapters/types';
+import { ScenarioSchema } from '../scenario/schema';
+import { LLMGraderEvaluator } from './llm-grader';
+
+function graderCase(strict: boolean) {
+  return ScenarioSchema.parse({
+    name: 'grader parsing',
+    cases: [
+      {
+        id: 'grade',
+        prompt: 'grade this',
+        expected: { type: 'llm_grader', rubric: 'Be correct', threshold: 0.7, strict },
+      },
+    ],
+  }).cases[0].expected;
+}
+
+function clientWith(text: string): ModelClient {
+  return {
+    provider: 'test',
+    generate: async () => ({
+      id: 'judge-response',
+      model: 'judge-model',
+      text,
+      tokens: { prompt: 1, completion: 1, total: 2 },
+      latencyMs: 1,
+      finishReason: 'stop',
+    }),
+    capabilities: async () => ({
+      streaming: false,
+      functionCalling: false,
+      toolUse: false,
+      maxContext: 1,
+    }),
+  };
+}
+
+describe('LLMGraderEvaluator strict parsing', () => {
+  test('accepts finite numeric boundary scores in exact JSON', async () => {
+    const evaluator = new LLMGraderEvaluator();
+    const passing = await evaluator.evaluate('response', graderCase(true), {
+      client: clientWith('{"score":1,"reason":"complete"}'),
+    });
+    const failing = await evaluator.evaluate('response', graderCase(true), {
+      client: clientWith('{"score":0,"reason":"missing requirement"}'),
+    });
+
+    expect(passing).toMatchObject({ passed: true, score: 1, status: 'passed' });
+    expect(failing).toMatchObject({ passed: false, score: 0, status: 'failed' });
+  });
+
+  test('marks malformed, coerced, and out-of-range strict judge scores invalid', async () => {
+    const evaluator = new LLMGraderEvaluator();
+    for (const text of [
+      'Score: 1',
+      '{"score":"1","reason":"coerced"}',
+      '{"score":1.01,"reason":"out of range"}',
+      '```json\n{"score":1,"reason":"wrapped"}\n```',
+    ]) {
+      const result = await evaluator.evaluate('response', graderCase(true), {
+        client: clientWith(text),
+      });
+      expect(result).toMatchObject({
+        passed: false,
+        score: 0,
+        status: 'invalid',
+        evidence: { validation: { status: 'invalid', code: 'grader_failure' } },
+      });
+    }
+  });
+
+  test('keeps permissive parsing for non-strict legacy graders', async () => {
+    const result = await new LLMGraderEvaluator().evaluate('response', graderCase(false), {
+      client: clientWith('Score: 0.8'),
+    });
+
+    expect(result).toMatchObject({ passed: true, score: 0.8, status: 'passed' });
+  });
+});

package/src/evaluators/llm-grader.ts

@@ -58,18 +58,18 @@ export class LLMGraderEvaluator implements Evaluator {
         maxTokens: 1000,
       });
 
-      const parsed = this.parseGraderResponse(result.text);
+      const parsed = this.parseGraderResponse(result.text, expected.strict);
       const passed = parsed.score >= expected.threshold;
 
       return {
         passed,
         score: parsed.score,
         reason: parsed.reason || `Score: ${parsed.score.toFixed(2)}`,
-        details: {
-          graderResponse: result.text,
-          rubric: expected.rubric,
+        status: passed ? 'passed' : 'failed',
+        evidence: {
           threshold: expected.threshold,
           model: result.model,
+          validation: { status: 'valid' },
         },
       };
     } catch (error) {
@@ -77,12 +77,50 @@ export class LLMGraderEvaluator implements Evaluator {
         passed: false,
         score: 0,
         reason: `Grader failed: ${(error as Error).message}`,
-        details: { error: (error as Error).message },
+        status: 'invalid',
+        evidence: {
+          threshold: expected.threshold,
+          validation: { status: 'invalid', code: 'grader_failure' },
+        },
       };
     }
   }
 
-  private parseGraderResponse(text: string): { score: number; reason?: string } {
+  private parseGraderResponse(text: string, strict: boolean): { score: number; reason?: string } {
+    if (strict) {
+      return this.parseStrictGraderResponse(text);
+    }
+
+    return this.parseLegacyGraderResponse(text);
+  }
+
+  private parseStrictGraderResponse(text: string): { score: number; reason: string } {
+    let parsed: unknown;
+    try {
+      parsed = JSON.parse(text);
+    } catch {
+      throw new Error('Grader response must be an exact JSON object');
+    }
+
+    if (
+      !parsed ||
+      typeof parsed !== 'object' ||
+      Array.isArray(parsed) ||
+      typeof (parsed as { score?: unknown }).score !== 'number' ||
+      !Number.isFinite((parsed as { score: number }).score) ||
+      (parsed as { score: number }).score < 0 ||
+      (parsed as { score: number }).score > 1 ||
+      typeof (parsed as { reason?: unknown }).reason !== 'string'
+    ) {
+      throw new Error(
+        'Grader response must contain a finite numeric score from 0 to 1 and a string reason'
+      );
+    }
+
+    return parsed as { score: number; reason: string };
+  }
+
+  private parseLegacyGraderResponse(text: string): { score: number; reason?: string } {
     // Clean up the response - remove markdown code blocks if present
     const cleanedText = text
       .replace(/```json\s*/gi, '')

package/src/evaluators/tool-trace.test.ts

@@ -0,0 +1,46 @@
+import { describe, expect, it } from 'bun:test';
+import { ToolTraceEvaluator } from './tool-trace';
+
+const trace = ['lookup_customer', 'create_ticket'].map((name, step) => ({
+  step,
+  toolCall: { id: `call-${step}`, type: 'function' as const, function: { name, arguments: '{}' } },
+  latencyMs: 1,
+}));
+
+describe('ToolTraceEvaluator', () => {
+  it('accepts required ordered calls', async () => {
+    await expect(
+      new ToolTraceEvaluator().evaluate(
+        '',
+        {
+          type: 'tool_trace',
+          requiredTools: ['lookup_customer', 'create_ticket'],
+          ordered: true,
+          maxCalls: 2,
+        },
+        { toolTrace: trace }
+      )
+    ).resolves.toMatchObject({ passed: true });
+  });
+
+  it('rejects missing, forbidden, and excessive calls', async () => {
+    const evaluator = new ToolTraceEvaluator();
+    await expect(
+      evaluator.evaluate(
+        '',
+        { type: 'tool_trace', requiredTools: ['delete_customer'] },
+        { toolTrace: trace }
+      )
+    ).resolves.toMatchObject({
+      passed: false,
+      reason: 'Required tools not called: delete_customer',
+    });
+    await expect(
+      evaluator.evaluate(
+        '',
+        { type: 'tool_trace', forbiddenTools: ['create_ticket'] },
+        { toolTrace: trace }
+      )
+    ).resolves.toMatchObject({ passed: false, reason: 'Forbidden tools called: create_ticket' });
+  });
+});

package/src/evaluators/tool-trace.ts

@@ -0,0 +1,50 @@
+import type { Expected } from '../scenario/schema';
+import type { Evaluator, EvaluatorContext, EvaluatorResult } from './types';
+
+type ToolTraceExpected = Extract<Expected, { type: 'tool_trace' }>;
+
+export class ToolTraceEvaluator implements Evaluator {
+  readonly type = 'tool_trace';
+
+  async evaluate(
+    _response: string,
+    expected: Expected,
+    context?: EvaluatorContext
+  ): Promise<EvaluatorResult> {
+    const expectation = expected as ToolTraceExpected;
+    const calls = (context?.toolTrace ?? []).map((entry) => entry.toolCall.function.name);
+    const required = expectation.requiredTools ?? [];
+    const forbidden = expectation.forbiddenTools ?? [];
+
+    const missing = required.filter((name) => !calls.includes(name));
+    if (missing.length) return fail(`Required tools not called: ${missing.join(', ')}`, calls);
+
+    const presentForbidden = forbidden.filter((name) => calls.includes(name));
+    if (presentForbidden.length)
+      return fail(`Forbidden tools called: ${presentForbidden.join(', ')}`, calls);
+
+    if (expectation.maxCalls !== undefined && calls.length > expectation.maxCalls) {
+      return fail(`Tool calls exceeded limit of ${expectation.maxCalls}`, calls);
+    }
+
+    if (expectation.ordered && !isOrdered(calls, required)) {
+      return fail('Required tools were not called in order', calls);
+    }
+
+    return { passed: true, score: 1, reason: 'Tool trace matched expectation', details: { calls } };
+  }
+}
+
+function isOrdered(calls: string[], required: string[]): boolean {
+  let start = 0;
+  for (const tool of required) {
+    const index = calls.indexOf(tool, start);
+    if (index === -1) return false;
+    start = index + 1;
+  }
+  return true;
+}
+
+function fail(reason: string, calls: string[]): EvaluatorResult {
+  return { passed: false, score: 0, reason, details: { calls } };
+}

package/src/evaluators/types.ts

@@ -4,6 +4,7 @@
 
 import type { ModelClient } from '../adapters/types';
 import type { Expected, TestCase } from '../scenario/schema';
+import type { ToolTraceEntry } from '../tools';
 
 /**
  * Context provided to evaluators
@@ -11,6 +12,7 @@ import type { Expected, TestCase } from '../scenario/schema';
 export interface EvaluatorContext {
   client?: ModelClient;
   testCase?: TestCase;
+  toolTrace?: ToolTraceEntry[];
 }
 
 /**
@@ -20,6 +22,24 @@ export interface EvaluatorResult {
   passed: boolean;
   score: number;
   reason?: string;
+  /**
+   * Whether this evaluator produced a valid measurement. Omitted by legacy
+   * evaluators; the executor derives it from `passed` for compatibility.
+   */
+  status?: 'passed' | 'failed' | 'invalid';
+  /** Bounded evaluator metadata that may be retained in a run artifact. */
+  evidence?: {
+    threshold?: number;
+    model?: string;
+    validation?: {
+      status: 'valid' | 'invalid';
+      code?: string;
+    };
+  };
+  /**
+   * Legacy evaluator-private metadata. The standard executor never serializes
+   * this field; use `evidence` for reviewed, bounded artifact data.
+   */
   details?: Record<string, unknown>;
 }
 

package/src/index.ts

@@ -33,5 +33,11 @@ export * from './redaction';
 // Cost estimation
 export * from './cost';
 
+// Safe tool execution
+export * from './tools';
+
+// Real-agent evaluation contracts
+export * from './agent-evaluation';
+
 // Validator
 export * from './validator';

package/src/runner/executor.test.ts

@@ -0,0 +1,374 @@
+import { describe, expect, it } from 'bun:test';
+import type { GenerateOptions, ModelClient } from '../adapters/types';
+import { registerEvaluator } from '../evaluators';
+import type { Evaluator } from '../evaluators';
+import { ScenarioSchema } from '../scenario/schema';
+import { executeCase } from './executor';
+
+function createEnabledToolLoopScenario() {
+  return ScenarioSchema.parse({
+    name: 'fixture-backed tool loop',
+    provider: 'ling',
+    model: 'Ling-3.0-flash',
+    setup: {
+      tools: [
+        {
+          type: 'function',
+          function: {
+            name: 'lookup_order',
+            parameters: { type: 'object', properties: { orderId: { type: 'string' } } },
+          },
+        },
+      ],
+      fixtures: {
+        lookup_order: [{ when: { orderId: 'A-1' }, result: { status: 'delivered' } }],
+      },
+      toolLoop: { enabled: true, maxSteps: 2 },
+    },
+    cases: [
+      {
+        id: 'order-status',
+        prompt: 'Where is order A-1?',
+        expected: { type: 'exact', value: 'Order A-1 was delivered.' },
+      },
+    ],
+  });
+}
+
+function createToolCallResponse() {
+  return {
+    id: 'first',
+    model: 'Ling-3.0-flash',
+    text: '',
+    tokens: { prompt: 7, completion: 2, total: 9 },
+    latencyMs: 4,
+    finishReason: 'tool_calls' as const,
+    toolCalls: [
+      {
+        id: 'call-order',
+        type: 'function' as const,
+        function: { name: 'lookup_order', arguments: '{"orderId":"A-1"}' },
+      },
+    ],
+  };
+}
+
+describe('executeCase tool loop', () => {
+  it('preserves tool call IDs while resolving fixture-backed calls', async () => {
+    const scenario = createEnabledToolLoopScenario();
+    const requests: GenerateOptions[] = [];
+    const responses = [
+      {
+        id: 'first',
+        model: 'Ling-3.0-flash',
+        text: '',
+        tokens: { prompt: 10, completion: 3, total: 13 },
+        latencyMs: 1,
+        finishReason: 'tool_calls' as const,
+        toolCalls: [
+          {
+            id: 'call-order',
+            type: 'function' as const,
+            function: { name: 'lookup_order', arguments: '{"orderId":"A-1"}' },
+          },
+        ],
+      },
+      {
+        id: 'second',
+        model: 'Ling-3.0-flash',
+        text: 'Order A-1 was delivered.',
+        tokens: { prompt: 20, completion: 5, total: 25 },
+        latencyMs: 1,
+        finishReason: 'stop' as const,
+      },
+    ];
+    const client: ModelClient = {
+      provider: 'ling',
+      generate: async (options) => {
+        requests.push(options);
+        const response = responses.shift();
+        if (!response) throw new Error('Unexpected generation request');
+        return response;
+      },
+      capabilities: async () => ({
+        streaming: true,
+        functionCalling: true,
+        toolUse: true,
+        maxContext: 256000,
+      }),
+    };
+
+    const result = await executeCase(scenario.cases[0], { client, scenario });
+
+    expect(result.ok).toBe(true);
+    expect(result.latencyMs).toBe(2);
+    expect(result.tokens).toEqual({ prompt: 30, completion: 8, total: 38 });
+    expect(requests).toHaveLength(2);
+    expect(requests[1].prompt).toEqual([
+      { role: 'user', content: 'Where is order A-1?' },
+      {
+        role: 'assistant',
+        content: '',
+        tool_calls: [
+          {
+            id: 'call-order',
+            type: 'function',
+            function: { name: 'lookup_order', arguments: '{"orderId":"A-1"}' },
+          },
+        ],
+      },
+      {
+        role: 'tool',
+        name: 'lookup_order',
+        toolCallId: 'call-order',
+        content: '{"status":"delivered"}',
+      },
+    ]);
+  });
+
+  it('retains generation metrics when the tool loop cannot start', async () => {
+    const scenario = ScenarioSchema.parse({
+      name: 'invalid tool loop',
+      provider: 'ling',
+      model: 'Ling-3.0-flash',
+      setup: { toolLoop: { enabled: true } },
+      cases: [
+        {
+          id: 'missing-tools',
+          prompt: 'Use a tool',
+          expected: { type: 'exact', value: '' },
+        },
+      ],
+    });
+    const client: ModelClient = {
+      provider: 'ling',
+      generate: async () => ({
+        id: 'first',
+        model: 'Ling-3.0-flash',
+        text: '',
+        tokens: { prompt: 7, completion: 2, total: 9 },
+        latencyMs: 4,
+        finishReason: 'tool_calls',
+        toolCalls: [
+          {
+            id: 'call-order',
+            type: 'function',
+            function: { name: 'lookup_order', arguments: '{"orderId":"A-1"}' },
+          },
+        ],
+      }),
+      capabilities: async () => ({
+        streaming: true,
+        functionCalling: true,
+        toolUse: true,
+        maxContext: 256000,
+      }),
+    };
+
+    const result = await executeCase(scenario.cases[0], { client, scenario });
+
+    expect(result.error).toBe('TOOL_EXECUTOR_REQUIRED');
+    expect(result.latencyMs).toBe(4);
+    expect(result.tokens).toEqual({ prompt: 7, completion: 2, total: 9 });
+  });
+
+  it('retains prior generation metrics when a later generation rejects', async () => {
+    const scenario = createEnabledToolLoopScenario();
+    let calls = 0;
+    const client: ModelClient = {
+      provider: 'ling',
+      generate: async () => {
+        calls++;
+        if (calls === 1) return createToolCallResponse();
+        throw new Error('later generation failed');
+      },
+      capabilities: async () => ({
+        streaming: true,
+        functionCalling: true,
+        toolUse: true,
+        maxContext: 256000,
+      }),
+    };
+
+    const result = await executeCase(scenario.cases[0], { client, scenario });
+
+    expect(calls).toBe(2);
+    expect(result.error).toBe('TOOL_GENERATION_FAILED');
+    expect(result.latencyMs).toBe(4);
+    expect(result.tokens).toEqual({ prompt: 7, completion: 2, total: 9 });
+    expect(result.toolLoop).toEqual({
+      status: 'error',
+      steps: 1,
+      terminationReason: 'tool_error',
+    });
+  });
+
+  it('retains prior generation metrics when a later generation times out', async () => {
+    const scenario = createEnabledToolLoopScenario();
+    let calls = 0;
+    const client: ModelClient = {
+      provider: 'ling',
+      generate: async () => {
+        calls++;
+        if (calls === 1) return createToolCallResponse();
+        return await new Promise<never>(() => {});
+      },
+      capabilities: async () => ({
+        streaming: true,
+        functionCalling: true,
+        toolUse: true,
+        maxContext: 256000,
+      }),
+    };
+
+    const result = await executeCase(scenario.cases[0], { client, scenario, timeout: 10 });
+
+    expect(calls).toBe(2);
+    expect(result.error).toBe('TOOL_LOOP_TIMEOUT');
+    expect(result.latencyMs).toBe(4);
+    expect(result.tokens).toEqual({ prompt: 7, completion: 2, total: 9 });
+    expect(result.toolLoop).toEqual({
+      status: 'error',
+      steps: 1,
+      terminationReason: 'timeout',
+    });
+  });
+});
+
+describe('executeCase measurement integrity', () => {
+  const scenario = ScenarioSchema.parse({
+    name: 'measurement integrity',
+    cases: [
+      {
+        id: 'custom-evaluation',
+        prompt: 'Evaluate this',
+        expected: { type: 'custom', evaluator: 'test' },
+      },
+    ],
+  });
+
+  const client: ModelClient = {
+    provider: 'test',
+    generate: async () => ({
+      id: 'response',
+      model: 'target-model',
+      text: 'target response',
+      tokens: { prompt: 1, completion: 1, total: 2 },
+      latencyMs: 1,
+      finishReason: 'stop',
+    }),
+    capabilities: async () => ({
+      streaming: false,
+      functionCalling: false,
+      toolUse: false,
+      maxContext: 1,
+    }),
+  };
+
+  it('marks an evaluator exception invalid after a target response is received', async () => {
+    registerEvaluator('custom', {
+      type: 'custom',
+      evaluate: async () => {
+        throw new Error('judge offline');
+      },
+    });
+
+    const result = await executeCase(scenario.cases[0], { client, scenario });
+
+    expect(result).toMatchObject({
+      ok: false,
+      status: 'invalid',
+      response: 'target response',
+      evidence: {
+        evaluator: 'custom',
+        validation: { status: 'invalid', code: 'evaluator_failure' },
+      },
+    });
+  });
+
+  it('marks a target generation failure as an execution error', async () => {
+    const unavailableClient: ModelClient = {
+      ...client,
+      generate: async () => {
+        throw new Error('provider unavailable');
+      },
+    };
+
+    const result = await executeCase(scenario.cases[0], {
+      client: unavailableClient,
+      scenario,
+    });
+
+    expect(result).toMatchObject({
+      ok: false,
+      status: 'error',
+      response: '',
+      error: 'provider unavailable',
+    });
+  });
+
+  it('retains only the bounded evidence contract rather than evaluator details', async () => {
+    const evaluator: Evaluator = {
+      type: 'custom',
+      evaluate: async () => ({
+        passed: true,
+        score: 1,
+        status: 'passed',
+        evidence: {
+          threshold: 0.7,
+          model: 'reviewer-model',
+          validation: { status: 'valid', code: 'accepted' },
+        },
+        details: { rawJudgeOutput: 'secret judge transcript', rubric: 'secret rubric' },
+      }),
+    };
+    registerEvaluator('custom', evaluator);
+
+    const result = await executeCase(scenario.cases[0], { client, scenario });
+
+    expect(result.status).toBe('passed');
+    expect(result.evidence).toEqual({
+      evaluator: 'custom',
+      score: 1,
+      threshold: 0.7,
+      model: 'reviewer-model',
+      validation: { status: 'valid', code: 'accepted' },
+    });
+    expect(JSON.stringify(result)).not.toContain('secret judge transcript');
+    expect(JSON.stringify(result)).not.toContain('secret rubric');
+  });
+
+  it('redacts evaluator reason text and discards malformed runtime evidence', async () => {
+    registerEvaluator('custom', {
+      type: 'custom',
+      evaluate: async () =>
+        ({
+          passed: false,
+          score: 0.4,
+          status: 'invalid-status',
+          reason: 'judge_error: token=super-secret-token',
+          evidence: {
+            threshold: '0.7',
+            model: 'reviewer@example.com',
+            validation: { status: 'unknown', code: 'token=super-secret-token' },
+          },
+        }) as never,
+    });
+    const redactedScenario = ScenarioSchema.parse({
+      ...scenario,
+      redaction: { enabled: true, patterns: ['email', 'secrets'] },
+    });
+
+    const result = await executeCase(redactedScenario.cases[0], {
+      client,
+      scenario: redactedScenario,
+    });
+
+    expect(result.status).toBe('failed');
+    expect(result.reason).toContain('[REDACTED]');
+    expect(result.evidence).toEqual({ evaluator: 'custom', score: 0.4, model: '[REDACTED]' });
+    expect(result.redaction).toMatchObject({ redacted: true, reasonRedacted: true });
+    expect(JSON.stringify(result)).not.toContain('super-secret-token');
+    expect(JSON.stringify(result)).not.toContain('reviewer@example.com');
+  });
+});