@artemiskit/core 0.3.0 → 0.4.0

package/src/storage/supabase.test.ts

@@ -181,6 +181,41 @@ describe('SupabaseStorageAdapter', () => {
       const result = await adapter.save(manifest);
 
       expect(result).toBe('test-project/test-run-123.json');
+      expect(mockUpsert.mock.calls[0][0]).toMatchObject({
+        total_attempts: 10,
+        valid_evaluations: 10,
+        invalid_evaluations: 0,
+        outcome_rate_denominator: 10,
+      });
+    });
+
+    it('should save manifest integrity counts without using invalid measurements as outcomes', async () => {
+      const manifest = createMockManifest({
+        cases: [],
+        metrics: {
+          ...createMockManifest().metrics,
+          total_attempts: 7,
+          valid_evaluations: 4,
+          invalid_evaluations: 2,
+          outcome_rate_denominator: 4,
+          passed_cases: 3,
+          failed_cases: 1,
+          success_rate: 0.75,
+        },
+      });
+      mockStorage.mockReturnValue({ upload: mock(() => Promise.resolve({ error: null })) });
+      const mockUpsert = mock(() => Promise.resolve({ error: null }));
+      mockFrom.mockReturnValue({ upsert: mockUpsert });
+
+      await adapter.save(manifest);
+
+      expect(mockUpsert.mock.calls[0][0]).toMatchObject({
+        total_attempts: 7,
+        valid_evaluations: 4,
+        invalid_evaluations: 2,
+        outcome_rate_denominator: 4,
+        success_rate: 0.75,
+      });
     });
 
     it('should throw error on storage upload failure', async () => {
@@ -577,6 +612,34 @@ describe('SupabaseStorageAdapter', () => {
       expect(result).toBe('uuid-123');
     });
 
+    it('should retain an invalid case status with bounded evidence and attempts', async () => {
+      const caseResult = createMockCaseResultRecord({
+        status: 'invalid',
+        attempts: 2,
+        evidence: {
+          evaluator: 'llm_grader',
+          validation: { status: 'invalid', code: 'grader_failure' },
+        },
+      });
+      const mockUpsert = mock(() => ({
+        select: mock(() => ({
+          single: mock(() => Promise.resolve({ data: { id: 'uuid-123' }, error: null })),
+        })),
+      }));
+      mockFrom.mockReturnValue({ upsert: mockUpsert });
+
+      await adapter.saveCaseResult(caseResult);
+
+      expect(mockUpsert.mock.calls[0][0]).toMatchObject({
+        status: 'invalid',
+        attempts: 2,
+        evidence: {
+          evaluator: 'llm_grader',
+          validation: { status: 'invalid', code: 'grader_failure' },
+        },
+      });
+    });
+
     it('should throw error on save failure', async () => {
       const caseResult = createMockCaseResultRecord();
 
@@ -980,9 +1043,10 @@ describe('Type Safety', () => {
   });
 
   it('should have valid CaseResultStatus types', () => {
-    const statuses: CaseResultRecord['status'][] = ['passed', 'failed', 'error'];
+    const statuses: CaseResultRecord['status'][] = ['passed', 'failed', 'invalid', 'error'];
     expect(statuses).toContain('passed');
     expect(statuses).toContain('failed');
+    expect(statuses).toContain('invalid');
     expect(statuses).toContain('error');
   });
 });

package/src/storage/supabase.ts

@@ -3,7 +3,7 @@
  */
 
 import { type SupabaseClient, createClient } from '@supabase/supabase-js';
-import type { CaseResult, RunManifest } from '../artifacts/types';
+import { type CaseResult, type RunManifest, getCaseEvaluationStatus } from '../artifacts/types';
 import type {
   AnalyticsStorageAdapter,
   BaselineMetadata,
@@ -31,7 +31,8 @@ function mapCaseToRecord(runId: string, caseResult: CaseResult): CaseResultRecor
     runId,
     caseId: caseResult.id,
     caseName: caseResult.name,
-    status: caseResult.error ? 'error' : caseResult.ok ? 'passed' : 'failed',
+    status: getCaseEvaluationStatus(caseResult),
+    attempts: caseResult.attempts ?? 1,
     score: caseResult.score,
     matcherType: caseResult.matcherType,
     reason: caseResult.reason,
@@ -41,6 +42,7 @@ function mapCaseToRecord(runId: string, caseResult: CaseResult): CaseResultRecor
     completionTokens: caseResult.tokens.completion,
     totalTokens: caseResult.tokens.total,
     error: caseResult.error,
+    evidence: caseResult.evidence,
     tags: caseResult.tags,
   };
 }
@@ -84,6 +86,11 @@ export class SupabaseStorageAdapter implements AnalyticsStorageAdapter {
       total_cases: manifest.metrics.total_cases,
       passed_cases: manifest.metrics.passed_cases,
       failed_cases: manifest.metrics.failed_cases,
+      total_attempts: manifest.metrics.total_attempts ?? manifest.metrics.total_cases,
+      valid_evaluations: manifest.metrics.valid_evaluations ?? manifest.metrics.total_cases,
+      invalid_evaluations: manifest.metrics.invalid_evaluations ?? 0,
+      outcome_rate_denominator:
+        manifest.metrics.outcome_rate_denominator ?? manifest.metrics.total_cases,
       median_latency_ms: manifest.metrics.median_latency_ms,
       p95_latency_ms: manifest.metrics.p95_latency_ms,
       total_tokens: manifest.metrics.total_tokens,
@@ -405,6 +412,7 @@ export class SupabaseStorageAdapter implements AnalyticsStorageAdapter {
       case_id: result.caseId,
       case_name: result.caseName,
       status: result.status,
+      attempts: result.attempts ?? 1,
       score: result.score,
       matcher_type: result.matcherType,
       reason: result.reason,
@@ -414,6 +422,7 @@ export class SupabaseStorageAdapter implements AnalyticsStorageAdapter {
       completion_tokens: result.completionTokens,
       total_tokens: result.totalTokens,
       error: result.error,
+      evidence: result.evidence,
       tags: result.tags || [],
     };
 
@@ -440,6 +449,7 @@ export class SupabaseStorageAdapter implements AnalyticsStorageAdapter {
       case_id: r.caseId,
       case_name: r.caseName,
       status: r.status,
+      attempts: r.attempts ?? 1,
       score: r.score,
       matcher_type: r.matcherType,
       reason: r.reason,
@@ -449,6 +459,7 @@ export class SupabaseStorageAdapter implements AnalyticsStorageAdapter {
       completion_tokens: r.completionTokens,
       total_tokens: r.totalTokens,
       error: r.error,
+      evidence: r.evidence,
       tags: r.tags || [],
     }));
 
@@ -481,6 +492,7 @@ export class SupabaseStorageAdapter implements AnalyticsStorageAdapter {
       caseId: r.case_id,
       caseName: r.case_name,
       status: r.status,
+      attempts: r.attempts,
       score: r.score,
       matcherType: r.matcher_type,
       reason: r.reason,
@@ -490,6 +502,7 @@ export class SupabaseStorageAdapter implements AnalyticsStorageAdapter {
       completionTokens: r.completion_tokens,
       totalTokens: r.total_tokens,
       error: r.error,
+      evidence: r.evidence,
       tags: r.tags,
       createdAt: r.created_at,
     }));
@@ -531,6 +544,7 @@ export class SupabaseStorageAdapter implements AnalyticsStorageAdapter {
       caseId: r.case_id,
       caseName: r.case_name,
       status: r.status,
+      attempts: r.attempts,
       score: r.score,
       matcherType: r.matcher_type,
       reason: r.reason,
@@ -540,6 +554,7 @@ export class SupabaseStorageAdapter implements AnalyticsStorageAdapter {
       completionTokens: r.completion_tokens,
       totalTokens: r.total_tokens,
       error: r.error,
+      evidence: r.evidence,
       tags: r.tags,
       createdAt: r.created_at,
     }));

package/src/storage/types.ts

@@ -2,7 +2,13 @@
  * Storage types and interfaces
  */
 
-import type { AnyManifest, RedTeamManifest, RunManifest, StressManifest } from '../artifacts/types';
+import type {
+  AnyManifest,
+  CaseEvaluationEvidence,
+  RedTeamManifest,
+  RunManifest,
+  StressManifest,
+} from '../artifacts/types';
 
 /**
  * Run listing item
@@ -181,7 +187,7 @@ export interface BaselineStorageAdapter extends StorageAdapter {
 /**
  * Status of an individual case result
  */
-export type CaseResultStatus = 'passed' | 'failed' | 'error';
+export type CaseResultStatus = 'passed' | 'failed' | 'invalid' | 'error';
 
 /**
  * Individual case result record for storage
@@ -197,6 +203,8 @@ export interface CaseResultRecord {
   caseName?: string;
   /** Result status */
   status: CaseResultStatus;
+  /** Number of execution attempts represented by this terminal result. */
+  attempts?: number;
   /** Score from 0.0 to 1.0 */
   score: number;
   /** Type of matcher used */
@@ -215,6 +223,8 @@ export interface CaseResultRecord {
   totalTokens: number;
   /** Error message if status is 'error' */
   error?: string;
+  /** Sanitized evaluator evidence retained for review. */
+  evidence?: CaseEvaluationEvidence;
   /** Tags for categorization */
   tags?: string[];
   /** ISO timestamp when created */

package/src/tools/fixture-executor.test.ts

@@ -0,0 +1,88 @@
+import { describe, expect, it } from 'bun:test';
+import { FixtureToolExecutor } from './fixture-executor';
+
+const tools = [
+  {
+    type: 'function' as const,
+    function: {
+      name: 'get_weather',
+      parameters: {
+        type: 'object',
+        additionalProperties: false,
+        required: ['city'],
+        properties: { city: { type: 'string' } },
+      },
+    },
+  },
+];
+
+const call = (arguments_: string, name = 'get_weather') => ({
+  id: 'call-1',
+  type: 'function' as const,
+  function: { name, arguments: arguments_ },
+});
+
+describe('FixtureToolExecutor', () => {
+  it('returns a deterministic matching fixture result', async () => {
+    const executor = new FixtureToolExecutor({
+      tools,
+      fixtures: { get_weather: [{ when: { city: 'Lagos' }, result: { temperature_c: 28 } }] },
+    });
+
+    await expect(executor.execute(call('{"city":"Lagos"}'))).resolves.toEqual({
+      status: 'success',
+      result: { temperature_c: 28 },
+    });
+  });
+
+  it('rejects undeclared tools, invalid JSON, and unmatched fixtures', async () => {
+    const executor = new FixtureToolExecutor({ tools, fixtures: { get_weather: [] } });
+
+    await expect(executor.execute(call('{}', 'delete_weather'))).resolves.toMatchObject({
+      status: 'error',
+      error: { code: 'TOOL_UNKNOWN' },
+    });
+    await expect(executor.execute(call('{city:Lagos}'))).resolves.toMatchObject({
+      status: 'error',
+      error: { code: 'TOOL_ARGUMENTS_INVALID' },
+    });
+    await expect(executor.execute(call('{"city":"Lagos"}'))).resolves.toMatchObject({
+      status: 'error',
+      error: { code: 'TOOL_FIXTURE_NOT_FOUND' },
+    });
+  });
+
+  it('returns controlled fixture errors and result-size failures', async () => {
+    const errorExecutor = new FixtureToolExecutor({
+      tools,
+      fixtures: { get_weather: [{ error: 'Weather service is unavailable.' }] },
+    });
+    const limitedExecutor = new FixtureToolExecutor({
+      tools,
+      fixtures: { get_weather: [{ result: { forecast: 'x'.repeat(32) } }] },
+      maxToolResultBytes: 16,
+    });
+
+    await expect(errorExecutor.execute(call('{"city":"Lagos"}'))).resolves.toMatchObject({
+      status: 'error',
+      error: { code: 'TOOL_EXECUTION_FAILED' },
+    });
+    await expect(limitedExecutor.execute(call('{"city":"Lagos"}'))).resolves.toMatchObject({
+      status: 'error',
+      error: { code: 'TOOL_RESULT_TOO_LARGE' },
+    });
+  });
+
+  it('rejects unsafe fixture configuration and fixtures for undeclared tools', () => {
+    expect(
+      () =>
+        new FixtureToolExecutor({
+          tools,
+          fixtures: { get_weather: [{ result: { command: 'curl example.invalid' } }] },
+        })
+    ).toThrow('TOOL_FIXTURE_UNSAFE');
+    expect(() => new FixtureToolExecutor({ tools, fixtures: { no_such_tool: [] } })).toThrow(
+      'TOOL_UNKNOWN'
+    );
+  });
+});

package/src/tools/fixture-executor.ts

@@ -0,0 +1,112 @@
+import type { ToolCall, ToolDefinition } from '../adapters/types';
+import { validateToolArguments } from './schema-validator';
+import type {
+  FixtureExecutorOptions,
+  ToolExecutionContext,
+  ToolExecutionResult,
+  ToolFixture,
+  ToolFixtures,
+} from './types';
+
+const DISALLOWED_FIXTURE_FIELDS = new Set(['command', 'url', 'path', 'code', 'environment', 'env']);
+
+export class FixtureToolExecutor {
+  private readonly tools = new Map<string, ToolDefinition>();
+  private readonly fixtures: ToolFixtures;
+  private readonly maxToolResultBytes: number;
+
+  constructor({ tools, fixtures, maxToolResultBytes = 32_768 }: FixtureExecutorOptions) {
+    for (const tool of tools) this.tools.set(tool.function.name, tool);
+    for (const [name, entries] of Object.entries(fixtures)) {
+      if (!this.tools.has(name)) {
+        throw new Error(`TOOL_UNKNOWN: fixture declared for undeclared tool '${name}'`);
+      }
+      entries.forEach(assertSafeFixture);
+    }
+    this.fixtures = fixtures;
+    this.maxToolResultBytes = maxToolResultBytes;
+  }
+
+  async execute(call: ToolCall, _context?: ToolExecutionContext): Promise<ToolExecutionResult> {
+    const tool = this.tools.get(call.function.name);
+    if (!tool) {
+      return error('TOOL_UNKNOWN', `Tool '${call.function.name}' is not declared.`);
+    }
+
+    const validated = validateToolArguments(call.function.arguments, tool.function.parameters);
+    if (!validated.valid) return { status: 'error', error: validated.error };
+
+    const fixture = this.fixtures[call.function.name]?.find((candidate) =>
+      matches(candidate.when, validated.arguments ?? {})
+    );
+    if (!fixture) {
+      return error('TOOL_FIXTURE_NOT_FOUND', `No fixture matched tool '${call.function.name}'.`);
+    }
+    if (fixture.error) return error('TOOL_EXECUTION_FAILED', fixture.error);
+
+    const serialized = JSON.stringify(fixture.result ?? {});
+    if (
+      serialized === undefined ||
+      new TextEncoder().encode(serialized).byteLength > this.maxToolResultBytes
+    ) {
+      return error(
+        'TOOL_RESULT_TOO_LARGE',
+        `Tool '${call.function.name}' result exceeds the configured limit.`
+      );
+    }
+    return { status: 'success', result: fixture.result ?? {} };
+  }
+}
+
+function error(
+  code: NonNullable<ToolExecutionResult['error']>['code'],
+  message: string
+): ToolExecutionResult {
+  return { status: 'error', error: { code, message } };
+}
+
+function matches(
+  when: Record<string, unknown> | undefined,
+  args: Record<string, unknown>
+): boolean {
+  return !when || Object.entries(when).every(([key, value]) => deepEqual(args[key], value));
+}
+
+function deepEqual(left: unknown, right: unknown): boolean {
+  if (Object.is(left, right)) return true;
+  if (!left || !right || typeof left !== 'object' || typeof right !== 'object') return false;
+  if (Array.isArray(left) || Array.isArray(right)) {
+    return (
+      Array.isArray(left) &&
+      Array.isArray(right) &&
+      left.length === right.length &&
+      left.every((value, index) => deepEqual(value, right[index]))
+    );
+  }
+  const leftRecord = left as Record<string, unknown>;
+  const rightRecord = right as Record<string, unknown>;
+  const keys = Object.keys(rightRecord);
+  return (
+    keys.length === Object.keys(leftRecord).length &&
+    keys.every((key) => deepEqual(leftRecord[key], rightRecord[key]))
+  );
+}
+
+function assertSafeFixture(fixture: ToolFixture): void {
+  assertSafeValue(fixture.when);
+  assertSafeValue(fixture.result);
+}
+
+function assertSafeValue(value: unknown): void {
+  if (!value || typeof value !== 'object') return;
+  if (Array.isArray(value)) {
+    value.forEach(assertSafeValue);
+    return;
+  }
+  for (const [key, child] of Object.entries(value as Record<string, unknown>)) {
+    if (DISALLOWED_FIXTURE_FIELDS.has(key.toLowerCase())) {
+      throw new Error(`TOOL_FIXTURE_UNSAFE: '${key}' is not permitted in fixture data.`);
+    }
+    assertSafeValue(child);
+  }
+}

package/src/tools/index.ts

@@ -0,0 +1,3 @@
+export * from './fixture-executor';
+export * from './schema-validator';
+export * from './types';

package/src/tools/schema-validator.test.ts

@@ -0,0 +1,32 @@
+import { describe, expect, it } from 'bun:test';
+import { validateToolArguments } from './schema-validator';
+
+const weatherSchema = {
+  type: 'object',
+  additionalProperties: false,
+  required: ['city'],
+  properties: { city: { type: 'string', minLength: 1 } },
+};
+
+describe('validateToolArguments', () => {
+  it('returns parsed arguments that satisfy the JSON schema', () => {
+    expect(validateToolArguments('{"city":"Lagos"}', weatherSchema)).toEqual({
+      valid: true,
+      arguments: { city: 'Lagos' },
+    });
+  });
+
+  it('rejects malformed JSON without echoing the arguments', () => {
+    expect(validateToolArguments('{city:Lagos}', weatherSchema)).toEqual({
+      valid: false,
+      error: { code: 'TOOL_ARGUMENTS_INVALID', message: 'Tool arguments must be valid JSON.' },
+    });
+  });
+
+  it('rejects arguments that fail the declared schema', () => {
+    const result = validateToolArguments('{"city":42}', weatherSchema);
+
+    expect(result.valid).toBe(false);
+    expect(result.error?.code).toBe('TOOL_ARGUMENTS_SCHEMA_INVALID');
+  });
+});

package/src/tools/schema-validator.ts

@@ -0,0 +1,56 @@
+import Ajv, { type ValidateFunction } from 'ajv';
+
+export interface ToolArgumentValidation {
+  valid: boolean;
+  arguments?: Record<string, unknown>;
+  error?: { code: 'TOOL_ARGUMENTS_INVALID' | 'TOOL_ARGUMENTS_SCHEMA_INVALID'; message: string };
+}
+
+const ajv = new Ajv({ allErrors: true, strict: false });
+const validators = new WeakMap<object, ValidateFunction>();
+
+function getValidator(schema: Record<string, unknown>): ValidateFunction {
+  const cached = validators.get(schema);
+  if (cached) return cached;
+  const validator = ajv.compile(schema);
+  validators.set(schema, validator);
+  return validator;
+}
+
+export function validateToolArguments(
+  rawArguments: string,
+  schema: Record<string, unknown>
+): ToolArgumentValidation {
+  let args: unknown;
+  try {
+    args = JSON.parse(rawArguments);
+  } catch {
+    return {
+      valid: false,
+      error: { code: 'TOOL_ARGUMENTS_INVALID', message: 'Tool arguments must be valid JSON.' },
+    };
+  }
+
+  if (!args || typeof args !== 'object' || Array.isArray(args)) {
+    return {
+      valid: false,
+      error: { code: 'TOOL_ARGUMENTS_INVALID', message: 'Tool arguments must be a JSON object.' },
+    };
+  }
+
+  const validator = getValidator(schema);
+  if (!validator(args)) {
+    const details = (validator.errors ?? [])
+      .map((error) => `${error.instancePath || '/'} ${error.message ?? 'is invalid'}`)
+      .join('; ');
+    return {
+      valid: false,
+      error: {
+        code: 'TOOL_ARGUMENTS_SCHEMA_INVALID',
+        message: `Tool arguments do not match the schema: ${details}`,
+      },
+    };
+  }
+
+  return { valid: true, arguments: args as Record<string, unknown> };
+}

package/src/tools/types.ts

@@ -0,0 +1,80 @@
+import type { ToolCall, ToolDefinition } from '../adapters/types';
+
+export type ToolLoopTerminationReason =
+  | 'completed'
+  | 'max_steps'
+  | 'timeout'
+  | 'duplicate_call'
+  | 'invalid_arguments'
+  | 'unknown_tool'
+  | 'tool_error';
+
+export interface ToolLoopPolicy {
+  enabled: boolean;
+  maxSteps: number;
+  timeoutMs: number;
+  maxToolResultBytes: number;
+  rejectDuplicateCalls: boolean;
+}
+
+export const DEFAULT_TOOL_LOOP_POLICY: ToolLoopPolicy = {
+  enabled: false,
+  maxSteps: 5,
+  timeoutMs: 60_000,
+  maxToolResultBytes: 32_768,
+  rejectDuplicateCalls: true,
+};
+
+export interface ToolExecutionContext {
+  caseId: string;
+  step: number;
+}
+
+export interface ToolExecutionError {
+  code:
+    | 'TOOL_ARGUMENTS_INVALID'
+    | 'TOOL_ARGUMENTS_SCHEMA_INVALID'
+    | 'TOOL_UNKNOWN'
+    | 'TOOL_FIXTURE_NOT_FOUND'
+    | 'TOOL_EXECUTION_FAILED'
+    | 'TOOL_RESULT_TOO_LARGE';
+  message: string;
+}
+
+export interface ToolExecutionResult {
+  status: 'success' | 'error';
+  result?: unknown;
+  error?: ToolExecutionError;
+}
+
+export interface ToolExecutor {
+  execute(call: ToolCall, context: ToolExecutionContext): Promise<ToolExecutionResult>;
+}
+
+export interface ToolTraceEntry {
+  step: number;
+  toolCall: ToolCall;
+  result?: unknown;
+  error?: ToolExecutionError;
+  latencyMs: number;
+}
+
+export interface ToolLoopSummary {
+  status: 'completed' | 'error';
+  steps: number;
+  terminationReason: ToolLoopTerminationReason;
+}
+
+export interface ToolFixture {
+  when?: Record<string, unknown>;
+  result?: unknown;
+  error?: string;
+}
+
+export type ToolFixtures = Record<string, ToolFixture[]>;
+
+export interface FixtureExecutorOptions {
+  tools: ToolDefinition[];
+  fixtures: ToolFixtures;
+  maxToolResultBytes?: number;
+}