@artemiskit/core 0.2.4 → 0.4.0

package/src/storage/types.ts

@@ -2,7 +2,13 @@
  * Storage types and interfaces
  */
 
-import type { AnyManifest, RedTeamManifest, RunManifest, StressManifest } from '../artifacts/types';
+import type {
+  AnyManifest,
+  CaseEvaluationEvidence,
+  RedTeamManifest,
+  RunManifest,
+  StressManifest,
+} from '../artifacts/types';
 
 /**
  * Run listing item
@@ -173,3 +179,203 @@ export interface BaselineStorageAdapter extends StorageAdapter {
     regressionThreshold: number;
   } | null>;
 }
+
+// ============================================================================
+// Case Results Types (for granular analytics)
+// ============================================================================
+
+/**
+ * Status of an individual case result
+ */
+export type CaseResultStatus = 'passed' | 'failed' | 'invalid' | 'error';
+
+/**
+ * Individual case result record for storage
+ */
+export interface CaseResultRecord {
+  /** Unique ID (auto-generated if not provided) */
+  id?: string;
+  /** Run ID this case belongs to */
+  runId: string;
+  /** Case ID from the test */
+  caseId: string;
+  /** Optional case name */
+  caseName?: string;
+  /** Result status */
+  status: CaseResultStatus;
+  /** Number of execution attempts represented by this terminal result. */
+  attempts?: number;
+  /** Score from 0.0 to 1.0 */
+  score: number;
+  /** Type of matcher used */
+  matcherType: string;
+  /** Reason for the status */
+  reason?: string;
+  /** Model response */
+  response: string;
+  /** Latency in milliseconds */
+  latencyMs: number;
+  /** Prompt tokens used */
+  promptTokens: number;
+  /** Completion tokens used */
+  completionTokens: number;
+  /** Total tokens used */
+  totalTokens: number;
+  /** Error message if status is 'error' */
+  error?: string;
+  /** Sanitized evaluator evidence retained for review. */
+  evidence?: CaseEvaluationEvidence;
+  /** Tags for categorization */
+  tags?: string[];
+  /** ISO timestamp when created */
+  createdAt?: string;
+}
+
+/**
+ * Options for querying case results
+ */
+export interface CaseResultQueryOptions {
+  /** Filter by run ID */
+  runId?: string;
+  /** Filter by case ID */
+  caseId?: string;
+  /** Filter by status */
+  status?: CaseResultStatus;
+  /** Filter by tags (any match) */
+  tags?: string[];
+  /** Maximum results to return */
+  limit?: number;
+  /** Offset for pagination */
+  offset?: number;
+}
+
+// ============================================================================
+// Metrics History Types (for trending)
+// ============================================================================
+
+/**
+ * Daily metrics snapshot for a project/scenario
+ */
+export interface MetricsSnapshot {
+  /** Unique ID (auto-generated if not provided) */
+  id?: string;
+  /** Date of the snapshot (YYYY-MM-DD) */
+  date: string;
+  /** Project name */
+  project: string;
+  /** Optional scenario name (null for project-wide) */
+  scenario?: string;
+  /** Total runs on this date */
+  totalRuns: number;
+  /** Total cases across all runs */
+  totalCases: number;
+  /** Total passed cases */
+  passedCases: number;
+  /** Total failed cases */
+  failedCases: number;
+  /** Average success rate */
+  avgSuccessRate: number;
+  /** Average latency in ms */
+  avgLatencyMs: number;
+  /** Average tokens per run */
+  avgTokensPerRun: number;
+  /** Minimum success rate */
+  minSuccessRate?: number;
+  /** Maximum success rate */
+  maxSuccessRate?: number;
+  /** Minimum latency in ms */
+  minLatencyMs?: number;
+  /** Maximum latency in ms */
+  maxLatencyMs?: number;
+  /** Total tokens consumed */
+  totalTokens: number;
+  /** ISO timestamp when created */
+  createdAt?: string;
+  /** ISO timestamp when last updated */
+  updatedAt?: string;
+}
+
+/**
+ * Options for querying metrics history
+ */
+export interface MetricsTrendOptions {
+  /** Project to query */
+  project: string;
+  /** Optional scenario filter */
+  scenario?: string;
+  /** Start date (YYYY-MM-DD) */
+  startDate?: string;
+  /** End date (YYYY-MM-DD) */
+  endDate?: string;
+  /** Maximum results to return */
+  limit?: number;
+}
+
+/**
+ * Trend data point for visualization
+ */
+export interface TrendDataPoint {
+  date: string;
+  successRate: number;
+  latencyMs: number;
+  totalRuns: number;
+  totalTokens: number;
+}
+
+// ============================================================================
+// Enhanced Storage Adapter with Analytics
+// ============================================================================
+
+/**
+ * Extended storage adapter with analytics capabilities
+ */
+export interface AnalyticsStorageAdapter extends BaselineStorageAdapter {
+  /**
+   * Save an individual case result
+   */
+  saveCaseResult(result: CaseResultRecord): Promise<string>;
+
+  /**
+   * Save multiple case results in batch
+   */
+  saveCaseResults(results: CaseResultRecord[]): Promise<string[]>;
+
+  /**
+   * Get case results for a run
+   */
+  getCaseResults(runId: string): Promise<CaseResultRecord[]>;
+
+  /**
+   * Query case results with filters
+   */
+  queryCaseResults(options: CaseResultQueryOptions): Promise<CaseResultRecord[]>;
+
+  /**
+   * Save a metrics snapshot
+   */
+  saveMetricsSnapshot(snapshot: MetricsSnapshot): Promise<string>;
+
+  /**
+   * Get metrics trend data
+   */
+  getMetricsTrend(options: MetricsTrendOptions): Promise<TrendDataPoint[]>;
+
+  /**
+   * Get a specific metrics snapshot
+   */
+  getMetricsSnapshot(
+    date: string,
+    project: string,
+    scenario?: string
+  ): Promise<MetricsSnapshot | null>;
+
+  /**
+   * Aggregate and save daily metrics from runs
+   * This can be called to build/update metrics_history from existing runs
+   */
+  aggregateDailyMetrics?(
+    date: string,
+    project: string,
+    scenario?: string
+  ): Promise<MetricsSnapshot>;
+}

package/src/tools/fixture-executor.test.ts

@@ -0,0 +1,88 @@
+import { describe, expect, it } from 'bun:test';
+import { FixtureToolExecutor } from './fixture-executor';
+
+const tools = [
+  {
+    type: 'function' as const,
+    function: {
+      name: 'get_weather',
+      parameters: {
+        type: 'object',
+        additionalProperties: false,
+        required: ['city'],
+        properties: { city: { type: 'string' } },
+      },
+    },
+  },
+];
+
+const call = (arguments_: string, name = 'get_weather') => ({
+  id: 'call-1',
+  type: 'function' as const,
+  function: { name, arguments: arguments_ },
+});
+
+describe('FixtureToolExecutor', () => {
+  it('returns a deterministic matching fixture result', async () => {
+    const executor = new FixtureToolExecutor({
+      tools,
+      fixtures: { get_weather: [{ when: { city: 'Lagos' }, result: { temperature_c: 28 } }] },
+    });
+
+    await expect(executor.execute(call('{"city":"Lagos"}'))).resolves.toEqual({
+      status: 'success',
+      result: { temperature_c: 28 },
+    });
+  });
+
+  it('rejects undeclared tools, invalid JSON, and unmatched fixtures', async () => {
+    const executor = new FixtureToolExecutor({ tools, fixtures: { get_weather: [] } });
+
+    await expect(executor.execute(call('{}', 'delete_weather'))).resolves.toMatchObject({
+      status: 'error',
+      error: { code: 'TOOL_UNKNOWN' },
+    });
+    await expect(executor.execute(call('{city:Lagos}'))).resolves.toMatchObject({
+      status: 'error',
+      error: { code: 'TOOL_ARGUMENTS_INVALID' },
+    });
+    await expect(executor.execute(call('{"city":"Lagos"}'))).resolves.toMatchObject({
+      status: 'error',
+      error: { code: 'TOOL_FIXTURE_NOT_FOUND' },
+    });
+  });
+
+  it('returns controlled fixture errors and result-size failures', async () => {
+    const errorExecutor = new FixtureToolExecutor({
+      tools,
+      fixtures: { get_weather: [{ error: 'Weather service is unavailable.' }] },
+    });
+    const limitedExecutor = new FixtureToolExecutor({
+      tools,
+      fixtures: { get_weather: [{ result: { forecast: 'x'.repeat(32) } }] },
+      maxToolResultBytes: 16,
+    });
+
+    await expect(errorExecutor.execute(call('{"city":"Lagos"}'))).resolves.toMatchObject({
+      status: 'error',
+      error: { code: 'TOOL_EXECUTION_FAILED' },
+    });
+    await expect(limitedExecutor.execute(call('{"city":"Lagos"}'))).resolves.toMatchObject({
+      status: 'error',
+      error: { code: 'TOOL_RESULT_TOO_LARGE' },
+    });
+  });
+
+  it('rejects unsafe fixture configuration and fixtures for undeclared tools', () => {
+    expect(
+      () =>
+        new FixtureToolExecutor({
+          tools,
+          fixtures: { get_weather: [{ result: { command: 'curl example.invalid' } }] },
+        })
+    ).toThrow('TOOL_FIXTURE_UNSAFE');
+    expect(() => new FixtureToolExecutor({ tools, fixtures: { no_such_tool: [] } })).toThrow(
+      'TOOL_UNKNOWN'
+    );
+  });
+});

package/src/tools/fixture-executor.ts

@@ -0,0 +1,112 @@
+import type { ToolCall, ToolDefinition } from '../adapters/types';
+import { validateToolArguments } from './schema-validator';
+import type {
+  FixtureExecutorOptions,
+  ToolExecutionContext,
+  ToolExecutionResult,
+  ToolFixture,
+  ToolFixtures,
+} from './types';
+
+const DISALLOWED_FIXTURE_FIELDS = new Set(['command', 'url', 'path', 'code', 'environment', 'env']);
+
+export class FixtureToolExecutor {
+  private readonly tools = new Map<string, ToolDefinition>();
+  private readonly fixtures: ToolFixtures;
+  private readonly maxToolResultBytes: number;
+
+  constructor({ tools, fixtures, maxToolResultBytes = 32_768 }: FixtureExecutorOptions) {
+    for (const tool of tools) this.tools.set(tool.function.name, tool);
+    for (const [name, entries] of Object.entries(fixtures)) {
+      if (!this.tools.has(name)) {
+        throw new Error(`TOOL_UNKNOWN: fixture declared for undeclared tool '${name}'`);
+      }
+      entries.forEach(assertSafeFixture);
+    }
+    this.fixtures = fixtures;
+    this.maxToolResultBytes = maxToolResultBytes;
+  }
+
+  async execute(call: ToolCall, _context?: ToolExecutionContext): Promise<ToolExecutionResult> {
+    const tool = this.tools.get(call.function.name);
+    if (!tool) {
+      return error('TOOL_UNKNOWN', `Tool '${call.function.name}' is not declared.`);
+    }
+
+    const validated = validateToolArguments(call.function.arguments, tool.function.parameters);
+    if (!validated.valid) return { status: 'error', error: validated.error };
+
+    const fixture = this.fixtures[call.function.name]?.find((candidate) =>
+      matches(candidate.when, validated.arguments ?? {})
+    );
+    if (!fixture) {
+      return error('TOOL_FIXTURE_NOT_FOUND', `No fixture matched tool '${call.function.name}'.`);
+    }
+    if (fixture.error) return error('TOOL_EXECUTION_FAILED', fixture.error);
+
+    const serialized = JSON.stringify(fixture.result ?? {});
+    if (
+      serialized === undefined ||
+      new TextEncoder().encode(serialized).byteLength > this.maxToolResultBytes
+    ) {
+      return error(
+        'TOOL_RESULT_TOO_LARGE',
+        `Tool '${call.function.name}' result exceeds the configured limit.`
+      );
+    }
+    return { status: 'success', result: fixture.result ?? {} };
+  }
+}
+
+function error(
+  code: NonNullable<ToolExecutionResult['error']>['code'],
+  message: string
+): ToolExecutionResult {
+  return { status: 'error', error: { code, message } };
+}
+
+function matches(
+  when: Record<string, unknown> | undefined,
+  args: Record<string, unknown>
+): boolean {
+  return !when || Object.entries(when).every(([key, value]) => deepEqual(args[key], value));
+}
+
+function deepEqual(left: unknown, right: unknown): boolean {
+  if (Object.is(left, right)) return true;
+  if (!left || !right || typeof left !== 'object' || typeof right !== 'object') return false;
+  if (Array.isArray(left) || Array.isArray(right)) {
+    return (
+      Array.isArray(left) &&
+      Array.isArray(right) &&
+      left.length === right.length &&
+      left.every((value, index) => deepEqual(value, right[index]))
+    );
+  }
+  const leftRecord = left as Record<string, unknown>;
+  const rightRecord = right as Record<string, unknown>;
+  const keys = Object.keys(rightRecord);
+  return (
+    keys.length === Object.keys(leftRecord).length &&
+    keys.every((key) => deepEqual(leftRecord[key], rightRecord[key]))
+  );
+}
+
+function assertSafeFixture(fixture: ToolFixture): void {
+  assertSafeValue(fixture.when);
+  assertSafeValue(fixture.result);
+}
+
+function assertSafeValue(value: unknown): void {
+  if (!value || typeof value !== 'object') return;
+  if (Array.isArray(value)) {
+    value.forEach(assertSafeValue);
+    return;
+  }
+  for (const [key, child] of Object.entries(value as Record<string, unknown>)) {
+    if (DISALLOWED_FIXTURE_FIELDS.has(key.toLowerCase())) {
+      throw new Error(`TOOL_FIXTURE_UNSAFE: '${key}' is not permitted in fixture data.`);
+    }
+    assertSafeValue(child);
+  }
+}

package/src/tools/index.ts

@@ -0,0 +1,3 @@
+export * from './fixture-executor';
+export * from './schema-validator';
+export * from './types';

package/src/tools/schema-validator.test.ts

@@ -0,0 +1,32 @@
+import { describe, expect, it } from 'bun:test';
+import { validateToolArguments } from './schema-validator';
+
+const weatherSchema = {
+  type: 'object',
+  additionalProperties: false,
+  required: ['city'],
+  properties: { city: { type: 'string', minLength: 1 } },
+};
+
+describe('validateToolArguments', () => {
+  it('returns parsed arguments that satisfy the JSON schema', () => {
+    expect(validateToolArguments('{"city":"Lagos"}', weatherSchema)).toEqual({
+      valid: true,
+      arguments: { city: 'Lagos' },
+    });
+  });
+
+  it('rejects malformed JSON without echoing the arguments', () => {
+    expect(validateToolArguments('{city:Lagos}', weatherSchema)).toEqual({
+      valid: false,
+      error: { code: 'TOOL_ARGUMENTS_INVALID', message: 'Tool arguments must be valid JSON.' },
+    });
+  });
+
+  it('rejects arguments that fail the declared schema', () => {
+    const result = validateToolArguments('{"city":42}', weatherSchema);
+
+    expect(result.valid).toBe(false);
+    expect(result.error?.code).toBe('TOOL_ARGUMENTS_SCHEMA_INVALID');
+  });
+});

package/src/tools/schema-validator.ts

@@ -0,0 +1,56 @@
+import Ajv, { type ValidateFunction } from 'ajv';
+
+export interface ToolArgumentValidation {
+  valid: boolean;
+  arguments?: Record<string, unknown>;
+  error?: { code: 'TOOL_ARGUMENTS_INVALID' | 'TOOL_ARGUMENTS_SCHEMA_INVALID'; message: string };
+}
+
+const ajv = new Ajv({ allErrors: true, strict: false });
+const validators = new WeakMap<object, ValidateFunction>();
+
+function getValidator(schema: Record<string, unknown>): ValidateFunction {
+  const cached = validators.get(schema);
+  if (cached) return cached;
+  const validator = ajv.compile(schema);
+  validators.set(schema, validator);
+  return validator;
+}
+
+export function validateToolArguments(
+  rawArguments: string,
+  schema: Record<string, unknown>
+): ToolArgumentValidation {
+  let args: unknown;
+  try {
+    args = JSON.parse(rawArguments);
+  } catch {
+    return {
+      valid: false,
+      error: { code: 'TOOL_ARGUMENTS_INVALID', message: 'Tool arguments must be valid JSON.' },
+    };
+  }
+
+  if (!args || typeof args !== 'object' || Array.isArray(args)) {
+    return {
+      valid: false,
+      error: { code: 'TOOL_ARGUMENTS_INVALID', message: 'Tool arguments must be a JSON object.' },
+    };
+  }
+
+  const validator = getValidator(schema);
+  if (!validator(args)) {
+    const details = (validator.errors ?? [])
+      .map((error) => `${error.instancePath || '/'} ${error.message ?? 'is invalid'}`)
+      .join('; ');
+    return {
+      valid: false,
+      error: {
+        code: 'TOOL_ARGUMENTS_SCHEMA_INVALID',
+        message: `Tool arguments do not match the schema: ${details}`,
+      },
+    };
+  }
+
+  return { valid: true, arguments: args as Record<string, unknown> };
+}

package/src/tools/types.ts

@@ -0,0 +1,80 @@
+import type { ToolCall, ToolDefinition } from '../adapters/types';
+
+export type ToolLoopTerminationReason =
+  | 'completed'
+  | 'max_steps'
+  | 'timeout'
+  | 'duplicate_call'
+  | 'invalid_arguments'
+  | 'unknown_tool'
+  | 'tool_error';
+
+export interface ToolLoopPolicy {
+  enabled: boolean;
+  maxSteps: number;
+  timeoutMs: number;
+  maxToolResultBytes: number;
+  rejectDuplicateCalls: boolean;
+}
+
+export const DEFAULT_TOOL_LOOP_POLICY: ToolLoopPolicy = {
+  enabled: false,
+  maxSteps: 5,
+  timeoutMs: 60_000,
+  maxToolResultBytes: 32_768,
+  rejectDuplicateCalls: true,
+};
+
+export interface ToolExecutionContext {
+  caseId: string;
+  step: number;
+}
+
+export interface ToolExecutionError {
+  code:
+    | 'TOOL_ARGUMENTS_INVALID'
+    | 'TOOL_ARGUMENTS_SCHEMA_INVALID'
+    | 'TOOL_UNKNOWN'
+    | 'TOOL_FIXTURE_NOT_FOUND'
+    | 'TOOL_EXECUTION_FAILED'
+    | 'TOOL_RESULT_TOO_LARGE';
+  message: string;
+}
+
+export interface ToolExecutionResult {
+  status: 'success' | 'error';
+  result?: unknown;
+  error?: ToolExecutionError;
+}
+
+export interface ToolExecutor {
+  execute(call: ToolCall, context: ToolExecutionContext): Promise<ToolExecutionResult>;
+}
+
+export interface ToolTraceEntry {
+  step: number;
+  toolCall: ToolCall;
+  result?: unknown;
+  error?: ToolExecutionError;
+  latencyMs: number;
+}
+
+export interface ToolLoopSummary {
+  status: 'completed' | 'error';
+  steps: number;
+  terminationReason: ToolLoopTerminationReason;
+}
+
+export interface ToolFixture {
+  when?: Record<string, unknown>;
+  result?: unknown;
+  error?: string;
+}
+
+export type ToolFixtures = Record<string, ToolFixture[]>;
+
+export interface FixtureExecutorOptions {
+  tools: ToolDefinition[];
+  fixtures: ToolFixtures;
+  maxToolResultBytes?: number;
+}