@artemiskit/core 0.1.2

package/src/evaluators/index.ts

@@ -0,0 +1,53 @@
+/**
+ * Evaluators module - exports all evaluator types and utilities
+ */
+
+import { ContainsEvaluator } from './contains';
+import { ExactEvaluator } from './exact';
+import { FuzzyEvaluator } from './fuzzy';
+import { JsonSchemaEvaluator } from './json-schema';
+import { LLMGraderEvaluator } from './llm-grader';
+import { RegexEvaluator } from './regex';
+import type { Evaluator } from './types';
+
+const evaluators = new Map<string, Evaluator>();
+evaluators.set('exact', new ExactEvaluator());
+evaluators.set('regex', new RegexEvaluator());
+evaluators.set('fuzzy', new FuzzyEvaluator());
+evaluators.set('contains', new ContainsEvaluator());
+evaluators.set('json_schema', new JsonSchemaEvaluator());
+evaluators.set('llm_grader', new LLMGraderEvaluator());
+
+/**
+ * Get an evaluator by type
+ */
+export function getEvaluator(type: string): Evaluator {
+  const evaluator = evaluators.get(type);
+  if (!evaluator) {
+    const available = Array.from(evaluators.keys()).join(', ');
+    throw new Error(`Unknown evaluator type: ${type}. Available: ${available}`);
+  }
+  return evaluator;
+}
+
+/**
+ * Register a custom evaluator
+ */
+export function registerEvaluator(type: string, evaluator: Evaluator): void {
+  evaluators.set(type, evaluator);
+}
+
+/**
+ * List available evaluator types
+ */
+export function listEvaluators(): string[] {
+  return Array.from(evaluators.keys());
+}
+
+export * from './types';
+export { ExactEvaluator } from './exact';
+export { RegexEvaluator } from './regex';
+export { FuzzyEvaluator } from './fuzzy';
+export { ContainsEvaluator } from './contains';
+export { JsonSchemaEvaluator } from './json-schema';
+export { LLMGraderEvaluator } from './llm-grader';

package/src/evaluators/json-schema.ts

@@ -0,0 +1,98 @@
+/**
+ * JSON Schema evaluator - validates response against a JSON schema
+ */
+
+import { z } from 'zod';
+import type { Expected } from '../scenario/schema';
+import type { Evaluator, EvaluatorResult } from './types';
+
+export class JsonSchemaEvaluator implements Evaluator {
+  readonly type = 'json_schema';
+
+  async evaluate(response: string, expected: Expected): Promise<EvaluatorResult> {
+    if (expected.type !== 'json_schema') {
+      throw new Error('Invalid expected type for JsonSchemaEvaluator');
+    }
+
+    let parsed: unknown;
+    try {
+      const jsonMatch = response.match(/```(?:json)?\s*([\s\S]*?)```/);
+      const jsonStr = jsonMatch ? jsonMatch[1].trim() : response.trim();
+      parsed = JSON.parse(jsonStr);
+    } catch (error) {
+      return {
+        passed: false,
+        score: 0,
+        reason: `Failed to parse JSON: ${(error as Error).message}`,
+        details: { parseError: (error as Error).message },
+      };
+    }
+
+    try {
+      const zodSchema = this.jsonSchemaToZod(expected.schema);
+      const result = zodSchema.safeParse(parsed);
+
+      if (result.success) {
+        return {
+          passed: true,
+          score: 1,
+          reason: 'Response matches JSON schema',
+          details: { parsed },
+        };
+      }
+      const issues = result.error.issues.map((i) => `${i.path.join('.')}: ${i.message}`);
+      return {
+        passed: false,
+        score: 0,
+        reason: `Schema validation failed: ${issues.join(', ')}`,
+        details: {
+          parsed,
+          errors: issues,
+        },
+      };
+    } catch (error) {
+      return {
+        passed: false,
+        score: 0,
+        reason: `Schema error: ${(error as Error).message}`,
+        details: { error: (error as Error).message },
+      };
+    }
+  }
+
+  private jsonSchemaToZod(schema: Record<string, unknown>): z.ZodTypeAny {
+    const type = schema.type as string;
+
+    switch (type) {
+      case 'string':
+        return z.string();
+      case 'number':
+        return z.number();
+      case 'integer':
+        return z.number().int();
+      case 'boolean':
+        return z.boolean();
+      case 'null':
+        return z.null();
+      case 'array':
+        if (schema.items) {
+          return z.array(this.jsonSchemaToZod(schema.items as Record<string, unknown>));
+        }
+        return z.array(z.unknown());
+      case 'object':
+        if (schema.properties) {
+          const shape: Record<string, z.ZodTypeAny> = {};
+          const required = (schema.required as string[]) || [];
+
+          for (const [key, value] of Object.entries(schema.properties as Record<string, unknown>)) {
+            const fieldSchema = this.jsonSchemaToZod(value as Record<string, unknown>);
+            shape[key] = required.includes(key) ? fieldSchema : fieldSchema.optional();
+          }
+          return z.object(shape);
+        }
+        return z.record(z.unknown());
+      default:
+        return z.unknown();
+    }
+  }
+}

package/src/evaluators/llm-grader.ts

@@ -0,0 +1,100 @@
+/**
+ * LLM-based grader evaluator
+ */
+
+import type { Expected } from '../scenario/schema';
+import type { Evaluator, EvaluatorContext, EvaluatorResult } from './types';
+
+const GRADER_PROMPT = `You are an evaluator grading an AI response based on a rubric.
+
+## RUBRIC
+{{rubric}}
+
+## RESPONSE TO EVALUATE
+{{response}}
+
+## INSTRUCTIONS
+Score the response from 0.0 to 1.0 based on the rubric.
+Be objective and consistent in your scoring.
+
+Respond with ONLY a JSON object in this exact format:
+{"score": <number between 0 and 1>, "reason": "<brief explanation of score>"}
+
+Do not include any other text, markdown, or formatting.`;
+
+export class LLMGraderEvaluator implements Evaluator {
+  readonly type = 'llm_grader';
+
+  async evaluate(
+    response: string,
+    expected: Expected,
+    context?: EvaluatorContext
+  ): Promise<EvaluatorResult> {
+    if (expected.type !== 'llm_grader') {
+      throw new Error('Invalid expected type for LLMGraderEvaluator');
+    }
+
+    if (!context?.client) {
+      throw new Error('LLM grader requires a ModelClient in context');
+    }
+
+    const prompt = GRADER_PROMPT.replace('{{rubric}}', expected.rubric).replace(
+      '{{response}}',
+      response
+    );
+
+    try {
+      const result = await context.client.generate({
+        prompt,
+        model: expected.model,
+        temperature: 0,
+        maxTokens: 200,
+      });
+
+      const parsed = this.parseGraderResponse(result.text);
+      const passed = parsed.score >= expected.threshold;
+
+      return {
+        passed,
+        score: parsed.score,
+        reason: parsed.reason || `Score: ${parsed.score.toFixed(2)}`,
+        details: {
+          graderResponse: result.text,
+          rubric: expected.rubric,
+          threshold: expected.threshold,
+          model: result.model,
+        },
+      };
+    } catch (error) {
+      return {
+        passed: false,
+        score: 0,
+        reason: `Grader failed: ${(error as Error).message}`,
+        details: { error: (error as Error).message },
+      };
+    }
+  }
+
+  private parseGraderResponse(text: string): { score: number; reason?: string } {
+    const jsonMatch = text.match(/\{[\s\S]*?\}/);
+    if (!jsonMatch) {
+      throw new Error('No JSON found in grader response');
+    }
+
+    try {
+      const parsed = JSON.parse(jsonMatch[0]);
+      const score = Number(parsed.score);
+
+      if (Number.isNaN(score) || score < 0 || score > 1) {
+        throw new Error(`Invalid score: ${parsed.score}`);
+      }
+
+      return {
+        score,
+        reason: parsed.reason,
+      };
+    } catch (error) {
+      throw new Error(`Failed to parse grader response: ${(error as Error).message}`);
+    }
+  }
+}

package/src/evaluators/regex.test.ts

@@ -0,0 +1,73 @@
+/**
+ * Tests for RegexEvaluator
+ */
+
+import { describe, expect, test } from 'bun:test';
+import { RegexEvaluator } from './regex';
+
+describe('RegexEvaluator', () => {
+  const evaluator = new RegexEvaluator();
+
+  test('passes when pattern matches', async () => {
+    const result = await evaluator.evaluate('The answer is 42', {
+      type: 'regex',
+      pattern: '\\d+',
+    });
+    expect(result.passed).toBe(true);
+    expect(result.score).toBe(1);
+    expect(result.details?.match).toBe('42');
+  });
+
+  test('fails when pattern does not match', async () => {
+    const result = await evaluator.evaluate('No numbers here', {
+      type: 'regex',
+      pattern: '\\d+',
+    });
+    expect(result.passed).toBe(false);
+    expect(result.score).toBe(0);
+  });
+
+  test('supports regex flags', async () => {
+    const result = await evaluator.evaluate('HELLO WORLD', {
+      type: 'regex',
+      pattern: 'hello',
+      flags: 'i',
+    });
+    expect(result.passed).toBe(true);
+  });
+
+  test('captures named groups', async () => {
+    const result = await evaluator.evaluate('Name: John, Age: 30', {
+      type: 'regex',
+      pattern: 'Name: (?<name>\\w+), Age: (?<age>\\d+)',
+    });
+    expect(result.passed).toBe(true);
+    expect(result.details?.groups?.name).toBe('John');
+    expect(result.details?.groups?.age).toBe('30');
+  });
+
+  test('handles invalid regex gracefully', async () => {
+    const result = await evaluator.evaluate('test', {
+      type: 'regex',
+      pattern: '[invalid',
+    });
+    expect(result.passed).toBe(false);
+    expect(result.reason).toContain('Invalid regex');
+  });
+
+  test('matches exact patterns', async () => {
+    const result = await evaluator.evaluate('42', {
+      type: 'regex',
+      pattern: '^\\d+$',
+    });
+    expect(result.passed).toBe(true);
+  });
+
+  test('fails partial match with anchors', async () => {
+    const result = await evaluator.evaluate('The answer is 42', {
+      type: 'regex',
+      pattern: '^\\d+$',
+    });
+    expect(result.passed).toBe(false);
+  });
+});

package/src/evaluators/regex.ts

@@ -0,0 +1,43 @@
+/**
+ * Regex pattern evaluator
+ */
+
+import type { Expected } from '../scenario/schema';
+import type { Evaluator, EvaluatorResult } from './types';
+
+export class RegexEvaluator implements Evaluator {
+  readonly type = 'regex';
+
+  async evaluate(response: string, expected: Expected): Promise<EvaluatorResult> {
+    if (expected.type !== 'regex') {
+      throw new Error('Invalid expected type for RegexEvaluator');
+    }
+
+    try {
+      const regex = new RegExp(expected.pattern, expected.flags);
+      const match = regex.exec(response);
+      const passed = match !== null;
+
+      return {
+        passed,
+        score: passed ? 1 : 0,
+        reason: passed
+          ? `Matched pattern: ${expected.pattern}`
+          : `Did not match pattern: ${expected.pattern}`,
+        details: {
+          pattern: expected.pattern,
+          flags: expected.flags,
+          match: match ? match[0] : null,
+          groups: match ? match.groups : null,
+        },
+      };
+    } catch (error) {
+      return {
+        passed: false,
+        score: 0,
+        reason: `Invalid regex pattern: ${(error as Error).message}`,
+        details: { error: (error as Error).message },
+      };
+    }
+  }
+}

package/src/evaluators/types.ts

@@ -0,0 +1,37 @@
+/**
+ * Evaluator types and interfaces
+ */
+
+import type { ModelClient } from '../adapters/types';
+import type { Expected, TestCase } from '../scenario/schema';
+
+/**
+ * Context provided to evaluators
+ */
+export interface EvaluatorContext {
+  client?: ModelClient;
+  testCase?: TestCase;
+}
+
+/**
+ * Result from an evaluation
+ */
+export interface EvaluatorResult {
+  passed: boolean;
+  score: number;
+  reason?: string;
+  details?: Record<string, unknown>;
+}
+
+/**
+ * Evaluator interface - implement to create custom evaluators
+ */
+export interface Evaluator {
+  readonly type: string;
+
+  evaluate(
+    response: string,
+    expected: Expected,
+    context?: EvaluatorContext
+  ): Promise<EvaluatorResult>;
+}

package/src/index.ts

@@ -0,0 +1,31 @@
+/**
+ * @artemiskit/core
+ * Core library for Artemis Agent Reliability Toolkit
+ */
+
+// Adapter types and factory
+export * from './adapters';
+
+// Scenario parsing
+export * from './scenario';
+
+// Evaluators
+export * from './evaluators';
+
+// Runner
+export * from './runner';
+
+// Storage
+export * from './storage';
+
+// Artifacts
+export * from './artifacts';
+
+// Provenance
+export * from './provenance';
+
+// Utilities
+export * from './utils';
+
+// Redaction
+export * from './redaction';

package/src/provenance/environment.ts

@@ -0,0 +1,18 @@
+/**
+ * Environment information utilities
+ */
+
+/**
+ * Get environment information
+ */
+export function getEnvironmentInfo(): {
+  node_version: string;
+  platform: string;
+  arch: string;
+} {
+  return {
+    node_version: process.version,
+    platform: process.platform,
+    arch: process.arch,
+  };
+}

package/src/provenance/git.ts

@@ -0,0 +1,48 @@
+/**
+ * Git provenance utilities
+ */
+
+import { execSync } from 'node:child_process';
+import type { GitInfo } from '../artifacts/types';
+
+/**
+ * Get git information for the current repository
+ */
+export function getGitInfo(): GitInfo {
+  try {
+    const commit = execGit('rev-parse HEAD');
+    const branch = execGit('rev-parse --abbrev-ref HEAD');
+    const dirty = execGit('status --porcelain').length > 0;
+    const remote = execGit('remote get-url origin', true);
+
+    return {
+      commit,
+      branch,
+      dirty,
+      remote: remote || undefined,
+    };
+  } catch {
+    return {
+      commit: 'unknown',
+      branch: 'unknown',
+      dirty: false,
+    };
+  }
+}
+
+/**
+ * Execute a git command
+ */
+function execGit(command: string, allowFailure = false): string {
+  try {
+    return execSync(`git ${command}`, {
+      encoding: 'utf-8',
+      stdio: ['pipe', 'pipe', 'pipe'],
+    }).trim();
+  } catch {
+    if (allowFailure) {
+      return '';
+    }
+    throw new Error(`Git command failed: ${command}`);
+  }
+}

package/src/provenance/index.ts

@@ -0,0 +1,6 @@
+/**
+ * Provenance module exports
+ */
+
+export { getGitInfo } from './git';
+export { getEnvironmentInfo } from './environment';

package/src/redaction/index.ts

@@ -0,0 +1,23 @@
+export {
+  BUILTIN_PATTERNS,
+  BUILTIN_REGEX_PATTERNS,
+  DEFAULT_REDACTION_PATTERNS,
+  RedactionConfigSchema,
+  type BuiltinPatternName,
+  type CaseRedactionDetails,
+  type RedactionConfig,
+  type RedactionMetadata,
+  type RedactionOptions,
+  type RedactionResult,
+} from './types';
+
+export {
+  Redactor,
+  createDefaultRedactor,
+  createNoOpRedactor,
+  createRedactionOptions,
+  hashText,
+  redactText,
+  redactWithHash,
+  resolvePatterns,
+} from './redactor';