@artemiskit/core 0.2.0 → 0.2.2

package/src/evaluators/llm-grader.ts

@@ -5,22 +5,27 @@
 import type { Expected } from '../scenario/schema';
 import type { Evaluator, EvaluatorContext, EvaluatorResult } from './types';
 
-const GRADER_PROMPT = `You are an evaluator grading an AI response based on a rubric.
+const GRADER_PROMPT = `You are a strict JSON-only evaluator. You grade AI responses based on rubrics.
 
-## RUBRIC
+RUBRIC:
 {{rubric}}
 
-## RESPONSE TO EVALUATE
+RESPONSE TO EVALUATE:
 {{response}}
 
-## INSTRUCTIONS
-Score the response from 0.0 to 1.0 based on the rubric.
-Be objective and consistent in your scoring.
+TASK: Score the response from 0.0 to 1.0 based on the rubric above.
 
-Respond with ONLY a JSON object in this exact format:
-{"score": <number between 0 and 1>, "reason": "<brief explanation of score>"}
+OUTPUT FORMAT: You MUST respond with ONLY this exact JSON structure, nothing else:
+{"score":0.0,"reason":"explanation"}
 
-Do not include any other text, markdown, or formatting.`;
+RULES:
+- Output ONLY valid JSON, no markdown, no code blocks, no extra text
+- "score" must be a number between 0.0 and 1.0
+- "reason" must be a brief string explaining the score
+- Do NOT wrap in \`\`\`json or any formatting
+- Your entire response must be parseable by JSON.parse()
+
+JSON OUTPUT:`;
 
 export class LLMGraderEvaluator implements Evaluator {
   readonly type = 'llm_grader';
@@ -44,11 +49,13 @@ export class LLMGraderEvaluator implements Evaluator {
     );
 
     try {
+      // Note: Some models (like o1, o3, gpt-5-mini, reasoning models) only support temperature=1
+      // We omit temperature to let the API use its default for maximum compatibility
+      // Use higher maxTokens for reasoning models which use tokens for internal "thinking"
       const result = await context.client.generate({
         prompt,
         model: expected.model,
-        temperature: 0,
-        maxTokens: 200,
+        maxTokens: 1000,
       });
 
       const parsed = this.parseGraderResponse(result.text);
@@ -76,9 +83,25 @@ export class LLMGraderEvaluator implements Evaluator {
   }
 
   private parseGraderResponse(text: string): { score: number; reason?: string } {
-    const jsonMatch = text.match(/\{[\s\S]*?\}/);
+    // Clean up the response - remove markdown code blocks if present
+    const cleanedText = text
+      .replace(/```json\s*/gi, '')
+      .replace(/```\s*/g, '')
+      .trim();
+
+    // Try to find JSON object in the response
+    const jsonMatch = cleanedText.match(/\{[\s\S]*?\}/);
+
     if (!jsonMatch) {
-      throw new Error('No JSON found in grader response');
+      // Fallback: try to extract score from plain text patterns like "Score: 0.8" or "0.85"
+      const scoreMatch = cleanedText.match(/(?:score[:\s]*)?(\d+\.?\d*)/i);
+      if (scoreMatch) {
+        const score = Number(scoreMatch[1]);
+        if (!Number.isNaN(score) && score >= 0 && score <= 1) {
+          return { score, reason: cleanedText };
+        }
+      }
+      throw new Error(`No JSON found in grader response: ${text.substring(0, 100)}...`);
     }
 
     try {
@@ -94,6 +117,15 @@ export class LLMGraderEvaluator implements Evaluator {
         reason: parsed.reason,
       };
     } catch (error) {
+      // If JSON parsing fails, try extracting score directly
+      const scoreMatch = jsonMatch[0].match(/"score"[:\s]*(\d+\.?\d*)/i);
+      if (scoreMatch) {
+        const score = Number(scoreMatch[1]);
+        if (!Number.isNaN(score) && score >= 0 && score <= 1) {
+          const reasonMatch = jsonMatch[0].match(/"reason"[:\s]*"([^"]+)"/i);
+          return { score, reason: reasonMatch?.[1] };
+        }
+      }
       throw new Error(`Failed to parse grader response: ${(error as Error).message}`);
     }
   }

package/src/scenario/schema.ts

@@ -44,6 +44,10 @@ export const ProviderConfigSchema = z
     apiVersion: z.string().optional(),
     embeddingDeploymentName: z.string().optional(),
 
+    // Model family for parameter detection (e.g., 'gpt-5-mini' when deployment is '5-mini')
+    // Used by OpenAI/Azure to determine which API parameters to use (max_tokens vs max_completion_tokens)
+    modelFamily: z.string().optional(),
+
     // Vercel AI specific
     underlyingProvider: z.enum(['openai', 'azure', 'anthropic', 'google', 'mistral']).optional(),
   })

package/src/storage/local.test.ts

@@ -162,4 +162,247 @@ describe('LocalStorageAdapter', () => {
     const runs = await emptyStorage.list();
     expect(runs).toEqual([]);
   });
+
+  // ==================== Baseline Tests ====================
+
+  describe('Baseline Management', () => {
+    const baselineManifest: RunManifest = {
+      ...mockManifest,
+      run_id: 'baseline-test-run',
+      config: {
+        ...mockManifest.config,
+        scenario: 'baseline-test-scenario',
+      },
+    };
+
+    test('sets a baseline for a scenario', async () => {
+      await storage.save(baselineManifest);
+
+      const baseline = await storage.setBaseline('baseline-test-scenario', 'baseline-test-run');
+
+      expect(baseline.scenario).toBe('baseline-test-scenario');
+      expect(baseline.runId).toBe('baseline-test-run');
+      expect(baseline.metrics.successRate).toBe(0.8);
+      expect(baseline.metrics.totalCases).toBe(10);
+      expect(baseline.metrics.passedCases).toBe(8);
+      expect(baseline.createdAt).toBeDefined();
+    });
+
+    test('sets a baseline with a tag', async () => {
+      await storage.save(baselineManifest);
+
+      const baseline = await storage.setBaseline(
+        'baseline-test-scenario',
+        'baseline-test-run',
+        'v1.0.0-release'
+      );
+
+      expect(baseline.tag).toBe('v1.0.0-release');
+    });
+
+    test('gets a baseline for a scenario', async () => {
+      await storage.save(baselineManifest);
+      await storage.setBaseline('baseline-test-scenario', 'baseline-test-run');
+
+      const baseline = await storage.getBaseline('baseline-test-scenario');
+
+      expect(baseline).not.toBeNull();
+      expect(baseline?.runId).toBe('baseline-test-run');
+      expect(baseline?.metrics.successRate).toBe(0.8);
+    });
+
+    test('returns null for non-existent baseline', async () => {
+      const baseline = await storage.getBaseline('non-existent-scenario');
+      expect(baseline).toBeNull();
+    });
+
+    test('gets a baseline by run ID', async () => {
+      await storage.save(baselineManifest);
+      await storage.setBaseline('baseline-test-scenario', 'baseline-test-run', 'by-run-id-test');
+
+      const baseline = await storage.getBaselineByRunId('baseline-test-run');
+
+      expect(baseline).not.toBeNull();
+      expect(baseline?.scenario).toBe('baseline-test-scenario');
+      expect(baseline?.runId).toBe('baseline-test-run');
+      expect(baseline?.tag).toBe('by-run-id-test');
+    });
+
+    test('returns null for non-existent baseline by run ID', async () => {
+      const baseline = await storage.getBaselineByRunId('non-existent-run-id');
+      expect(baseline).toBeNull();
+    });
+
+    test('lists all baselines', async () => {
+      // Clear any existing baselines by creating a new storage instance
+      const freshStorage = new LocalStorageAdapter(TEST_DIR);
+
+      // Create multiple runs and baselines
+      const manifest1 = {
+        ...mockManifest,
+        run_id: 'list-baseline-1',
+        config: { ...mockManifest.config, scenario: 'scenario-1' },
+      };
+      const manifest2 = {
+        ...mockManifest,
+        run_id: 'list-baseline-2',
+        config: { ...mockManifest.config, scenario: 'scenario-2' },
+      };
+
+      await freshStorage.save(manifest1);
+      await freshStorage.save(manifest2);
+      await freshStorage.setBaseline('scenario-1', 'list-baseline-1');
+      await freshStorage.setBaseline('scenario-2', 'list-baseline-2');
+
+      const baselines = await freshStorage.listBaselines();
+
+      expect(baselines.length).toBeGreaterThanOrEqual(2);
+      expect(baselines.some((b) => b.scenario === 'scenario-1')).toBe(true);
+      expect(baselines.some((b) => b.scenario === 'scenario-2')).toBe(true);
+    });
+
+    test('removes a baseline', async () => {
+      await storage.save(baselineManifest);
+      await storage.setBaseline('baseline-test-scenario', 'baseline-test-run');
+
+      // Verify it exists
+      const before = await storage.getBaseline('baseline-test-scenario');
+      expect(before).not.toBeNull();
+
+      // Remove
+      const removed = await storage.removeBaseline('baseline-test-scenario');
+      expect(removed).toBe(true);
+
+      // Verify it's gone
+      const after = await storage.getBaseline('baseline-test-scenario');
+      expect(after).toBeNull();
+    });
+
+    test('returns false when removing non-existent baseline', async () => {
+      const removed = await storage.removeBaseline('non-existent-baseline');
+      expect(removed).toBe(false);
+    });
+
+    test('removes a baseline by run ID', async () => {
+      const removeByIdManifest = {
+        ...mockManifest,
+        run_id: 'remove-by-id-run',
+        config: { ...mockManifest.config, scenario: 'remove-by-id-scenario' },
+      };
+      await storage.save(removeByIdManifest);
+      await storage.setBaseline('remove-by-id-scenario', 'remove-by-id-run');
+
+      // Verify it exists
+      const before = await storage.getBaselineByRunId('remove-by-id-run');
+      expect(before).not.toBeNull();
+
+      // Remove by run ID
+      const removed = await storage.removeBaselineByRunId('remove-by-id-run');
+      expect(removed).toBe(true);
+
+      // Verify it's gone
+      const after = await storage.getBaselineByRunId('remove-by-id-run');
+      expect(after).toBeNull();
+    });
+
+    test('returns false when removing non-existent baseline by run ID', async () => {
+      const removed = await storage.removeBaselineByRunId('non-existent-run-id');
+      expect(removed).toBe(false);
+    });
+
+    test('updates existing baseline when set again', async () => {
+      await storage.save(baselineManifest);
+
+      // Set initial baseline
+      await storage.setBaseline('baseline-test-scenario', 'baseline-test-run', 'v1.0');
+
+      // Create a new run with different metrics
+      const newManifest = {
+        ...baselineManifest,
+        run_id: 'baseline-test-run-2',
+        metrics: {
+          ...baselineManifest.metrics,
+          success_rate: 0.95,
+          passed_cases: 9,
+          failed_cases: 1,
+        },
+      };
+      await storage.save(newManifest);
+
+      // Update baseline
+      await storage.setBaseline('baseline-test-scenario', 'baseline-test-run-2', 'v2.0');
+
+      const baseline = await storage.getBaseline('baseline-test-scenario');
+      expect(baseline?.runId).toBe('baseline-test-run-2');
+      expect(baseline?.tag).toBe('v2.0');
+      expect(baseline?.metrics.successRate).toBe(0.95);
+    });
+
+    test('compares run to baseline and detects no regression', async () => {
+      // Create baseline run
+      const baselineRun = {
+        ...mockManifest,
+        run_id: 'compare-baseline-run',
+        config: { ...mockManifest.config, scenario: 'compare-scenario' },
+        metrics: { ...mockManifest.metrics, success_rate: 0.8 },
+      };
+      await storage.save(baselineRun);
+      await storage.setBaseline('compare-scenario', 'compare-baseline-run');
+
+      // Create current run with same or better success rate
+      const currentRun = {
+        ...mockManifest,
+        run_id: 'compare-current-run',
+        config: { ...mockManifest.config, scenario: 'compare-scenario' },
+        metrics: { ...mockManifest.metrics, success_rate: 0.85 },
+      };
+      await storage.save(currentRun);
+
+      const result = await storage.compareToBaseline('compare-current-run', 0.05);
+
+      expect(result).not.toBeNull();
+      expect(result?.hasRegression).toBe(false);
+      expect(result?.comparison.delta.successRate).toBeCloseTo(0.05, 2);
+    });
+
+    test('compares run to baseline and detects regression', async () => {
+      // Create baseline run
+      const baselineRun = {
+        ...mockManifest,
+        run_id: 'regression-baseline-run',
+        config: { ...mockManifest.config, scenario: 'regression-scenario' },
+        metrics: { ...mockManifest.metrics, success_rate: 0.9 },
+      };
+      await storage.save(baselineRun);
+      await storage.setBaseline('regression-scenario', 'regression-baseline-run');
+
+      // Create current run with worse success rate (regression)
+      const currentRun = {
+        ...mockManifest,
+        run_id: 'regression-current-run',
+        config: { ...mockManifest.config, scenario: 'regression-scenario' },
+        metrics: { ...mockManifest.metrics, success_rate: 0.7 },
+      };
+      await storage.save(currentRun);
+
+      const result = await storage.compareToBaseline('regression-current-run', 0.05);
+
+      expect(result).not.toBeNull();
+      expect(result?.hasRegression).toBe(true);
+      expect(result?.comparison.delta.successRate).toBeCloseTo(-0.2, 2);
+    });
+
+    test('returns null when no baseline exists for scenario', async () => {
+      const noBaselineRun = {
+        ...mockManifest,
+        run_id: 'no-baseline-run',
+        config: { ...mockManifest.config, scenario: 'no-baseline-scenario' },
+      };
+      await storage.save(noBaselineRun);
+
+      const result = await storage.compareToBaseline('no-baseline-run');
+
+      expect(result).toBeNull();
+    });
+  });
 });

package/src/storage/local.ts

@@ -5,7 +5,13 @@
 import { mkdir, readFile, readdir, unlink, writeFile } from 'node:fs/promises';
 import { join, resolve } from 'node:path';
 import type { AnyManifest, RedTeamManifest, RunManifest, StressManifest } from '../artifacts/types';
-import type { ComparisonResult, ListOptions, RunListItem, StorageAdapter } from './types';
+import type {
+  BaselineMetadata,
+  BaselineStorageAdapter,
+  ComparisonResult,
+  ListOptions,
+  RunListItem,
+} from './types';
 
 /**
  * Get manifest type from a manifest object
@@ -39,11 +45,21 @@ function getScenario(manifest: AnyManifest): string {
   return manifest.config.scenario;
 }
 
-export class LocalStorageAdapter implements StorageAdapter {
+/**
+ * Baselines file structure
+ */
+interface BaselinesFile {
+  version: string;
+  baselines: Record<string, BaselineMetadata>;
+}
+
+export class LocalStorageAdapter implements BaselineStorageAdapter {
   private basePath: string;
+  private baselinesPath: string;
 
   constructor(basePath = './artemis-runs') {
     this.basePath = resolve(basePath);
+    this.baselinesPath = join(this.basePath, '.artemis', 'baselines.json');
   }
 
   async save(manifest: AnyManifest): Promise<string> {
@@ -191,4 +207,148 @@ export class LocalStorageAdapter implements StorageAdapter {
       return [];
     }
   }
+
+  // ==================== Baseline Methods ====================
+
+  /**
+   * Load baselines file
+   */
+  private async loadBaselinesFile(): Promise<BaselinesFile> {
+    try {
+      const content = await readFile(this.baselinesPath, 'utf-8');
+      return JSON.parse(content);
+    } catch {
+      return { version: '1.0', baselines: {} };
+    }
+  }
+
+  /**
+   * Save baselines file
+   */
+  private async saveBaselinesFile(data: BaselinesFile): Promise<void> {
+    const dir = join(this.basePath, '.artemis');
+    await mkdir(dir, { recursive: true });
+    await writeFile(this.baselinesPath, JSON.stringify(data, null, 2));
+  }
+
+  /**
+   * Set a baseline for a scenario
+   */
+  async setBaseline(scenario: string, runId: string, tag?: string): Promise<BaselineMetadata> {
+    // Load the run to extract metrics
+    const manifest = await this.loadRun(runId);
+    const scenarioName = scenario || getScenario(manifest);
+
+    const baseline: BaselineMetadata = {
+      scenario: scenarioName,
+      runId,
+      createdAt: new Date().toISOString(),
+      metrics: {
+        successRate: manifest.metrics.success_rate,
+        medianLatencyMs: manifest.metrics.median_latency_ms,
+        totalTokens: manifest.metrics.total_tokens,
+        passedCases: manifest.metrics.passed_cases,
+        failedCases: manifest.metrics.failed_cases,
+        totalCases: manifest.metrics.total_cases,
+      },
+      tag,
+    };
+
+    // Load existing baselines and add/update
+    const data = await this.loadBaselinesFile();
+    data.baselines[scenarioName] = baseline;
+    await this.saveBaselinesFile(data);
+
+    return baseline;
+  }
+
+  /**
+   * Get the baseline for a scenario
+   */
+  async getBaseline(scenario: string): Promise<BaselineMetadata | null> {
+    const data = await this.loadBaselinesFile();
+    return data.baselines[scenario] || null;
+  }
+
+  /**
+   * Get a baseline by run ID
+   */
+  async getBaselineByRunId(runId: string): Promise<BaselineMetadata | null> {
+    const data = await this.loadBaselinesFile();
+    const baselines = Object.values(data.baselines);
+    return baselines.find((b) => b.runId === runId) || null;
+  }
+
+  /**
+   * List all baselines
+   */
+  async listBaselines(): Promise<BaselineMetadata[]> {
+    const data = await this.loadBaselinesFile();
+    return Object.values(data.baselines).sort(
+      (a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime()
+    );
+  }
+
+  /**
+   * Remove a baseline by scenario name
+   */
+  async removeBaseline(scenario: string): Promise<boolean> {
+    const data = await this.loadBaselinesFile();
+    if (data.baselines[scenario]) {
+      delete data.baselines[scenario];
+      await this.saveBaselinesFile(data);
+      return true;
+    }
+    return false;
+  }
+
+  /**
+   * Remove a baseline by run ID
+   */
+  async removeBaselineByRunId(runId: string): Promise<boolean> {
+    const data = await this.loadBaselinesFile();
+    const entry = Object.entries(data.baselines).find(([_, b]) => b.runId === runId);
+    if (entry) {
+      delete data.baselines[entry[0]];
+      await this.saveBaselinesFile(data);
+      return true;
+    }
+    return false;
+  }
+
+  /**
+   * Compare a run against its baseline (if exists)
+   */
+  async compareToBaseline(
+    runId: string,
+    regressionThreshold = 0.05
+  ): Promise<{
+    baseline: BaselineMetadata;
+    comparison: ComparisonResult;
+    hasRegression: boolean;
+    regressionThreshold: number;
+  } | null> {
+    // Load the current run
+    const currentManifest = await this.loadRun(runId);
+    const scenario = getScenario(currentManifest);
+
+    // Get baseline for this scenario
+    const baseline = await this.getBaseline(scenario);
+    if (!baseline) {
+      return null;
+    }
+
+    // Load baseline manifest for full comparison
+    const comparison = await this.compare(baseline.runId, runId);
+
+    // Check for regression (negative delta in success rate)
+    const hasRegression = comparison.delta.successRate < -regressionThreshold;
+
+    return {
+      baseline,
+      comparison,
+      hasRegression,
+      regressionThreshold,
+    };
+  }
 }

package/src/storage/types.ts

@@ -96,3 +96,76 @@ export interface StorageConfig {
   bucket?: string;
   basePath?: string;
 }
+
+/**
+ * Baseline metadata for regression comparison
+ */
+export interface BaselineMetadata {
+  /** Scenario name or identifier */
+  scenario: string;
+  /** Run ID of the baseline */
+  runId: string;
+  /** ISO timestamp when baseline was set */
+  createdAt: string;
+  /** Key metrics captured at baseline time */
+  metrics: {
+    successRate: number;
+    medianLatencyMs: number;
+    totalTokens: number;
+    passedCases: number;
+    failedCases: number;
+    totalCases: number;
+  };
+  /** Optional description or tag */
+  tag?: string;
+}
+
+/**
+ * Extended storage adapter with baseline support
+ */
+export interface BaselineStorageAdapter extends StorageAdapter {
+  /**
+   * Set a baseline for a scenario
+   */
+  setBaseline(scenario: string, runId: string, tag?: string): Promise<BaselineMetadata>;
+
+  /**
+   * Get the baseline by scenario name
+   */
+  getBaseline(scenario: string): Promise<BaselineMetadata | null>;
+
+  /**
+   * Get the baseline by run ID
+   */
+  getBaselineByRunId(runId: string): Promise<BaselineMetadata | null>;
+
+  /**
+   * List all baselines
+   */
+  listBaselines(): Promise<BaselineMetadata[]>;
+
+  /**
+   * Remove a baseline by scenario name
+   */
+  removeBaseline(scenario: string): Promise<boolean>;
+
+  /**
+   * Remove a baseline by run ID
+   */
+  removeBaselineByRunId(runId: string): Promise<boolean>;
+
+  /**
+   * Compare a run against its baseline (if exists)
+   * @param runId - The run ID to compare
+   * @param regressionThreshold - Threshold for regression detection (0-1), default 0.05
+   */
+  compareToBaseline?(
+    runId: string,
+    regressionThreshold?: number
+  ): Promise<{
+    baseline: BaselineMetadata;
+    comparison: ComparisonResult;
+    hasRegression: boolean;
+    regressionThreshold: number;
+  } | null>;
+}