@artemiskit/core 0.1.2

package/src/storage/local.ts

@@ -0,0 +1,194 @@
+/**
+ * Local filesystem storage adapter
+ */
+
+import { mkdir, readFile, readdir, unlink, writeFile } from 'node:fs/promises';
+import { join, resolve } from 'node:path';
+import type { AnyManifest, RedTeamManifest, RunManifest, StressManifest } from '../artifacts/types';
+import type { ComparisonResult, ListOptions, RunListItem, StorageAdapter } from './types';
+
+/**
+ * Get manifest type from a manifest object
+ */
+function getManifestType(manifest: AnyManifest): 'run' | 'redteam' | 'stress' {
+  if ('type' in manifest) {
+    if (manifest.type === 'redteam') return 'redteam';
+    if (manifest.type === 'stress') return 'stress';
+  }
+  return 'run';
+}
+
+/**
+ * Get success/defense rate from any manifest type
+ */
+function getSuccessRate(manifest: AnyManifest): number {
+  const type = getManifestType(manifest);
+  if (type === 'redteam') {
+    return (manifest as RedTeamManifest).metrics.defense_rate;
+  }
+  if (type === 'stress') {
+    return (manifest as StressManifest).metrics.success_rate;
+  }
+  return (manifest as RunManifest).metrics.success_rate;
+}
+
+/**
+ * Get scenario name from any manifest type
+ */
+function getScenario(manifest: AnyManifest): string {
+  return manifest.config.scenario;
+}
+
+export class LocalStorageAdapter implements StorageAdapter {
+  private basePath: string;
+
+  constructor(basePath = './artemis-runs') {
+    this.basePath = resolve(basePath);
+  }
+
+  async save(manifest: AnyManifest): Promise<string> {
+    const dir = join(this.basePath, manifest.project);
+    await mkdir(dir, { recursive: true });
+
+    const filePath = join(dir, `${manifest.run_id}.json`);
+    await writeFile(filePath, JSON.stringify(manifest, null, 2));
+
+    return filePath;
+  }
+
+  async load(runId: string): Promise<AnyManifest> {
+    const projects = await this.listDirectories(this.basePath);
+
+    for (const project of projects) {
+      const filePath = join(this.basePath, project, `${runId}.json`);
+      try {
+        const content = await readFile(filePath, 'utf-8');
+        return JSON.parse(content);
+      } catch {}
+    }
+
+    throw new Error(`Run not found: ${runId}`);
+  }
+
+  async loadRun(runId: string): Promise<RunManifest> {
+    const manifest = await this.load(runId);
+    if (getManifestType(manifest) !== 'run') {
+      throw new Error(`Run ${runId} is not a standard run manifest`);
+    }
+    return manifest as RunManifest;
+  }
+
+  async loadRedTeam(runId: string): Promise<RedTeamManifest> {
+    const manifest = await this.load(runId);
+    if (getManifestType(manifest) !== 'redteam') {
+      throw new Error(`Run ${runId} is not a red team manifest`);
+    }
+    return manifest as RedTeamManifest;
+  }
+
+  async loadStress(runId: string): Promise<StressManifest> {
+    const manifest = await this.load(runId);
+    if (getManifestType(manifest) !== 'stress') {
+      throw new Error(`Run ${runId} is not a stress test manifest`);
+    }
+    return manifest as StressManifest;
+  }
+
+  async list(options?: ListOptions): Promise<RunListItem[]> {
+    const results: RunListItem[] = [];
+
+    const projects = options?.project
+      ? [options.project]
+      : await this.listDirectories(this.basePath);
+
+    for (const project of projects) {
+      const projectDir = join(this.basePath, project);
+      const files = await this.listFiles(projectDir);
+
+      for (const file of files) {
+        if (!file.endsWith('.json')) continue;
+
+        try {
+          const content = await readFile(join(projectDir, file), 'utf-8');
+          const manifest: AnyManifest = JSON.parse(content);
+          const manifestType = getManifestType(manifest);
+
+          // Filter by type if specified
+          if (options?.type && manifestType !== options.type) {
+            continue;
+          }
+
+          if (options?.scenario && getScenario(manifest) !== options.scenario) {
+            continue;
+          }
+
+          results.push({
+            runId: manifest.run_id,
+            scenario: getScenario(manifest),
+            successRate: getSuccessRate(manifest),
+            createdAt: manifest.start_time,
+            type: manifestType,
+          });
+        } catch {}
+      }
+    }
+
+    results.sort((a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime());
+
+    const offset = options?.offset || 0;
+    const limit = options?.limit;
+
+    if (limit) {
+      return results.slice(offset, offset + limit);
+    }
+
+    return results.slice(offset);
+  }
+
+  async delete(runId: string): Promise<void> {
+    const projects = await this.listDirectories(this.basePath);
+
+    for (const project of projects) {
+      const filePath = join(this.basePath, project, `${runId}.json`);
+      try {
+        await unlink(filePath);
+        return;
+      } catch {}
+    }
+  }
+
+  async compare(baselineId: string, currentId: string): Promise<ComparisonResult> {
+    const [baseline, current] = await Promise.all([
+      this.loadRun(baselineId),
+      this.loadRun(currentId),
+    ]);
+
+    return {
+      baseline,
+      current,
+      delta: {
+        successRate: current.metrics.success_rate - baseline.metrics.success_rate,
+        latency: current.metrics.median_latency_ms - baseline.metrics.median_latency_ms,
+        tokens: current.metrics.total_tokens - baseline.metrics.total_tokens,
+      },
+    };
+  }
+
+  private async listDirectories(path: string): Promise<string[]> {
+    try {
+      const entries = await readdir(path, { withFileTypes: true });
+      return entries.filter((e) => e.isDirectory()).map((e) => e.name);
+    } catch {
+      return [];
+    }
+  }
+
+  private async listFiles(path: string): Promise<string[]> {
+    try {
+      const entries = await readdir(path, { withFileTypes: true });
+      return entries.filter((e) => e.isFile()).map((e) => e.name);
+    } catch {
+      return [];
+    }
+  }
+}

package/src/storage/supabase.ts

@@ -0,0 +1,151 @@
+/**
+ * Supabase storage adapter
+ */
+
+import { type SupabaseClient, createClient } from '@supabase/supabase-js';
+import type { RunManifest } from '../artifacts/types';
+import type { ComparisonResult, ListOptions, RunListItem, StorageAdapter } from './types';
+
+export interface SupabaseStorageConfig {
+  url: string;
+  anonKey: string;
+  bucket?: string;
+}
+
+export class SupabaseStorageAdapter implements StorageAdapter {
+  private client: SupabaseClient;
+  private bucket: string;
+
+  constructor(config: SupabaseStorageConfig) {
+    this.client = createClient(config.url, config.anonKey);
+    this.bucket = config.bucket || 'artemis-runs';
+  }
+
+  async save(manifest: RunManifest): Promise<string> {
+    const filePath = `${manifest.project}/${manifest.run_id}.json`;
+
+    const { error: uploadError } = await this.client.storage
+      .from(this.bucket)
+      .upload(filePath, JSON.stringify(manifest, null, 2), {
+        contentType: 'application/json',
+        upsert: true,
+      });
+
+    if (uploadError) {
+      throw new Error(`Failed to upload manifest: ${uploadError.message}`);
+    }
+
+    const { error: dbError } = await this.client.from('runs').upsert({
+      run_id: manifest.run_id,
+      project: manifest.project,
+      scenario: manifest.config.scenario,
+      provider: manifest.config.provider,
+      model: manifest.config.model,
+      success_rate: manifest.metrics.success_rate,
+      total_cases: manifest.metrics.total_cases,
+      passed_cases: manifest.metrics.passed_cases,
+      failed_cases: manifest.metrics.failed_cases,
+      median_latency_ms: manifest.metrics.median_latency_ms,
+      p95_latency_ms: manifest.metrics.p95_latency_ms,
+      total_tokens: manifest.metrics.total_tokens,
+      git_commit: manifest.git.commit,
+      git_branch: manifest.git.branch,
+      git_dirty: manifest.git.dirty,
+      run_by: manifest.provenance.run_by,
+      run_reason: manifest.provenance.run_reason,
+      started_at: manifest.start_time,
+      ended_at: manifest.end_time,
+      manifest_path: filePath,
+    });
+
+    if (dbError) {
+      throw new Error(`Failed to save run metadata: ${dbError.message}`);
+    }
+
+    return filePath;
+  }
+
+  async load(runId: string): Promise<RunManifest> {
+    const { data: run, error: dbError } = await this.client
+      .from('runs')
+      .select('manifest_path')
+      .eq('run_id', runId)
+      .single();
+
+    if (dbError || !run) {
+      throw new Error(`Run not found: ${runId}`);
+    }
+
+    const { data, error: downloadError } = await this.client.storage
+      .from(this.bucket)
+      .download(run.manifest_path);
+
+    if (downloadError || !data) {
+      throw new Error(`Failed to download manifest: ${downloadError?.message}`);
+    }
+
+    const text = await data.text();
+    return JSON.parse(text);
+  }
+
+  async list(options?: ListOptions): Promise<RunListItem[]> {
+    let query = this.client
+      .from('runs')
+      .select('run_id, scenario, success_rate, started_at')
+      .order('started_at', { ascending: false });
+
+    if (options?.project) {
+      query = query.eq('project', options.project);
+    }
+    if (options?.scenario) {
+      query = query.eq('scenario', options.scenario);
+    }
+    if (options?.limit) {
+      query = query.limit(options.limit);
+    }
+    if (options?.offset) {
+      query = query.range(options.offset, options.offset + (options.limit || 10) - 1);
+    }
+
+    const { data, error } = await query;
+
+    if (error) {
+      throw new Error(`Failed to list runs: ${error.message}`);
+    }
+
+    return (data || []).map((r) => ({
+      runId: r.run_id,
+      scenario: r.scenario,
+      successRate: r.success_rate,
+      createdAt: r.started_at,
+    }));
+  }
+
+  async delete(runId: string): Promise<void> {
+    const { data: run } = await this.client
+      .from('runs')
+      .select('manifest_path')
+      .eq('run_id', runId)
+      .single();
+
+    if (run) {
+      await this.client.storage.from(this.bucket).remove([run.manifest_path]);
+    }
+
+    await this.client.from('runs').delete().eq('run_id', runId);
+  }
+
+  async compare(baselineId: string, currentId: string): Promise<ComparisonResult> {
+    const [baseline, current] = await Promise.all([this.load(baselineId), this.load(currentId)]);
+
+    return {
+      baseline,
+      current,
+      delta: {
+        successRate: current.metrics.success_rate - baseline.metrics.success_rate,
+        latency: current.metrics.median_latency_ms - baseline.metrics.median_latency_ms,
+        tokens: current.metrics.total_tokens - baseline.metrics.total_tokens,
+      },
+    };
+  }
+}

package/src/storage/types.ts

@@ -0,0 +1,98 @@
+/**
+ * Storage types and interfaces
+ */
+
+import type { AnyManifest, RedTeamManifest, RunManifest, StressManifest } from '../artifacts/types';
+
+/**
+ * Run listing item
+ */
+export interface RunListItem {
+  runId: string;
+  scenario: string;
+  successRate: number;
+  createdAt: string;
+  /** Type of manifest (run, redteam, stress) */
+  type?: 'run' | 'redteam' | 'stress';
+}
+
+/**
+ * Comparison result between two runs
+ */
+export interface ComparisonResult {
+  baseline: RunManifest;
+  current: RunManifest;
+  delta: {
+    successRate: number;
+    latency: number;
+    tokens: number;
+  };
+}
+
+/**
+ * List options for filtering runs
+ */
+export interface ListOptions {
+  project?: string;
+  scenario?: string;
+  limit?: number;
+  offset?: number;
+  /** Filter by manifest type */
+  type?: 'run' | 'redteam' | 'stress';
+}
+
+/**
+ * Storage adapter interface - implement to create custom storage backends
+ */
+export interface StorageAdapter {
+  /**
+   * Save a run manifest (any type)
+   */
+  save(manifest: AnyManifest): Promise<string>;
+
+  /**
+   * Load a run manifest by ID
+   */
+  load(runId: string): Promise<AnyManifest>;
+
+  /**
+   * Load a standard run manifest by ID
+   */
+  loadRun?(runId: string): Promise<RunManifest>;
+
+  /**
+   * Load a red team manifest by ID
+   */
+  loadRedTeam?(runId: string): Promise<RedTeamManifest>;
+
+  /**
+   * Load a stress manifest by ID
+   */
+  loadStress?(runId: string): Promise<StressManifest>;
+
+  /**
+   * List runs with optional filters
+   */
+  list(options?: ListOptions): Promise<RunListItem[]>;
+
+  /**
+   * Delete a run
+   */
+  delete(runId: string): Promise<void>;
+
+  /**
+   * Compare two runs
+   */
+  compare?(baselineId: string, currentId: string): Promise<ComparisonResult>;
+}
+
+/**
+ * Storage configuration
+ */
+export interface StorageConfig {
+  type: 'supabase' | 'local';
+  url?: string;
+  anonKey?: string;
+  bucket?: string;
+  basePath?: string;
+}

package/src/utils/errors.ts

@@ -0,0 +1,76 @@
+/**
+ * Error handling utilities for Artemis
+ */
+
+/**
+ * Error codes used throughout Artemis
+ */
+export type ArtemisErrorCode =
+  | 'UNKNOWN_PROVIDER'
+  | 'PROVIDER_UNAVAILABLE'
+  | 'SCENARIO_READ_ERROR'
+  | 'SCENARIO_PARSE_ERROR'
+  | 'SCENARIO_VALIDATION_ERROR'
+  | 'ADAPTER_ERROR'
+  | 'GENERATION_ERROR'
+  | 'EVALUATION_ERROR'
+  | 'STORAGE_ERROR'
+  | 'CONFIG_ERROR'
+  | 'UNKNOWN_ERROR';
+
+/**
+ * Custom error class for Artemis
+ */
+export class ArtemisError extends Error {
+  readonly code: ArtemisErrorCode;
+  readonly details?: Record<string, unknown>;
+
+  constructor(
+    message: string,
+    code: ArtemisErrorCode = 'UNKNOWN_ERROR',
+    details?: Record<string, unknown>
+  ) {
+    super(message);
+    this.name = 'ArtemisError';
+    this.code = code;
+    this.details = details;
+
+    Error.captureStackTrace(this, this.constructor);
+  }
+
+  toJSON(): Record<string, unknown> {
+    return {
+      name: this.name,
+      code: this.code,
+      message: this.message,
+      details: this.details,
+      stack: this.stack,
+    };
+  }
+}
+
+/**
+ * Check if error is an ArtemisError
+ */
+export function isArtemisError(error: unknown): error is ArtemisError {
+  return error instanceof ArtemisError;
+}
+
+/**
+ * Wrap unknown errors in ArtemisError
+ */
+export function wrapError(
+  error: unknown,
+  code: ArtemisErrorCode = 'UNKNOWN_ERROR',
+  context?: string
+): ArtemisError {
+  if (error instanceof ArtemisError) {
+    return error;
+  }
+
+  const message = error instanceof Error ? error.message : String(error);
+
+  return new ArtemisError(context ? `${context}: ${message}` : message, code, {
+    originalError: error,
+  });
+}

package/src/utils/index.ts

@@ -0,0 +1,6 @@
+/**
+ * Utilities module exports
+ */
+
+export * from './errors';
+export * from './logger';

package/src/utils/logger.ts

@@ -0,0 +1,59 @@
+/**
+ * Logger utility for Artemis
+ */
+
+import pino from 'pino';
+
+export type LogLevel = 'debug' | 'info' | 'warn' | 'error';
+
+const level = process.env.ARTEMIS_LOG_LEVEL || 'info';
+
+const baseLogger = pino({
+  level,
+  transport:
+    process.env.NODE_ENV === 'development'
+      ? { target: 'pino-pretty', options: { colorize: true } }
+      : undefined,
+});
+
+/**
+ * Logger class for consistent logging across Artemis
+ */
+export class Logger {
+  private logger: pino.Logger;
+
+  constructor(name: string) {
+    this.logger = baseLogger.child({ name });
+  }
+
+  debug(message: string, data?: Record<string, unknown>): void {
+    this.logger.debug(data, message);
+  }
+
+  info(message: string, data?: Record<string, unknown>): void {
+    this.logger.info(data, message);
+  }
+
+  warn(message: string, data?: Record<string, unknown>): void {
+    this.logger.warn(data, message);
+  }
+
+  error(message: string, error?: Error | unknown, data?: Record<string, unknown>): void {
+    const errorData =
+      error instanceof Error
+        ? { error: { message: error.message, stack: error.stack, name: error.name } }
+        : { error };
+    this.logger.error({ ...data, ...errorData }, message);
+  }
+
+  child(bindings: Record<string, unknown>): Logger {
+    const childLogger = new Logger('');
+    childLogger.logger = this.logger.child(bindings);
+    return childLogger;
+  }
+}
+
+/**
+ * Default logger instance
+ */
+export const logger = new Logger('artemis');

package/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "noEmit": false,
+    "declaration": true,
+    "declarationMap": true,
+    "emitDeclarationOnly": true
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist", "**/*.test.ts"]
+}