@artemiskit/core 0.1.5 → 0.2.0

package/src/scenario/discovery.test.ts

@@ -0,0 +1,153 @@
+/**
+ * Tests for scenario discovery
+ */
+
+import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
+import { mkdir, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { discoverScenarios, matchScenarioGlob, resolveScenarioPaths } from './discovery';
+
+describe('Scenario Discovery', () => {
+  const testDir = join(tmpdir(), `artemis-discovery-test-${Date.now()}`);
+
+  beforeAll(async () => {
+    // Create test directory structure
+    await mkdir(testDir, { recursive: true });
+    await mkdir(join(testDir, 'scenarios'), { recursive: true });
+    await mkdir(join(testDir, 'scenarios', 'auth'), { recursive: true });
+    await mkdir(join(testDir, 'scenarios', 'api'), { recursive: true });
+    await mkdir(join(testDir, 'node_modules'), { recursive: true });
+    await mkdir(join(testDir, 'drafts'), { recursive: true });
+
+    // Create test scenario files
+    const scenarioContent =
+      'name: test\ncases:\n  - id: t1\n    prompt: test\n    expected:\n      type: exact\n      value: test';
+
+    await writeFile(join(testDir, 'scenarios', 'basic.yaml'), scenarioContent);
+    await writeFile(join(testDir, 'scenarios', 'advanced.yml'), scenarioContent);
+    await writeFile(join(testDir, 'scenarios', 'auth', 'login.yaml'), scenarioContent);
+    await writeFile(join(testDir, 'scenarios', 'auth', 'logout.yaml'), scenarioContent);
+    await writeFile(join(testDir, 'scenarios', 'api', 'users.yaml'), scenarioContent);
+    await writeFile(join(testDir, 'scenarios', 'api', 'posts.yml'), scenarioContent);
+    await writeFile(join(testDir, 'node_modules', 'ignored.yaml'), scenarioContent);
+    await writeFile(join(testDir, 'drafts', 'draft.yaml'), scenarioContent);
+    await writeFile(join(testDir, 'scenarios', 'skip.draft.yaml'), scenarioContent);
+    await writeFile(join(testDir, 'scenarios', 'readme.md'), '# Not a scenario');
+  });
+
+  afterAll(async () => {
+    // Cleanup test directory
+    await rm(testDir, { recursive: true, force: true });
+  });
+
+  describe('discoverScenarios', () => {
+    test('finds all yaml and yml files in directory', async () => {
+      const files = await discoverScenarios(join(testDir, 'scenarios'));
+      expect(files.length).toBe(7); // 2 root + 2 auth + 2 api + 1 draft
+    });
+
+    test('excludes node_modules by default', async () => {
+      const files = await discoverScenarios(testDir);
+      const nodeModulesFiles = files.filter((f) => f.includes('node_modules'));
+      expect(nodeModulesFiles.length).toBe(0);
+    });
+
+    test('respects custom exclude patterns', async () => {
+      const files = await discoverScenarios(join(testDir, 'scenarios'), {
+        exclude: ['*.draft.yaml'],
+      });
+      const draftFiles = files.filter((f) => f.includes('.draft.yaml'));
+      expect(draftFiles.length).toBe(0);
+    });
+
+    test('respects custom extensions', async () => {
+      const files = await discoverScenarios(join(testDir, 'scenarios'), {
+        extensions: ['.yaml'],
+      });
+      const ymlFiles = files.filter((f) => f.endsWith('.yml'));
+      expect(ymlFiles.length).toBe(0);
+    });
+
+    test('respects maxDepth option', async () => {
+      const files = await discoverScenarios(join(testDir, 'scenarios'), {
+        maxDepth: 0,
+      });
+      // Should only find files in the root scenarios directory
+      expect(files.length).toBe(3); // basic.yaml, advanced.yml, skip.draft.yaml
+    });
+
+    test('throws error for non-directory path', async () => {
+      await expect(discoverScenarios(join(testDir, 'scenarios', 'basic.yaml'))).rejects.toThrow(
+        'not a directory'
+      );
+    });
+
+    test('returns sorted results', async () => {
+      const files = await discoverScenarios(join(testDir, 'scenarios'));
+      const sorted = [...files].sort();
+      expect(files).toEqual(sorted);
+    });
+  });
+
+  describe('matchScenarioGlob', () => {
+    test('matches simple wildcard pattern', async () => {
+      const files = await matchScenarioGlob('scenarios/*.yaml', testDir);
+      expect(files.length).toBe(2); // basic.yaml, skip.draft.yaml
+    });
+
+    test('matches recursive pattern with **', async () => {
+      const files = await matchScenarioGlob('scenarios/**/*.yaml', testDir);
+      expect(files.length).toBeGreaterThan(2);
+    });
+
+    test('matches specific prefix pattern', async () => {
+      const files = await matchScenarioGlob('scenarios/auth/*.yaml', testDir);
+      expect(files.length).toBe(2); // login.yaml, logout.yaml
+    });
+
+    test('returns single file for non-glob file path', async () => {
+      const files = await matchScenarioGlob('scenarios/basic.yaml', testDir);
+      expect(files.length).toBe(1);
+      expect(files[0]).toContain('basic.yaml');
+    });
+
+    test('returns all files for non-glob directory path', async () => {
+      const files = await matchScenarioGlob('scenarios/auth', testDir);
+      expect(files.length).toBe(2);
+    });
+
+    test('returns empty array for non-existent path', async () => {
+      const files = await matchScenarioGlob('nonexistent/*.yaml', testDir);
+      expect(files.length).toBe(0);
+    });
+
+    test('matches single character with ?', async () => {
+      const files = await matchScenarioGlob('scenarios/api/?????.yaml', testDir);
+      expect(files.length).toBe(1); // users.yaml (5 chars)
+    });
+  });
+
+  describe('resolveScenarioPaths', () => {
+    test('resolves single file', async () => {
+      const files = await resolveScenarioPaths(join(testDir, 'scenarios', 'basic.yaml'));
+      expect(files.length).toBe(1);
+    });
+
+    test('resolves directory', async () => {
+      const files = await resolveScenarioPaths(join(testDir, 'scenarios', 'auth'));
+      expect(files.length).toBe(2);
+    });
+
+    test('resolves glob pattern', async () => {
+      const files = await resolveScenarioPaths('scenarios/*.yaml', testDir);
+      expect(files.length).toBe(2);
+    });
+
+    test('throws for non-existent path', async () => {
+      await expect(resolveScenarioPaths(join(testDir, 'nonexistent.yaml'))).rejects.toThrow(
+        'not found'
+      );
+    });
+  });
+});

package/src/scenario/discovery.ts

@@ -0,0 +1,277 @@
+/**
+ * Scenario discovery - find scenario files by directory scanning and glob patterns
+ */
+
+import { readdir, stat } from 'node:fs/promises';
+import { join, resolve } from 'node:path';
+
+/**
+ * Options for scenario discovery
+ */
+export interface DiscoveryOptions {
+  /** File extensions to include (default: ['.yaml', '.yml']) */
+  extensions?: string[];
+  /** Maximum directory depth to scan (default: 10) */
+  maxDepth?: number;
+  /** Patterns to exclude (glob-like, e.g., 'node_modules', '*.draft.yaml') */
+  exclude?: string[];
+}
+
+const DEFAULT_EXTENSIONS = ['.yaml', '.yml'];
+const DEFAULT_MAX_DEPTH = 10;
+const DEFAULT_EXCLUDE = ['node_modules', '.git', 'dist', 'build', 'coverage'];
+
+/**
+ * Check if a path matches any of the exclude patterns
+ */
+function matchesExcludePattern(name: string, patterns: string[]): boolean {
+  for (const pattern of patterns) {
+    // Simple wildcard matching
+    if (pattern.includes('*')) {
+      const regex = new RegExp(`^${pattern.replace(/\./g, '\\.').replace(/\*/g, '.*')}$`);
+      if (regex.test(name)) return true;
+    } else {
+      // Exact match
+      if (name === pattern) return true;
+    }
+  }
+  return false;
+}
+
+/**
+ * Check if a filename has a valid scenario extension
+ */
+function hasValidExtension(filename: string, extensions: string[]): boolean {
+  return extensions.some((ext) => filename.endsWith(ext));
+}
+
+/**
+ * Recursively scan a directory for scenario files
+ */
+async function scanDirectoryRecursive(
+  dirPath: string,
+  options: Required<DiscoveryOptions>,
+  currentDepth: number
+): Promise<string[]> {
+  if (currentDepth > options.maxDepth) {
+    return [];
+  }
+
+  const results: string[] = [];
+  const entries = await readdir(dirPath, { withFileTypes: true });
+
+  for (const entry of entries) {
+    const fullPath = join(dirPath, entry.name);
+
+    // Skip excluded patterns
+    if (matchesExcludePattern(entry.name, options.exclude)) {
+      continue;
+    }
+
+    if (entry.isDirectory()) {
+      // Recursively scan subdirectories
+      const subResults = await scanDirectoryRecursive(fullPath, options, currentDepth + 1);
+      results.push(...subResults);
+    } else if (entry.isFile() && hasValidExtension(entry.name, options.extensions)) {
+      results.push(fullPath);
+    }
+  }
+
+  return results;
+}
+
+/**
+ * Discover scenario files in a directory
+ *
+ * @param dirPath - Path to the directory to scan
+ * @param options - Discovery options
+ * @returns Array of absolute paths to scenario files
+ *
+ * @example
+ * ```ts
+ * // Scan a directory for all .yaml and .yml files
+ * const files = await discoverScenarios('./scenarios');
+ *
+ * // Scan with custom options
+ * const files = await discoverScenarios('./tests', {
+ *   extensions: ['.yaml'],
+ *   maxDepth: 3,
+ *   exclude: ['drafts', '*.skip.yaml']
+ * });
+ * ```
+ */
+export async function discoverScenarios(
+  dirPath: string,
+  options: DiscoveryOptions = {}
+): Promise<string[]> {
+  const resolvedPath = resolve(dirPath);
+  const pathStat = await stat(resolvedPath);
+
+  if (!pathStat.isDirectory()) {
+    throw new Error(`Path is not a directory: ${dirPath}`);
+  }
+
+  const fullOptions: Required<DiscoveryOptions> = {
+    extensions: options.extensions ?? DEFAULT_EXTENSIONS,
+    maxDepth: options.maxDepth ?? DEFAULT_MAX_DEPTH,
+    exclude: [...DEFAULT_EXCLUDE, ...(options.exclude ?? [])],
+  };
+
+  const files = await scanDirectoryRecursive(resolvedPath, fullOptions, 0);
+
+  // Sort for consistent ordering
+  return files.sort();
+}
+
+/**
+ * Match scenario files using glob-like patterns
+ *
+ * Supports basic glob patterns:
+ * - `*` matches any characters except path separator
+ * - `**` matches any characters including path separator (recursive)
+ * - `?` matches single character
+ *
+ * @param pattern - Glob pattern to match
+ * @param basePath - Base path to resolve relative patterns (default: cwd)
+ * @returns Array of absolute paths to matching scenario files
+ *
+ * @example
+ * ```ts
+ * // Match all yaml files in scenarios directory
+ * const files = await matchScenarioGlob('scenarios/*.yaml');
+ *
+ * // Match recursively
+ * const files = await matchScenarioGlob('tests/**\/*.yaml');
+ *
+ * // Match specific patterns
+ * const files = await matchScenarioGlob('scenarios/auth-*.yaml');
+ * ```
+ */
+export async function matchScenarioGlob(
+  pattern: string,
+  basePath: string = process.cwd()
+): Promise<string[]> {
+  const resolvedBase = resolve(basePath);
+
+  // Check if the pattern contains glob characters
+  const hasGlob = /[*?]/.test(pattern);
+
+  if (!hasGlob) {
+    // Not a glob pattern - check if it's a file or directory
+    const fullPath = resolve(resolvedBase, pattern);
+    try {
+      const pathStat = await stat(fullPath);
+      if (pathStat.isFile()) {
+        return [fullPath];
+      }
+      if (pathStat.isDirectory()) {
+        return discoverScenarios(fullPath);
+      }
+    } catch {
+      // Path doesn't exist
+      return [];
+    }
+    return [];
+  }
+
+  // Convert glob pattern to regex
+  const globToRegex = (glob: string): RegExp => {
+    const regexStr = glob
+      // Escape special regex characters except * and ?
+      .replace(/[.+^${}()|[\]\\]/g, '\\$&')
+      // Convert ** to match any path
+      .replace(/\*\*/g, '{{GLOBSTAR}}')
+      // Convert * to match any characters except /
+      .replace(/\*/g, '[^/]*')
+      // Convert ? to match single character
+      .replace(/\?/g, '.')
+      // Restore ** as match-all including /
+      .replace(/\{\{GLOBSTAR\}\}/g, '.*');
+
+    return new RegExp(`^${regexStr}$`);
+  };
+
+  // Check if pattern is an absolute path
+  const isAbsolute = pattern.startsWith('/');
+
+  // Extract the base directory (non-glob prefix)
+  const patternParts = pattern.split('/');
+  let baseDir = isAbsolute ? '/' : resolvedBase;
+  let globPart = pattern;
+
+  // Start from index 1 if absolute path (skip empty string from leading /)
+  const startIndex = isAbsolute ? 1 : 0;
+
+  for (let i = startIndex; i < patternParts.length; i++) {
+    const part = patternParts[i];
+    if (/[*?]/.test(part)) {
+      // Found first glob character - everything before is base
+      globPart = patternParts.slice(i).join('/');
+      break;
+    }
+    baseDir = join(baseDir, part);
+  }
+
+  // Check if base directory exists
+  try {
+    const baseStat = await stat(baseDir);
+    if (!baseStat.isDirectory()) {
+      return [];
+    }
+  } catch {
+    return [];
+  }
+
+  // Scan from base directory and filter by pattern
+  const allFiles = await discoverScenarios(baseDir, { maxDepth: 20 });
+  const regex = globToRegex(globPart);
+
+  const matchedFiles = allFiles.filter((filePath) => {
+    // Get relative path from base directory
+    const relativePath = filePath.slice(baseDir.length + 1);
+    return regex.test(relativePath);
+  });
+
+  return matchedFiles.sort();
+}
+
+/**
+ * Resolve a scenario path argument which can be:
+ * - A single file path
+ * - A directory path
+ * - A glob pattern
+ *
+ * @param pathArg - Path argument from CLI
+ * @param basePath - Base path for relative resolution
+ * @returns Array of resolved scenario file paths
+ */
+export async function resolveScenarioPaths(
+  pathArg: string,
+  basePath: string = process.cwd()
+): Promise<string[]> {
+  const resolvedBase = resolve(basePath);
+  const hasGlob = /[*?]/.test(pathArg);
+
+  if (hasGlob) {
+    return matchScenarioGlob(pathArg, resolvedBase);
+  }
+
+  const fullPath = resolve(resolvedBase, pathArg);
+
+  try {
+    const pathStat = await stat(fullPath);
+
+    if (pathStat.isFile()) {
+      return [fullPath];
+    }
+
+    if (pathStat.isDirectory()) {
+      return discoverScenarios(fullPath);
+    }
+  } catch {
+    // Path doesn't exist
+    throw new Error(`Scenario path not found: ${pathArg}`);
+  }
+
+  return [];
+}

package/src/scenario/index.ts

@@ -5,3 +5,4 @@
 export * from './schema';
 export * from './parser';
 export * from './variables';
+export * from './discovery';

package/src/scenario/schema.ts

@@ -42,6 +42,7 @@ export const ProviderConfigSchema = z
     resourceName: z.string().optional(),
     deploymentName: z.string().optional(),
     apiVersion: z.string().optional(),
+    embeddingDeploymentName: z.string().optional(),
 
     // Vercel AI specific
     underlyingProvider: z.enum(['openai', 'azure', 'anthropic', 'google', 'mistral']).optional(),
@@ -49,9 +50,9 @@ export const ProviderConfigSchema = z
   .optional();
 
 /**
- * Expected result types - how to evaluate responses
+ * Base expected types (non-recursive)
  */
-export const ExpectedSchema = z.discriminatedUnion('type', [
+const BaseExpectedSchema = z.discriminatedUnion('type', [
   z.object({
     type: z.literal('exact'),
     value: z.string(),
@@ -84,6 +85,12 @@ export const ExpectedSchema = z.discriminatedUnion('type', [
     mode: z.enum(['all', 'any']).default('all'),
   }),
 
+  z.object({
+    type: z.literal('not_contains'),
+    values: z.array(z.string()),
+    mode: z.enum(['all', 'any']).default('all'),
+  }),
+
   z.object({
     type: z.literal('json_schema'),
     schema: z.record(z.unknown()),
@@ -94,8 +101,42 @@ export const ExpectedSchema = z.discriminatedUnion('type', [
     evaluator: z.string(),
     config: z.record(z.unknown()).optional(),
   }),
+
+  z.object({
+    type: z.literal('similarity'),
+    value: z.string(),
+    threshold: z.number().min(0).max(1).default(0.75),
+    /** Mode for similarity evaluation: 'embedding' uses vector embeddings, 'llm' uses LLM-based comparison */
+    mode: z.enum(['embedding', 'llm']).optional(),
+    /** Model for LLM-based similarity comparison (required when mode is 'llm') */
+    model: z.string().optional(),
+    /** Embedding model to use for vector similarity (required when mode is 'embedding') */
+    embeddingModel: z.string().optional(),
+  }),
+
+  z.object({
+    type: z.literal('inline'),
+    expression: z.string(),
+    value: z.string().optional(),
+  }),
 ]);
 
+/**
+ * Combined expectation schema - allows combining multiple expectations with and/or logic
+ * Note: Combined expectations can only contain base expectations (no nested combined)
+ */
+const CombinedExpectedSchema = z.object({
+  type: z.literal('combined'),
+  operator: z.enum(['and', 'or']),
+  expectations: z.array(BaseExpectedSchema).min(1),
+});
+
+/**
+ * Expected result types - how to evaluate responses
+ * Includes base types and combined type for logical grouping
+ */
+export const ExpectedSchema = z.union([BaseExpectedSchema, CombinedExpectedSchema]);
+
 /**
  * Chat message schema
  */

package/src/utils/logger.ts

@@ -2,53 +2,82 @@
  * Logger utility for Artemis
  */
 
-import pino from 'pino';
+import { type ConsolaInstance, LogLevels, createConsola } from 'consola';
 
 export type LogLevel = 'debug' | 'info' | 'warn' | 'error';
 
-const level = process.env.ARTEMIS_LOG_LEVEL || 'info';
+const LOG_LEVEL_MAP: Record<LogLevel, number> = {
+  debug: LogLevels.debug,
+  info: LogLevels.info,
+  warn: LogLevels.warn,
+  error: LogLevels.error,
+};
 
-const baseLogger = pino({
-  level,
-  transport:
-    process.env.NODE_ENV === 'development'
-      ? { target: 'pino-pretty', options: { colorize: true } }
-      : undefined,
+const level = (process.env.ARTEMIS_LOG_LEVEL as LogLevel) || 'info';
+
+const baseLogger = createConsola({
+  level: LOG_LEVEL_MAP[level] ?? LogLevels.info,
+  formatOptions: {
+    colors: true,
+    date: true,
+  },
 });
 
 /**
  * Logger class for consistent logging across Artemis
  */
 export class Logger {
-  private logger: pino.Logger;
+  private logger: ConsolaInstance;
 
   constructor(name: string) {
-    this.logger = baseLogger.child({ name });
+    this.logger = baseLogger.withTag(name);
   }
 
   debug(message: string, data?: Record<string, unknown>): void {
-    this.logger.debug(data, message);
+    if (data) {
+      this.logger.debug(message, data);
+    } else {
+      this.logger.debug(message);
+    }
   }
 
   info(message: string, data?: Record<string, unknown>): void {
-    this.logger.info(data, message);
+    if (data) {
+      this.logger.info(message, data);
+    } else {
+      this.logger.info(message);
+    }
   }
 
   warn(message: string, data?: Record<string, unknown>): void {
-    this.logger.warn(data, message);
+    if (data) {
+      this.logger.warn(message, data);
+    } else {
+      this.logger.warn(message);
+    }
   }
 
   error(message: string, error?: Error | unknown, data?: Record<string, unknown>): void {
     const errorData =
       error instanceof Error
         ? { error: { message: error.message, stack: error.stack, name: error.name } }
-        : { error };
-    this.logger.error({ ...data, ...errorData }, message);
+        : error
+          ? { error }
+          : undefined;
+
+    const mergedData = errorData || data ? { ...data, ...errorData } : undefined;
+
+    if (mergedData) {
+      this.logger.error(message, mergedData);
+    } else {
+      this.logger.error(message);
+    }
   }
 
   child(bindings: Record<string, unknown>): Logger {
     const childLogger = new Logger('');
-    childLogger.logger = this.logger.child(bindings);
+    const tag = bindings.name ? String(bindings.name) : '';
+    childLogger.logger = this.logger.withTag(tag);
     return childLogger;
   }
 }