@modular-prompt/experiment 0.1.1

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 otolab
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,347 @@
+# @modular-prompt/experiment
+
+Experiment framework for comparing and evaluating modular prompt modules.
+
+## Overview
+
+This framework provides tools to compare and evaluate different prompt module variations under identical conditions. It integrates with the `@modular-prompt/core` system to test multiple prompt variations and evaluate their output quality.
+
+### Use Cases
+
+- **Prompt Engineering**: Validate the effectiveness of new prompt structures
+- **Module Separation**: Verify that modularized prompts produce equivalent outputs
+- **Quality Evaluation**: Assess output stability and consistency through repeated executions
+- **Multi-Model Testing**: Test across different LLM providers (MLX, VertexAI, GoogleGenAI, etc.)
+
+## Features
+
+- ✅ **Dynamic Module Loading**: Load prompt modules from external files or inline definitions
+- ✅ **Flexible Evaluators**: Support both code-based and AI-based evaluation
+- ✅ **Statistical Analysis**: Analyze success rates, execution times, and output consistency
+- ✅ **Prompt Diff Detection**: Automatically detect differences between module outputs
+- ✅ **Driver Caching**: Reuse drivers for improved memory efficiency
+- ✅ **Detailed Logging**: Comprehensive logging of all executions
+
+## Installation
+
+```bash
+pnpm add @modular-prompt/experiment
+```
+
+## Quick Start
+
+### 1. Create Configuration File
+
+You can use either YAML or TypeScript format.
+
+#### Option A: YAML Configuration (Recommended for static configurations)
+
+Create `examples/experiment.yaml`:
+
+```yaml
+models:
+  gemini-fast:
+    provider: vertexai
+    model: gemini-2.0-flash-exp
+    capabilities: ["tools", "fast"]
+    enabled: true
+
+drivers:
+  vertexai:
+    projectId: your-project-id
+    location: us-central1
+    # Paths are resolved relative to this config file
+    # Can use ~/ for home directory or absolute paths
+    credentialsPath: ./credentials.json
+
+modules:
+  - name: my-module
+    path: ./my-module.ts
+    description: My custom prompt module
+
+testCases:
+  - name: Basic Test
+    description: Test basic functionality
+    input:  # Structured context object (passed to module.compile)
+      query: user question
+      context: additional information
+    models:  # Optional: specify which models to test (uses all enabled if not specified)
+      - gemini-fast
+
+evaluators:
+  - name: json-validator
+    path: ./evaluators/json-validator.ts
+  # Or inline prompt evaluator
+  - name: quality-check
+    prompt:
+      objective:
+        - Evaluate output quality
+      instructions:
+        - Check clarity and accuracy
+
+evaluation:
+  enabled: true
+  model: gemini-fast  # Reference by model name
+```
+
+#### Option B: TypeScript Configuration (For dynamic configurations)
+
+Create `examples/experiment.ts`:
+
+```typescript
+export default {
+  models: {
+    'gemini-fast': {
+      provider: 'vertexai',
+      model: 'gemini-2.0-flash-exp',
+      capabilities: ['tools', 'fast'],
+      enabled: true,
+    },
+  },
+  drivers: {
+    vertexai: {
+      projectId: 'your-project-id',
+      location: 'us-central1',
+      credentialsPath: './credentials.json',
+    },
+  },
+  modules: [
+    {
+      name: 'my-module',
+      path: './my-module.ts',
+      description: 'My custom prompt module',
+    },
+  ],
+  testCases: [
+    {
+      name: 'Basic Test',
+      description: 'Test basic functionality',
+      input: {  // Structured context object
+        query: 'user question',
+        options: { temperature: 0.7 },
+      },
+      models: ['gemini-fast'],  // Optional
+    },
+  ],
+  evaluators: [
+    {
+      name: 'json-validator',
+      path: './evaluators/json-validator.ts',
+    },
+  ],
+  evaluation: {
+    enabled: true,
+    model: 'gemini-fast',  // Reference by model name
+  },
+};
+```
+
+**TypeScript Support**: TypeScript configuration files are automatically transpiled using [jiti](https://github.com/unjs/jiti). You can use TypeScript syntax directly without pre-compilation. Type annotations are stripped automatically, and the file is executed as JavaScript.
+
+**Important**: All file paths in the configuration (modules, evaluators, credentials) are resolved relative to the config file location.
+
+### 2. Run Experiment
+
+```bash
+# Run with YAML config
+npx modular-experiment examples/experiment.yaml
+
+# Run with TypeScript config
+npx modular-experiment examples/experiment.ts
+
+# Run specific module
+npx modular-experiment examples/experiment.yaml --modules my-module
+
+# Run with evaluation
+npx modular-experiment examples/experiment.yaml --evaluate
+
+# Run multiple times for statistics
+npx modular-experiment examples/experiment.yaml --repeat 10
+```
+
+## Configuration
+
+### Module Definition
+
+Modules can be defined inline or loaded from external files:
+
+```typescript
+// External file
+export const modules: ModuleReference[] = [
+  {
+    name: 'my-module',
+    path: './modules/my-module.ts',
+    description: 'Description',
+  },
+];
+```
+
+A module file should export a default object with:
+
+```typescript
+import { compile } from '@modular-prompt/core';
+import { myPromptModule } from './prompts.js';
+
+export default {
+  name: 'My Module',
+  description: 'Module description',
+  compile: (context: any) => compile(myPromptModule, context),
+};
+```
+
+### Evaluator Definition
+
+Two types of evaluators are supported:
+
+#### 1. Code Evaluator
+
+Programmatic validation (e.g., JSON structure validation):
+
+```typescript
+import type { CodeEvaluator, EvaluationContext, EvaluationResult } from '@modular-prompt/experiment';
+
+export default {
+  name: 'JSON Validator',
+  description: 'Validates JSON structure in output',
+
+  async evaluate(context: EvaluationContext): Promise<EvaluationResult> {
+    // Validation logic
+    return {
+      evaluator: 'json-validator',
+      moduleName: context.moduleName,
+      score: 10,
+      reasoning: 'Valid JSON structure',
+    };
+  },
+} satisfies CodeEvaluator;
+```
+
+#### 2. Prompt Evaluator
+
+AI-based evaluation using LLM:
+
+```typescript
+import type { PromptEvaluator, EvaluationContext } from '@modular-prompt/experiment';
+import type { PromptModule } from '@modular-prompt/core';
+
+const evaluationModule: PromptModule<EvaluationContext> = {
+  createContext: (): EvaluationContext => ({
+    moduleName: '',
+    prompt: '',
+    runs: [],
+  }),
+
+  objective: [
+    '- Assess output quality',
+  ],
+
+  instructions: [
+    '- Evaluate clarity and accuracy',
+  ],
+};
+
+export default {
+  name: 'Quality Evaluator',
+  description: 'Evaluates output quality',
+  module: evaluationModule,
+} satisfies PromptEvaluator;
+```
+
+All prompt evaluators are automatically merged with the base evaluation module.
+
+## Architecture
+
+```
+┌─────────────────────────────────────────┐
+│ run-comparison.ts (CLI Entry Point)    │
+└─────────────────────────────────────────┘
+              │
+    ┌─────────┼─────────┐
+    ▼         ▼         ▼
+┌────────┐ ┌────────┐ ┌────────┐
+│ Config │ │ Runner │ │Reporter│
+│ Loader │ │        │ │        │
+└────────┘ └────────┘ └────────┘
+    │         │
+    ▼         ▼
+┌────────┐ ┌────────┐
+│Dynamic │ │Driver  │
+│Loader  │ │Manager │
+└────────┘ └────────┘
+```
+
+### Components
+
+| Component | Responsibility |
+|-----------|----------------|
+| `config/loader.ts` | Load YAML configuration |
+| `config/dynamic-loader.ts` | Dynamic module/evaluator loading |
+| `runner/experiment.ts` | Orchestrate experiment execution |
+| `runner/evaluator.ts` | Execute evaluations |
+| `runner/driver-manager.ts` | Cache and manage AI drivers |
+| `reporter/statistics.ts` | Generate statistical reports |
+| `evaluators/base-module.ts` | Base evaluation prompt module |
+
+## Examples
+
+See `examples/experiment.yaml` for a complete configuration template with:
+- Model definitions (MLX, Vertex AI, Google GenAI)
+- Driver configurations with credential paths
+- Evaluation settings
+- Empty sections for modules, test cases, and evaluators (ready for your content)
+
+## API
+
+### Programmatic Usage
+
+```typescript
+import {
+  loadExperimentConfig,
+  loadModules,
+  loadEvaluators,
+  ExperimentRunner,
+  DriverManager,
+} from '@modular-prompt/experiment';
+
+const { serverConfig, aiService } = loadExperimentConfig('config.yaml');
+const modules = await loadModules(moduleRefs, basePath);
+const evaluators = await loadEvaluators(evaluatorRefs, basePath);
+
+const driverManager = new DriverManager();
+const runner = new ExperimentRunner(
+  aiService,
+  driverManager,
+  modules,
+  testCases,
+  models,
+  repeatCount,
+  evaluators,
+  evaluatorModel
+);
+
+const results = await runner.run();
+await driverManager.cleanup();
+```
+
+## CLI Options
+
+```
+Usage: modular-experiment <config> [options]
+
+Arguments:
+  <config>                Config file path (YAML or TypeScript)
+
+Options:
+  --test-case <name>      Test case name filter
+  --model <provider>      Model provider filter
+  --modules <names>       Comma-separated module names (default: all)
+  --repeat <count>        Number of repetitions (default: 1)
+  --evaluate              Enable evaluation phase
+  --evaluators <names>    Comma-separated evaluator names (default: all)
+```
+
+**Note**: All paths specified in the config file are resolved relative to the config file's directory.
+
+## License
+
+MIT

package/dist/src/cli/args.d.ts

@@ -0,0 +1,6 @@
+/**
+ * CLI argument parsing using commander
+ */
+import type { ExtendedExperimentOptions } from '../types.js';
+export declare function parseArgs(): ExtendedExperimentOptions;
+//# sourceMappingURL=args.d.ts.map

package/dist/src/cli/args.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"args.d.ts","sourceRoot":"","sources":["../../../src/cli/args.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,aAAa,CAAC;AAE7D,wBAAgB,SAAS,IAAI,yBAAyB,CA2BrD"}

package/dist/src/cli/args.js

@@ -0,0 +1,31 @@
+/**
+ * CLI argument parsing using commander
+ */
+import { Command } from 'commander';
+import { resolve } from 'path';
+export function parseArgs() {
+    const program = new Command();
+    program
+        .name('moduler-experiment')
+        .description('Compare multiple prompt module variations')
+        .argument('<config>', 'Config file path (YAML, TypeScript, or JavaScript)')
+        .option('--test-case <name>', 'Test case name filter')
+        .option('--model <provider>', 'Model provider filter (mlx, vertexai, googlegenai)')
+        .option('--modules <names>', 'Comma-separated module names to test (default: all)')
+        .option('--repeat <count>', 'Number of repetitions', '1')
+        .option('--evaluate', 'Enable AI-based evaluation of outputs', false)
+        .option('--evaluators <names>', 'Comma-separated evaluator names (default: all)')
+        .parse();
+    const config = program.args[0];
+    const options = program.opts();
+    return {
+        configPath: resolve(process.cwd(), config),
+        testCaseFilter: options.testCase,
+        modelFilter: options.model,
+        moduleFilter: options.modules?.split(',').map((s) => s.trim()),
+        repeatCount: parseInt(options.repeat, 10),
+        enableEvaluation: options.evaluate,
+        evaluatorFilter: options.evaluators?.split(',').map((s) => s.trim()),
+    };
+}
+//# sourceMappingURL=args.js.map

package/dist/src/cli/args.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"args.js","sourceRoot":"","sources":["../../../src/cli/args.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAG/B,MAAM,UAAU,SAAS;IACvB,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;IAE9B,OAAO;SACJ,IAAI,CAAC,oBAAoB,CAAC;SAC1B,WAAW,CAAC,2CAA2C,CAAC;SACxD,QAAQ,CAAC,UAAU,EAAE,oDAAoD,CAAC;SAC1E,MAAM,CAAC,oBAAoB,EAAE,uBAAuB,CAAC;SACrD,MAAM,CAAC,oBAAoB,EAAE,oDAAoD,CAAC;SAClF,MAAM,CAAC,mBAAmB,EAAE,qDAAqD,CAAC;SAClF,MAAM,CAAC,kBAAkB,EAAE,uBAAuB,EAAE,GAAG,CAAC;SACxD,MAAM,CAAC,YAAY,EAAE,uCAAuC,EAAE,KAAK,CAAC;SACpE,MAAM,CAAC,sBAAsB,EAAE,gDAAgD,CAAC;SAChF,KAAK,EAAE,CAAC;IAEX,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC/B,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAE/B,OAAO;QACL,UAAU,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,CAAC;QAC1C,cAAc,EAAE,OAAO,CAAC,QAAQ;QAChC,WAAW,EAAE,OAAO,CAAC,KAAK;QAC1B,YAAY,EAAE,OAAO,CAAC,OAAO,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACtE,WAAW,EAAE,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;QACzC,gBAAgB,EAAE,OAAO,CAAC,QAAQ;QAClC,eAAe,EAAE,OAAO,CAAC,UAAU,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;KAC7E,CAAC;AACJ,CAAC"}

package/dist/src/config/dynamic-loader.d.ts

@@ -0,0 +1,41 @@
+/**
+ * Dynamic loader for evaluators and modules
+ *
+ * Loads evaluators and modules from external files or inline definitions
+ */
+import type { EvaluatorReference, CodeEvaluator, PromptEvaluator, ModuleDefinition } from '../types.js';
+/**
+ * Loaded evaluator (unified type)
+ */
+export interface LoadedEvaluator {
+    name: string;
+    description: string;
+    type: 'code' | 'prompt';
+    codeEvaluator?: CodeEvaluator;
+    promptEvaluator?: PromptEvaluator;
+}
+/**
+ * Load evaluators from references
+ *
+ * @param refs - Array of evaluator references
+ * @param basePath - Base path for resolving relative paths
+ * @returns Array of loaded evaluators
+ */
+export declare function loadEvaluators(refs: EvaluatorReference[], basePath: string): Promise<LoadedEvaluator[]>;
+/**
+ * Module reference in config file
+ */
+export type ModuleReference = {
+    name: string;
+    path: string;
+    description?: string;
+};
+/**
+ * Load modules from references
+ *
+ * @param refs - Array of module references
+ * @param basePath - Base path for resolving relative paths
+ * @returns Array of module definitions
+ */
+export declare function loadModules(refs: ModuleReference[], basePath: string): Promise<ModuleDefinition[]>;
+//# sourceMappingURL=dynamic-loader.d.ts.map

package/dist/src/config/dynamic-loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dynamic-loader.d.ts","sourceRoot":"","sources":["../../../src/config/dynamic-loader.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAKH,OAAO,KAAK,EACV,kBAAkB,EAClB,aAAa,EACb,eAAe,EACf,gBAAgB,EACjB,MAAM,aAAa,CAAC;AAGrB;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,GAAG,QAAQ,CAAC;IACxB,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,eAAe,CAAC,EAAE,eAAe,CAAC;CACnC;AAED;;;;;;GAMG;AACH,wBAAsB,cAAc,CAClC,IAAI,EAAE,kBAAkB,EAAE,EAC1B,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,eAAe,EAAE,CAAC,CA0D5B;AAED;;GAEG;AACH,MAAM,MAAM,eAAe,GACvB;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAEzD;;;;;;GAMG;AACH,wBAAsB,WAAW,CAC/B,IAAI,EAAE,eAAe,EAAE,EACvB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAsB7B"}

package/dist/src/config/dynamic-loader.js

@@ -0,0 +1,101 @@
+/**
+ * Dynamic loader for evaluators and modules
+ *
+ * Loads evaluators and modules from external files or inline definitions
+ */
+import { merge } from '@modular-prompt/core';
+import { pathToFileURL } from 'url';
+import { resolve } from 'path';
+import { baseEvaluationModule } from '../evaluators/base-module.js';
+/**
+ * Load evaluators from references
+ *
+ * @param refs - Array of evaluator references
+ * @param basePath - Base path for resolving relative paths
+ * @returns Array of loaded evaluators
+ */
+export async function loadEvaluators(refs, basePath) {
+    const evaluators = [];
+    for (const ref of refs) {
+        if ('path' in ref) {
+            // External file
+            const filePath = resolve(basePath, ref.path);
+            const fileUrl = pathToFileURL(filePath).href;
+            const imported = await import(fileUrl);
+            const evaluator = imported.default;
+            if (!evaluator) {
+                console.warn(`⚠️  No default export in ${ref.path}`);
+                continue;
+            }
+            // Detect type by checking properties
+            if ('evaluate' in evaluator && typeof evaluator.evaluate === 'function') {
+                // Code evaluator
+                evaluators.push({
+                    name: ref.name,
+                    description: ref.description || evaluator.description || '',
+                    type: 'code',
+                    codeEvaluator: evaluator,
+                });
+            }
+            else if ('module' in evaluator) {
+                // Prompt evaluator - merge with base module
+                const mergedModule = merge(baseEvaluationModule, evaluator.module);
+                evaluators.push({
+                    name: ref.name,
+                    description: ref.description || evaluator.description || '',
+                    type: 'prompt',
+                    promptEvaluator: {
+                        name: evaluator.name,
+                        description: evaluator.description,
+                        module: mergedModule,
+                    },
+                });
+            }
+            else {
+                console.warn(`⚠️  Unknown evaluator type in ${ref.path}`);
+            }
+        }
+        else if ('prompt' in ref) {
+            // Inline prompt definition - merge with base module
+            const mergedModule = merge(baseEvaluationModule, ref.prompt);
+            evaluators.push({
+                name: ref.name,
+                description: ref.description || '',
+                type: 'prompt',
+                promptEvaluator: {
+                    name: ref.name,
+                    description: ref.description || '',
+                    module: mergedModule,
+                },
+            });
+        }
+    }
+    return evaluators;
+}
+/**
+ * Load modules from references
+ *
+ * @param refs - Array of module references
+ * @param basePath - Base path for resolving relative paths
+ * @returns Array of module definitions
+ */
+export async function loadModules(refs, basePath) {
+    const modules = [];
+    for (const ref of refs) {
+        const filePath = resolve(basePath, ref.path);
+        const fileUrl = pathToFileURL(filePath).href;
+        const imported = await import(fileUrl);
+        const module = imported.default;
+        if (!module) {
+            console.warn(`⚠️  No default export in ${ref.path}`);
+            continue;
+        }
+        modules.push({
+            name: ref.name,
+            description: ref.description || module.description || '',
+            compile: module.compile,
+        });
+    }
+    return modules;
+}
+//# sourceMappingURL=dynamic-loader.js.map

package/dist/src/config/dynamic-loader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dynamic-loader.js","sourceRoot":"","sources":["../../../src/config/dynamic-loader.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAO/B,OAAO,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAC;AAapE;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,IAA0B,EAC1B,QAAgB;IAEhB,MAAM,UAAU,GAAsB,EAAE,CAAC;IAEzC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;YAClB,gBAAgB;YAChB,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;YAC7C,MAAM,OAAO,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC;YAC7C,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC;YACvC,MAAM,SAAS,GAAG,QAAQ,CAAC,OAAO,CAAC;YAEnC,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC,4BAA4B,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;gBACrD,SAAS;YACX,CAAC;YAED,qCAAqC;YACrC,IAAI,UAAU,IAAI,SAAS,IAAI,OAAO,SAAS,CAAC,QAAQ,KAAK,UAAU,EAAE,CAAC;gBACxE,iBAAiB;gBACjB,UAAU,CAAC,IAAI,CAAC;oBACd,IAAI,EAAE,GAAG,CAAC,IAAI;oBACd,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,SAAS,CAAC,WAAW,IAAI,EAAE;oBAC3D,IAAI,EAAE,MAAM;oBACZ,aAAa,EAAE,SAA0B;iBAC1C,CAAC,CAAC;YACL,CAAC;iBAAM,IAAI,QAAQ,IAAI,SAAS,EAAE,CAAC;gBACjC,4CAA4C;gBAC5C,MAAM,YAAY,GAAG,KAAK,CAAC,oBAAoB,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;gBACnE,UAAU,CAAC,IAAI,CAAC;oBACd,IAAI,EAAE,GAAG,CAAC,IAAI;oBACd,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,SAAS,CAAC,WAAW,IAAI,EAAE;oBAC3D,IAAI,EAAE,QAAQ;oBACd,eAAe,EAAE;wBACf,IAAI,EAAE,SAAS,CAAC,IAAI;wBACpB,WAAW,EAAE,SAAS,CAAC,WAAW;wBAClC,MAAM,EAAE,YAAY;qBACrB;iBACF,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,IAAI,CAAC,iCAAiC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;YAC5D,CAAC;QACH,CAAC;aAAM,IAAI,QAAQ,IAAI,GAAG,EAAE,CAAC;YAC3B,oDAAoD;YACpD,MAAM,YAAY,GAAG,KAAK,CAAC,oBAAoB,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;YAC7D,UAAU,CAAC,IAAI,CAAC;gBACd,IAAI,EAAE,GAAG,CAAC,IAAI;gBACd,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,EAAE;gBAClC,IAAI,EAAE,QAAQ;gBACd,eAAe,EAAE;oBACf,IAAI,EAAE,GAAG,CAAC,IAAI;oBACd,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,EAAE;oBAClC,MAAM,EAAE,YAAY;iBACrB;aACF,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AAQD;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,IAAuB,EACvB,QAAgB;IAEhB,MAAM,OAAO,GAAuB,EAAE,CAAC;IAEvC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;QAC7C,MAAM,OAAO,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC;QAC7C,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC;QACvC,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC;QAEhC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,CAAC,IAAI,CAAC,4BAA4B,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;YACrD,SAAS;QACX,CAAC;QAED,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,MAAM,CAAC,WAAW,IAAI,EAAE;YACxD,OAAO,EAAE,MAAM,CAAC,OAAO;SACxB,CAAC,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/dist/src/config/loader.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Configuration loader
+ */
+import { AIService } from '@modular-prompt/driver';
+import type { ModuleReference } from './dynamic-loader.js';
+import type { EvaluatorReference, TestCase } from '../types.js';
+export interface LoadedConfig {
+    serverConfig: any;
+    modules: ModuleReference[];
+    testCases: TestCase[];
+    evaluators: EvaluatorReference[];
+    aiService: AIService;
+    configDir: string;
+}
+export type ExperimentConfig = LoadedConfig;
+/**
+ * Load experiment configuration
+ *
+ * @param configPath - Path to config file (YAML or TypeScript)
+ * @returns LoadedConfig with all configuration
+ */
+export declare function loadExperimentConfig(configPath: string): Promise<LoadedConfig>;
+//# sourceMappingURL=loader.d.ts.map

package/dist/src/config/loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"loader.d.ts","sourceRoot":"","sources":["../../../src/config/loader.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH,OAAO,EAAE,SAAS,EAA0B,MAAM,wBAAwB,CAAC;AAC3E,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC3D,OAAO,KAAK,EAAE,kBAAkB,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAEhE,MAAM,WAAW,YAAY;IAC3B,YAAY,EAAE,GAAG,CAAC;IAClB,OAAO,EAAE,eAAe,EAAE,CAAC;IAC3B,SAAS,EAAE,QAAQ,EAAE,CAAC;IACtB,UAAU,EAAE,kBAAkB,EAAE,CAAC;IACjC,SAAS,EAAE,SAAS,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;CACnB;AAGD,MAAM,MAAM,gBAAgB,GAAG,YAAY,CAAC;AAsB5C;;;;;GAKG;AACH,wBAAsB,oBAAoB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAuGpF"}