@modular-prompt/experiment 0.1.1 → 0.1.3

package/README.md

@@ -69,8 +69,12 @@ testCases:
       - gemini-fast
 
 evaluators:
-  - name: json-validator
-    path: ./evaluators/json-validator.ts
+  # Built-in evaluators (name only)
+  - name: structured-output-presence
+  - name: llm-requirement-fulfillment
+  # Or external evaluator (with path)
+  - name: custom-validator
+    path: ./evaluators/custom-validator.ts
   # Or inline prompt evaluator
   - name: quality-check
     prompt:
@@ -124,9 +128,13 @@ export default {
     },
   ],
   evaluators: [
+    // Built-in evaluators (name only)
+    { name: 'structured-output-presence' },
+    { name: 'llm-requirement-fulfillment' },
+    // Or external evaluator (with path)
     {
-      name: 'json-validator',
-      path: './evaluators/json-validator.ts',
+      name: 'custom-validator',
+      path: './evaluators/custom-validator.ts',
     },
   ],
   evaluation: {
@@ -143,6 +151,9 @@ export default {
 ### 2. Run Experiment
 
 ```bash
+# Validate configuration and display execution plan (recommended first step)
+npx modular-experiment examples/experiment.yaml --dry-run
+
 # Run with YAML config
 npx modular-experiment examples/experiment.yaml
 
@@ -157,6 +168,15 @@ npx modular-experiment examples/experiment.yaml --evaluate
 
 # Run multiple times for statistics
 npx modular-experiment examples/experiment.yaml --repeat 10
+
+# Run with detailed logging to JSONL file
+npx modular-experiment examples/experiment.yaml --log-file experiment.jsonl
+
+# Run with verbose output (show internal operations)
+npx modular-experiment examples/experiment.yaml --verbose
+
+# Combine options
+npx modular-experiment examples/experiment.yaml --evaluate --log-file experiment.jsonl --verbose
 ```
 
 ## Configuration
@@ -249,6 +269,48 @@ export default {
 
 All prompt evaluators are automatically merged with the base evaluation module.
 
+## Built-in Evaluators
+
+The framework includes built-in evaluators that can be referenced by name only (no path required):
+
+### structured-output-presence
+
+- **Type**: Code Evaluator
+- **What it measures**: Checks if `structuredOutput` exists and is a valid object
+- **Evaluation logic**:
+  - Verifies presence of `structuredOutput` in query result
+  - Confirms it's a non-null object type
+- **Score**: `(validCount / totalRuns) * 10`
+- **Use case**: Verify that the model returns structured JSON output (essential for structured output workflows)
+- **Usage**:
+  ```yaml
+  evaluators:
+    - name: "structured-output-presence"
+  ```
+
+### llm-requirement-fulfillment
+
+- **Type**: Prompt Evaluator (uses LLM for evaluation)
+- **What it measures**: Uses LLM to comprehensively evaluate whether output meets functional requirements
+- **Evaluation criteria**:
+  1. **Requirement Fulfillment**: Does it satisfy the intent described in the prompt?
+  2. **Parameter Correctness**: Are all required parameters present and correct?
+  3. **Parameter Completeness**: Are optional parameters appropriately used or omitted?
+  4. **Logical Consistency**: Is the output logically consistent with the facts?
+- **Score**: 0-10 overall score with detailed sub-scores for each criterion
+- **Use case**: Comprehensive quality assessment of output (requires evaluation model to be configured)
+- **Usage**:
+  ```yaml
+  evaluators:
+    - name: "llm-requirement-fulfillment"
+
+  evaluation:
+    enabled: true
+    model: "gemini-fast"  # Model used for evaluation
+  ```
+
+**Note**: `llm-requirement-fulfillment` requires an evaluation model to be configured in the `evaluation` section.
+
 ## Architecture
 
 ```
@@ -280,7 +342,8 @@ All prompt evaluators are automatically merged with the base evaluation module.
 | `runner/evaluator.ts` | Execute evaluations |
 | `runner/driver-manager.ts` | Cache and manage AI drivers |
 | `reporter/statistics.ts` | Generate statistical reports |
-| `evaluators/base-module.ts` | Base evaluation prompt module |
+| `base-evaluation-module.ts` | Base evaluation prompt module |
+| `evaluators/index.ts` | Built-in evaluator registry |
 
 ## Examples
 
@@ -338,6 +401,9 @@ Options:
   --repeat <count>        Number of repetitions (default: 1)
   --evaluate              Enable evaluation phase
   --evaluators <names>    Comma-separated evaluator names (default: all)
+  --dry-run               Display execution plan without running the experiment
+  --log-file <path>       Log file path for JSONL output (detailed logs)
+  --verbose               Enable verbose output (show detailed internal operations)
 ```
 
 **Note**: All paths specified in the config file are resolved relative to the config file's directory.

package/dist/{src/evaluators/base-module.d.ts → base-evaluation-module.d.ts}

@@ -5,6 +5,6 @@
  * It defines how test data is presented to the evaluator.
  */
 import type { PromptModule } from '@modular-prompt/core';
-import type { EvaluationContext } from '../types.js';
+import type { EvaluationContext } from './types.js';
 export declare const baseEvaluationModule: PromptModule<EvaluationContext>;
-//# sourceMappingURL=base-module.d.ts.map
+//# sourceMappingURL=base-evaluation-module.d.ts.map

package/dist/base-evaluation-module.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"base-evaluation-module.d.ts","sourceRoot":"","sources":["../src/base-evaluation-module.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,YAAY,EAA4B,MAAM,sBAAsB,CAAC;AACnF,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAEpD,eAAO,MAAM,oBAAoB,EAAE,YAAY,CAAC,iBAAiB,CAuGhE,CAAC"}

package/dist/{src/evaluators/base-module.js → base-evaluation-module.js}

@@ -100,4 +100,4 @@ export const baseEvaluationModule = {
         },
     ],
 };
-//# sourceMappingURL=base-module.js.map
+//# sourceMappingURL=base-evaluation-module.js.map

package/dist/base-evaluation-module.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"base-evaluation-module.js","sourceRoot":"","sources":["../src/base-evaluation-module.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,MAAM,CAAC,MAAM,oBAAoB,GAAoC;IACnE,aAAa,EAAE,GAAsB,EAAE,CAAC,CAAC;QACvC,UAAU,EAAE,EAAE;QACd,MAAM,EAAE,EAAE;QACV,IAAI,EAAE,EAAE;KACT,CAAC;IAEF,SAAS,EAAE;QACT,0CAA0C;QAC1C,yDAAyD;KAC1D;IAED,KAAK,EAAE;QACL,2CAA2C;QAC3C,2DAA2D;QAC3D,iDAAiD;QACjD,yCAAyC;KAC1C;IAED,YAAY,EAAE;QACZ;YACE,IAAI,EAAE,YAAY;YAClB,KAAK,EAAE,eAAe;YACtB,KAAK,EAAE;gBACL,gEAAgE;gBAChE,+BAA+B;gBAC/B,6CAA6C;gBAC7C,sDAAsD;aACvD;SACF;KACF;IAED,SAAS,EAAE;QACT;YACE,IAAI,EAAE,YAAY;YAClB,KAAK,EAAE,aAAa;YACpB,KAAK,EAAE;gBACL,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU;aACxB;SACF;QACD;YACE,IAAI,EAAE,YAAY;YAClB,KAAK,EAAE,aAAa;YACpB,KAAK,EAAE;gBACL,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;oBACR,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE,GAAG,CAAC,MAAM;iBACJ,CAAA;aAClB;SACF;KACF;IAED,MAAM,EAAE;QACN,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;YACrC,MAAM,MAAM,GAAG,GAAG,CAAC,WAAW,CAAC;YAC/B,MAAM,QAAQ,GAAqC,EAAE,CAAC;YAEtD,aAAa;YACb,QAAQ,CAAC,IAAI,CAAC;gBACZ,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE,OAAO,GAAG,GAAG,CAAC,EAAE;aAC1B,CAAC,CAAC;YAEH,gDAAgD;YAChD,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,QAAQ,CAAC,IAAI,CAAC;oBACZ,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE,MAAM,CAAC,gBAAgB;iBACjC,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,QAAQ,CAAC,IAAI,CAAC;oBACZ,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE,MAAM,CAAC,OAAO;iBACxB,CAAC,CAAC;YACL,CAAC;YAED,OAAO,QAAQ,CAAC;QAClB,CAAC,CAAC;KACH;IAED,MAAM,EAAE;QACN;YACE,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE;gBACP,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,KAAK,EAAE;wBACL,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,sBAAsB;qBACpC;oBACD,SAAS,EAAE;wBACT,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,0BAA0B;qBACxC;oBACD,OAAO,EAAE;wBACP,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,+BAA+B;qBAC7C;iBACF;gBACD,QAAQ,EAAE,CAAC,OAAO,EAAE,WAAW,CAAC;aACjC;SACF;KACF;CACF,CAAC"}

package/dist/cli/args.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"args.d.ts","sourceRoot":"","sources":["../../src/cli/args.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,aAAa,CAAC;AAE7D,wBAAgB,SAAS,IAAI,yBAAyB,CAiCrD"}

package/dist/{src/cli → cli}/args.js

@@ -15,6 +15,9 @@ export function parseArgs() {
         .option('--repeat <count>', 'Number of repetitions', '1')
         .option('--evaluate', 'Enable AI-based evaluation of outputs', false)
         .option('--evaluators <names>', 'Comma-separated evaluator names (default: all)')
+        .option('--dry-run', 'Display execution plan without running the experiment', false)
+        .option('--log-file <path>', 'Log file path for JSONL output (detailed logs)')
+        .option('--verbose', 'Enable verbose output (show detailed internal operations)', false)
         .parse();
     const config = program.args[0];
     const options = program.opts();
@@ -26,6 +29,9 @@ export function parseArgs() {
         repeatCount: parseInt(options.repeat, 10),
         enableEvaluation: options.evaluate,
         evaluatorFilter: options.evaluators?.split(',').map((s) => s.trim()),
+        dryRun: options.dryRun,
+        logFile: options.logFile,
+        verbose: options.verbose,
     };
 }
 //# sourceMappingURL=args.js.map

package/dist/cli/args.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"args.js","sourceRoot":"","sources":["../../src/cli/args.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAG/B,MAAM,UAAU,SAAS;IACvB,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;IAE9B,OAAO;SACJ,IAAI,CAAC,oBAAoB,CAAC;SAC1B,WAAW,CAAC,2CAA2C,CAAC;SACxD,QAAQ,CAAC,UAAU,EAAE,oDAAoD,CAAC;SAC1E,MAAM,CAAC,oBAAoB,EAAE,uBAAuB,CAAC;SACrD,MAAM,CAAC,oBAAoB,EAAE,oDAAoD,CAAC;SAClF,MAAM,CAAC,mBAAmB,EAAE,qDAAqD,CAAC;SAClF,MAAM,CAAC,kBAAkB,EAAE,uBAAuB,EAAE,GAAG,CAAC;SACxD,MAAM,CAAC,YAAY,EAAE,uCAAuC,EAAE,KAAK,CAAC;SACpE,MAAM,CAAC,sBAAsB,EAAE,gDAAgD,CAAC;SAChF,MAAM,CAAC,WAAW,EAAE,uDAAuD,EAAE,KAAK,CAAC;SACnF,MAAM,CAAC,mBAAmB,EAAE,gDAAgD,CAAC;SAC7E,MAAM,CAAC,WAAW,EAAE,2DAA2D,EAAE,KAAK,CAAC;SACvF,KAAK,EAAE,CAAC;IAEX,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC/B,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAE/B,OAAO;QACL,UAAU,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,CAAC;QAC1C,cAAc,EAAE,OAAO,CAAC,QAAQ;QAChC,WAAW,EAAE,OAAO,CAAC,KAAK;QAC1B,YAAY,EAAE,OAAO,CAAC,OAAO,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACtE,WAAW,EAAE,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;QACzC,gBAAgB,EAAE,OAAO,CAAC,QAAQ;QAClC,eAAe,EAAE,OAAO,CAAC,UAAU,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC5E,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,OAAO,EAAE,OAAO,CAAC,OAAO;KACzB,CAAC;AACJ,CAAC"}

package/dist/config/dynamic-loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dynamic-loader.d.ts","sourceRoot":"","sources":["../../src/config/dynamic-loader.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAKH,OAAO,KAAK,EACV,kBAAkB,EAClB,aAAa,EACb,eAAe,EACf,gBAAgB,EACjB,MAAM,aAAa,CAAC;AAOrB;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,GAAG,QAAQ,CAAC;IACxB,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,eAAe,CAAC,EAAE,eAAe,CAAC;CACnC;AAED;;;;;;GAMG;AACH,wBAAsB,cAAc,CAClC,IAAI,EAAE,kBAAkB,EAAE,EAC1B,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,eAAe,EAAE,CAAC,CAqE5B;AAED;;GAEG;AACH,MAAM,MAAM,eAAe,GACvB;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAEzD;;;;;;GAMG;AACH,wBAAsB,WAAW,CAC/B,IAAI,EAAE,eAAe,EAAE,EACvB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAsB7B"}

package/dist/{src/config → config}/dynamic-loader.js

@@ -6,7 +6,10 @@
 import { merge } from '@modular-prompt/core';
 import { pathToFileURL } from 'url';
 import { resolve } from 'path';
-import { baseEvaluationModule } from '../evaluators/base-module.js';
+import { baseEvaluationModule } from '../base-evaluation-module.js';
+import { getBuiltinEvaluator } from '../evaluators/index.js';
+import { logger as baseLogger } from '../logger.js';
+const logger = baseLogger.context('dynamic-loader');
 /**
  * Load evaluators from references
  *
@@ -17,43 +20,17 @@ import { baseEvaluationModule } from '../evaluators/base-module.js';
 export async function loadEvaluators(refs, basePath) {
     const evaluators = [];
     for (const ref of refs) {
+        let evaluator;
         if ('path' in ref) {
             // External file
             const filePath = resolve(basePath, ref.path);
             const fileUrl = pathToFileURL(filePath).href;
             const imported = await import(fileUrl);
-            const evaluator = imported.default;
+            evaluator = imported.default;
             if (!evaluator) {
-                console.warn(`⚠️  No default export in ${ref.path}`);
+                logger.warn(`No default export in ${ref.path}`);
                 continue;
             }
-            // Detect type by checking properties
-            if ('evaluate' in evaluator && typeof evaluator.evaluate === 'function') {
-                // Code evaluator
-                evaluators.push({
-                    name: ref.name,
-                    description: ref.description || evaluator.description || '',
-                    type: 'code',
-                    codeEvaluator: evaluator,
-                });
-            }
-            else if ('module' in evaluator) {
-                // Prompt evaluator - merge with base module
-                const mergedModule = merge(baseEvaluationModule, evaluator.module);
-                evaluators.push({
-                    name: ref.name,
-                    description: ref.description || evaluator.description || '',
-                    type: 'prompt',
-                    promptEvaluator: {
-                        name: evaluator.name,
-                        description: evaluator.description,
-                        module: mergedModule,
-                    },
-                });
-            }
-            else {
-                console.warn(`⚠️  Unknown evaluator type in ${ref.path}`);
-            }
         }
         else if ('prompt' in ref) {
             // Inline prompt definition - merge with base module
@@ -68,6 +45,42 @@ export async function loadEvaluators(refs, basePath) {
                     module: mergedModule,
                 },
             });
+            continue;
+        }
+        else {
+            // Builtin evaluator (name only)
+            evaluator = getBuiltinEvaluator(ref.name);
+            if (!evaluator) {
+                logger.warn(`Builtin evaluator not found: ${ref.name}`);
+                continue;
+            }
+        }
+        // Detect type by checking properties
+        if ('evaluate' in evaluator && typeof evaluator.evaluate === 'function') {
+            // Code evaluator
+            evaluators.push({
+                name: ref.name,
+                description: ref.description || evaluator.description || '',
+                type: 'code',
+                codeEvaluator: evaluator,
+            });
+        }
+        else if ('module' in evaluator) {
+            // Prompt evaluator - merge with base module
+            const mergedModule = merge(baseEvaluationModule, evaluator.module);
+            evaluators.push({
+                name: ref.name,
+                description: ref.description || evaluator.description || '',
+                type: 'prompt',
+                promptEvaluator: {
+                    name: evaluator.name,
+                    description: evaluator.description,
+                    module: mergedModule,
+                },
+            });
+        }
+        else {
+            logger.warn(`Unknown evaluator type: ${ref.name}`);
         }
     }
     return evaluators;
@@ -87,7 +100,7 @@ export async function loadModules(refs, basePath) {
         const imported = await import(fileUrl);
         const module = imported.default;
         if (!module) {
-            console.warn(`⚠️  No default export in ${ref.path}`);
+            logger.warn(`No default export in ${ref.path}`);
             continue;
         }
         modules.push({

package/dist/config/dynamic-loader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dynamic-loader.js","sourceRoot":"","sources":["../../src/config/dynamic-loader.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAO/B,OAAO,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAC7D,OAAO,EAAE,MAAM,IAAI,UAAU,EAAE,MAAM,cAAc,CAAC;AAEpD,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;AAapD;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,IAA0B,EAC1B,QAAgB;IAEhB,MAAM,UAAU,GAAsB,EAAE,CAAC;IAEzC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,SAAsD,CAAC;QAE3D,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;YAClB,gBAAgB;YAChB,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;YAC7C,MAAM,OAAO,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC;YAC7C,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC;YACvC,SAAS,GAAG,QAAQ,CAAC,OAAO,CAAC;YAE7B,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,MAAM,CAAC,IAAI,CAAC,wBAAwB,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;gBAChD,SAAS;YACX,CAAC;QACH,CAAC;aAAM,IAAI,QAAQ,IAAI,GAAG,EAAE,CAAC;YAC3B,oDAAoD;YACpD,MAAM,YAAY,GAAG,KAAK,CAAC,oBAAoB,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;YAC7D,UAAU,CAAC,IAAI,CAAC;gBACd,IAAI,EAAE,GAAG,CAAC,IAAI;gBACd,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,EAAE;gBAClC,IAAI,EAAE,QAAQ;gBACd,eAAe,EAAE;oBACf,IAAI,EAAE,GAAG,CAAC,IAAI;oBACd,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,EAAE;oBAClC,MAAM,EAAE,YAAY;iBACrB;aACF,CAAC,CAAC;YACH,SAAS;QACX,CAAC;aAAM,CAAC;YACN,gCAAgC;YAChC,SAAS,GAAG,mBAAmB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAE1C,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,MAAM,CAAC,IAAI,CAAC,gCAAgC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;gBACxD,SAAS;YACX,CAAC;QACH,CAAC;QAED,qCAAqC;QACrC,IAAI,UAAU,IAAI,SAAS,IAAI,OAAO,SAAS,CAAC,QAAQ,KAAK,UAAU,EAAE,CAAC;YACxE,iBAAiB;YACjB,UAAU,CAAC,IAAI,CAAC;gBACd,IAAI,EAAE,GAAG,CAAC,IAAI;gBACd,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,SAAS,CAAC,WAAW,IAAI,EAAE;gBAC3D,IAAI,EAAE,MAAM;gBACZ,aAAa,EAAE,SAA0B;aAC1C,CAAC,CAAC;QACL,CAAC;aAAM,IAAI,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,4CAA4C;YAC5C,MAAM,YAAY,GAAG,KAAK,CAAC,oBAAoB,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;YACnE,UAAU,CAAC,IAAI,CAAC;gBACd,IAAI,EAAE,GAAG,CAAC,IAAI;gBACd,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,SAAS,CAAC,WAAW,IAAI,EAAE;gBAC3D,IAAI,EAAE,QAAQ;gBACd,eAAe,EAAE;oBACf,IAAI,EAAE,SAAS,CAAC,IAAI;oBACpB,WAAW,EAAE,SAAS,CAAC,WAAW;oBAClC,MAAM,EAAE,YAAY;iBACrB;aACF,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,2BAA2B,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;QACrD,CAAC;IACH,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AAQD;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,IAAuB,EACvB,QAAgB;IAEhB,MAAM,OAAO,GAAuB,EAAE,CAAC;IAEvC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;QAC7C,MAAM,OAAO,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC;QAC7C,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC;QACvC,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC;QAEhC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,CAAC,IAAI,CAAC,wBAAwB,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;YAChD,SAAS;QACX,CAAC;QAED,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,MAAM,CAAC,WAAW,IAAI,EAAE;YACxD,OAAO,EAAE,MAAM,CAAC,OAAO;SACxB,CAAC,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/dist/config/loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"loader.d.ts","sourceRoot":"","sources":["../../src/config/loader.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH,OAAO,EAAE,SAAS,EAA0B,MAAM,wBAAwB,CAAC;AAC3E,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC3D,OAAO,KAAK,EAAE,kBAAkB,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAKhE,MAAM,WAAW,YAAY;IAC3B,YAAY,EAAE,GAAG,CAAC;IAClB,OAAO,EAAE,eAAe,EAAE,CAAC;IAC3B,SAAS,EAAE,QAAQ,EAAE,CAAC;IACtB,UAAU,EAAE,kBAAkB,EAAE,CAAC;IACjC,SAAS,EAAE,SAAS,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;CACnB;AAGD,MAAM,MAAM,gBAAgB,GAAG,YAAY,CAAC;AAsB5C;;;;;GAKG;AACH,wBAAsB,oBAAoB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAsGpF"}

package/dist/{src/config → config}/loader.js

@@ -6,6 +6,8 @@ import { parse as parseYaml } from 'yaml';
 import { resolve, dirname, extname } from 'path';
 import { createJiti } from 'jiti';
 import { AIService } from '@modular-prompt/driver';
+import { logger as baseLogger } from '../logger.js';
+const logger = baseLogger.context('config-loader');
 /**
  * Resolve path relative to config file directory
  *
@@ -67,9 +69,6 @@ export async function loadExperimentConfig(configPath) {
         drivers: config.drivers,
         evaluation: config.evaluation,
         credentials: config.credentials,
-        selection: config.selection,
-        server: config.server,
-        logging: config.logging,
     };
     // Resolve paths in driver configurations relative to config file
     if (serverConfig.drivers) {
@@ -85,7 +84,7 @@ export async function loadExperimentConfig(configPath) {
     if (serverConfig.credentials?.googleApplicationCredentials) {
         const resolvedPath = resolveConfigPath(configDir, serverConfig.credentials.googleApplicationCredentials);
         process.env.GOOGLE_APPLICATION_CREDENTIALS = resolvedPath;
-        console.log(`Setting GOOGLE_APPLICATION_CREDENTIALS=${resolvedPath}`);
+        logger.verbose(`Setting GOOGLE_APPLICATION_CREDENTIALS=${resolvedPath}`);
     }
     // Validation
     if (!serverConfig.models || Object.keys(serverConfig.models).length === 0) {
@@ -104,6 +103,8 @@ export async function loadExperimentConfig(configPath) {
         }
     }
     // Initialize AIService
+    // Note: AIService is used only as a driver factory.
+    // Model selection is explicit in experiment configuration, not capability-based.
     const aiServiceConfig = {
         models: serverConfig.models,
         drivers: serverConfig.drivers || {},

package/dist/config/loader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"loader.js","sourceRoot":"","sources":["../../src/config/loader.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AAClC,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAC1C,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AACjD,OAAO,EAAE,UAAU,EAAE,MAAM,MAAM,CAAC;AAClC,OAAO,EAAE,SAAS,EAA0B,MAAM,wBAAwB,CAAC;AAG3E,OAAO,EAAE,MAAM,IAAI,UAAU,EAAE,MAAM,cAAc,CAAC;AAEpD,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;AAcnD;;;;;;GAMG;AACH,SAAS,iBAAiB,CAAC,SAAiB,EAAE,IAAY;IACxD,sDAAsD;IACtD,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACzB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC;IACpD,CAAC;IACD,oCAAoC;IACpC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACzB,OAAO,IAAI,CAAC;IACd,CAAC;IACD,kDAAkD;IAClD,OAAO,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;AAClC,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,UAAkB;IAC3D,0CAA0C;IAC1C,MAAM,SAAS,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IACtC,MAAM,GAAG,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IAEhC,iCAAiC;IACjC,IAAI,MAAW,CAAC;IAEhB,IAAI,GAAG,KAAK,OAAO,IAAI,GAAG,KAAK,MAAM,EAAE,CAAC;QACtC,cAAc;QACd,MAAM,OAAO,GAAG,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QAClD,MAAM,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;IAC9B,CAAC;SAAM,IAAI,GAAG,KAAK,KAAK,IAAI,GAAG,KAAK,KAAK,IAAI,GAAG,KAAK,MAAM,IAAI,GAAG,KAAK,MAAM,EAAE,CAAC;QAC9E,0DAA0D;QAC1D,MAAM,IAAI,GAAG,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE;YACvC,cAAc,EAAE,IAAI,EAAG,mCAAmC;YAC1D,KAAK,EAAE,IAAI,EAAY,wCAAwC;YAC/D,YAAY,EAAE,KAAK,EAAI,0BAA0B;SAClD,CAAC,CAAC;QAEH,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAEvC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,0BAA0B,UAAU,EAAE,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CAAC,qCAAqC,GAAG,4CAA4C,CAAC,CAAC;IACxG,CAAC;IAED,qBAAqB;IACrB,MAAM,OAAO,GAAsB,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC;IACxD,MAAM,SAAS,GAAe,MAAM,CAAC,SAAS,IAAI,EAAE,CAAC;IACrD,MAAM,UAAU,GAAyB,MAAM,CAAC,UAAU,IAAI,EAAE,CAAC;IAEjE,oDAAoD;IACpD,MAAM,YAAY,GAAG;QACnB,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,WAAW,EAAE,MAAM,CAAC,WAAW;KAChC,CAAC;IAEF,iEAAiE;IACjE,IAAI,YAAY,CAAC,OAAO,EAAE,CAAC;QACzB,KAAK,MAAM,UAAU,IAAI,YAAY,CAAC,OAAO,EAAE,CAAC;YAC9C,MAAM,YAAY,GAAG,YAAY,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;YAEtD,8CAA8C;YAC9C,IAAI,YAAY,CAAC,eAAe,EAAE,CAAC;gBACjC,YAAY,CAAC,eAAe,GAAG,iBAAiB,CAAC,SAAS,EAAE,YAAY,CAAC,eAAe,CAAC,CAAC;YAC5F,CAAC;QACH,CAAC;IACH,CAAC;IAED,kDAAkD;IAClD,IAAI,YAAY,CAAC,WAAW,EAAE,4BAA4B,EAAE,CAAC;QAC3D,MAAM,YAAY,GAAG,iBAAiB,CAAC,SAAS,EAAE,YAAY,CAAC,WAAW,CAAC,4BAA4B,CAAC,CAAC;QACzG,OAAO,CAAC,GAAG,CAAC,8BAA8B,GAAG,YAAY,CAAC;QAC1D,MAAM,CAAC,OAAO,CAAC,0CAA0C,YAAY,EAAE,CAAC,CAAC;IAC3E,CAAC;IAED,aAAa;IACb,IAAI,CAAC,YAAY,CAAC,MAAM,IAAI,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1E,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IAC3D,CAAC;IAED,mCAAmC;IACnC,MAAM,UAAU,GAAG,IAAI,GAAG,CAAS,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC;IAErE,qCAAqC;IACrC,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;YACpB,KAAK,MAAM,SAAS,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;gBACxC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;oBAC/B,MAAM,IAAI,KAAK,CAAC,eAAe,QAAQ,CAAC,IAAI,+BAA+B,SAAS,GAAG,CAAC,CAAC;gBAC3F,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,uBAAuB;IACvB,oDAAoD;IACpD,iFAAiF;IACjF,MAAM,eAAe,GAAsB;QACzC,MAAM,EAAE,YAAY,CAAC,MAAM;QAC3B,OAAO,EAAE,YAAY,CAAC,OAAO,IAAI,EAAE;QACnC,cAAc,EAAE;YACd,WAAW,EAAE,GAAG;YAChB,SAAS,EAAE,IAAI;SAChB;KACF,CAAC;IAEF,MAAM,SAAS,GAAG,IAAI,SAAS,CAAC,eAAe,CAAC,CAAC;IAEjD,OAAO;QACL,YAAY;QACZ,OAAO;QACP,SAAS;QACT,UAAU;QACV,SAAS;QACT,SAAS;KACV,CAAC;AACJ,CAAC"}

package/dist/evaluators/index.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Built-in evaluators
+ */
+import type { CodeEvaluator, PromptEvaluator } from '../types.js';
+type BuiltinEvaluator = CodeEvaluator | PromptEvaluator;
+export declare const builtinEvaluators: Record<string, BuiltinEvaluator>;
+/**
+ * Get builtin evaluator by name
+ */
+export declare function getBuiltinEvaluator(name: string): BuiltinEvaluator | undefined;
+export {};
+//# sourceMappingURL=index.d.ts.map

package/dist/evaluators/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/evaluators/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,KAAK,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAElE,KAAK,gBAAgB,GAAG,aAAa,GAAG,eAAe,CAAC;AAExD,eAAO,MAAM,iBAAiB,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAG9D,CAAC;AAEF;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,gBAAgB,GAAG,SAAS,CAE9E"}

package/dist/evaluators/index.js

@@ -0,0 +1,16 @@
+/**
+ * Built-in evaluators
+ */
+import structuredOutputPresence from './structured-output-presence.js';
+import llmRequirementFulfillment from './llm-requirement-fulfillment.js';
+export const builtinEvaluators = {
+    'structured-output-presence': structuredOutputPresence,
+    'llm-requirement-fulfillment': llmRequirementFulfillment,
+};
+/**
+ * Get builtin evaluator by name
+ */
+export function getBuiltinEvaluator(name) {
+    return builtinEvaluators[name];
+}
+//# sourceMappingURL=index.js.map

package/dist/evaluators/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/evaluators/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,wBAAwB,MAAM,iCAAiC,CAAC;AACvE,OAAO,yBAAyB,MAAM,kCAAkC,CAAC;AAKzE,MAAM,CAAC,MAAM,iBAAiB,GAAqC;IACjE,4BAA4B,EAAE,wBAAwB;IACtD,6BAA6B,EAAE,yBAAyB;CACzD,CAAC;AAEF;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,IAAY;IAC9C,OAAO,iBAAiB,CAAC,IAAI,CAAC,CAAC;AACjC,CAAC"}

package/dist/{src/evaluators/functional-correctness.d.ts → evaluators/llm-requirement-fulfillment.d.ts}

@@ -1,7 +1,7 @@
 /**
- * Functional Correctness Evaluator
+ * LLM Requirement Fulfillment Evaluator
  *
- * Evaluates whether the output meets the functional requirements
+ * Uses LLM to evaluate whether the output meets the functional requirements
  */
 import type { PromptModule } from '@modular-prompt/core';
 import type { EvaluationContext } from '../types.js';
@@ -11,4 +11,4 @@ declare const _default: {
     module: PromptModule<EvaluationContext>;
 };
 export default _default;
-//# sourceMappingURL=functional-correctness.d.ts.map
+//# sourceMappingURL=llm-requirement-fulfillment.d.ts.map

package/dist/evaluators/llm-requirement-fulfillment.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-requirement-fulfillment.d.ts","sourceRoot":"","sources":["../../src/evaluators/llm-requirement-fulfillment.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,KAAK,EAAmB,iBAAiB,EAAE,MAAM,aAAa,CAAC;;;;;;AA0FtE,wBAI4B"}

package/dist/{src/evaluators/functional-correctness.js → evaluators/llm-requirement-fulfillment.js}

@@ -1,9 +1,9 @@
 /**
- * Functional Correctness Evaluator
+ * LLM Requirement Fulfillment Evaluator
  *
- * Evaluates whether the output meets the functional requirements
+ * Uses LLM to evaluate whether the output meets the functional requirements
  */
-const functionalCorrectnessModule = {
+const llmRequirementFulfillmentModule = {
     createContext: () => ({
         moduleName: '',
         prompt: '',
@@ -88,8 +88,8 @@ const functionalCorrectnessModule = {
     ],
 };
 export default {
-    name: 'Functional Correctness',
-    description: 'Evaluates whether the output meets the functional requirements',
-    module: functionalCorrectnessModule,
+    name: 'LLM Requirement Fulfillment',
+    description: 'Overall requirement fulfillment score based on LLM evaluation. Compares prompt and output to evaluate requirement fulfillment, parameter correctness, completeness, and logical consistency.',
+    module: llmRequirementFulfillmentModule,
 };
-//# sourceMappingURL=functional-correctness.js.map
+//# sourceMappingURL=llm-requirement-fulfillment.js.map

package/dist/evaluators/llm-requirement-fulfillment.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-requirement-fulfillment.js","sourceRoot":"","sources":["../../src/evaluators/llm-requirement-fulfillment.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAKH,MAAM,+BAA+B,GAAoC;IACvE,aAAa,EAAE,GAAsB,EAAE,CAAC,CAAC;QACvC,UAAU,EAAE,EAAE;QACd,MAAM,EAAE,EAAE;QACV,IAAI,EAAE,EAAE;KACT,CAAC;IAEF,SAAS,EAAE;QACT,uEAAuE;KACxE;IAED,YAAY,EAAE;QACZ,6CAA6C;QAC7C;YACE,IAAI,EAAE,YAAY;YAClB,KAAK,EAAE,qBAAqB;YAC5B,KAAK,EAAE;gBACL,qFAAqF;gBACrF,gFAAgF;gBAChF,uFAAuF;gBACvF,gFAAgF;aACjF;SACF;QACD;YACE,IAAI,EAAE,YAAY;YAClB,KAAK,EAAE,SAAS;YAChB,KAAK,EAAE;gBACL,2CAA2C;gBAC3C,iDAAiD;gBACjD,0CAA0C;aAC3C;SACF;KACF;IAED,MAAM,EAAE;QACN;YACE,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE;gBACP,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,KAAK,EAAE;wBACL,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,sBAAsB;qBACpC;oBACD,SAAS,EAAE;wBACT,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,uBAAuB;qBACrC;oBACD,OAAO,EAAE;wBACP,IAAI,EAAE,QAAQ;wBACd,UAAU,EAAE;4BACV,sBAAsB,EAAE;gCACtB,IAAI,EAAE,QAAQ;gCACd,UAAU,EAAE;oCACV,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;oCACzB,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;iCAC9B;6BACF;4BACD,oBAAoB,EAAE;gCACpB,IAAI,EAAE,QAAQ;gCACd,UAAU,EAAE;oCACV,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;oCACzB,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;iCAC9B;6BACF;4BACD,qBAAqB,EAAE;gCACrB,IAAI,EAAE,QAAQ;gCACd,UAAU,EAAE;oCACV,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;oCACzB,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;iCAC9B;6BACF;4BACD,kBAAkB,EAAE;gCAClB,IAAI,EAAE,QAAQ;gCACd,UAAU,EAAE;oCACV,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;oCACzB,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;iCAC9B;6BACF;yBACF;qBACF;iBACF;gBACD,QAAQ,EAAE,CAAC,OAAO,EAAE,WAAW,EAAE,SAAS,CAAC;aAC5C;SACF;KACF;CACF,CAAC;AAEF,eAAe;IACb,IAAI,EAAE,6BAA6B;IACnC,WAAW,EAAE,8LAA8L;IAC3M,MAAM,EAAE,+BAA+B;CACd,CAAC"}

package/dist/{src/evaluators/json-validator.d.ts → evaluators/structured-output-presence.d.ts}

@@ -1,7 +1,7 @@
 /**
- * JSON Validator Evaluator
+ * Structured Output Presence Evaluator
  *
- * Validates JSON structure in structured output
+ * Checks if structuredOutput exists and is a valid object
  */
 import type { EvaluationContext, EvaluationResult } from '../types.js';
 declare const _default: {
@@ -10,4 +10,4 @@ declare const _default: {
     evaluate(context: EvaluationContext): Promise<EvaluationResult>;
 };
 export default _default;
-//# sourceMappingURL=json-validator.d.ts.map
+//# sourceMappingURL=structured-output-presence.d.ts.map

package/dist/evaluators/structured-output-presence.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"structured-output-presence.d.ts","sourceRoot":"","sources":["../../src/evaluators/structured-output-presence.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAiB,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;;;;sBAM5D,iBAAiB,GAAG,OAAO,CAAC,gBAAgB,CAAC;;AAJvE,wBAiD0B"}

package/dist/{src/evaluators/json-validator.js → evaluators/structured-output-presence.js}

@@ -1,11 +1,11 @@
 /**
- * JSON Validator Evaluator
+ * Structured Output Presence Evaluator
  *
- * Validates JSON structure in structured output
+ * Checks if structuredOutput exists and is a valid object
  */
 export default {
-    name: 'JSON Validator',
-    description: 'Validates JSON structure in output',
+    name: 'Structured Output Presence',
+    description: 'Measures structured output presence rate (percentage of runs with valid structuredOutput). Checks if structuredOutput exists and is an object type for each run.',
     async evaluate(context) {
         const errors = [];
         let validCount = 0;
@@ -33,7 +33,7 @@ export default {
             ? (validCount / context.runs.length) * 10
             : 0;
         return {
-            evaluator: 'json-validator',
+            evaluator: 'structured-output-presence',
             moduleName: context.moduleName,
             score,
             reasoning: errors.length > 0
@@ -48,4 +48,4 @@ export default {
         };
     },
 };
-//# sourceMappingURL=json-validator.js.map
+//# sourceMappingURL=structured-output-presence.js.map

package/dist/evaluators/structured-output-presence.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"structured-output-presence.js","sourceRoot":"","sources":["../../src/evaluators/structured-output-presence.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAIH,eAAe;IACb,IAAI,EAAE,4BAA4B;IAClC,WAAW,EAAE,kKAAkK;IAE/K,KAAK,CAAC,QAAQ,CAAC,OAA0B;QACvC,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,MAAM,UAAU,GAA2D,EAAE,CAAC;QAE9E,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7C,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC5B,MAAM,EAAE,gBAAgB,EAAE,GAAG,GAAG,CAAC,WAAW,CAAC;YAE7C,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACtB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;gBAClD,UAAU,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,sBAAsB,EAAE,CAAC,CAAC;gBAC7E,SAAS;YACX,CAAC;YAED,wBAAwB;YACxB,IAAI,OAAO,gBAAgB,KAAK,QAAQ,IAAI,gBAAgB,KAAK,IAAI,EAAE,CAAC;gBACtE,UAAU,EAAE,CAAC;gBACb,UAAU,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAC/C,CAAC;iBAAM,CAAC;gBACN,MAAM,KAAK,GAAG,wBAAwB,CAAC;gBACvC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,EAAE,CAAC,CAAC;gBACtC,UAAU,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;YACvD,CAAC;QACH,CAAC;QAED,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC;YACnC,CAAC,CAAC,CAAC,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE;YACzC,CAAC,CAAC,CAAC,CAAC;QAEN,OAAO;YACL,SAAS,EAAE,4BAA4B;YACvC,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,KAAK;YACL,SAAS,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC;gBAC1B,CAAC,CAAC,GAAG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,2BAA2B,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBACpF,CAAC,CAAC,OAAO,UAAU,oCAAoC;YACzD,OAAO,EAAE;gBACP,UAAU;gBACV,UAAU,EAAE,OAAO,CAAC,IAAI,CAAC,MAAM;gBAC/B,MAAM;gBACN,IAAI,EAAE,UAAU;aACjB;SACF,CAAC;IACJ,CAAC;CACsB,CAAC"}

package/dist/{src/index.d.ts → index.d.ts}

@@ -6,7 +6,7 @@
 export * from './types.js';
 export { loadExperimentConfig } from './config/loader.js';
 export { loadModules, loadEvaluators } from './config/dynamic-loader.js';
-export { baseEvaluationModule } from './evaluators/base-module.js';
+export { baseEvaluationModule } from './base-evaluation-module.js';
 export { DriverManager } from './runner/driver-manager.js';
 export { ExperimentRunner } from './runner/experiment.js';
 export { EvaluatorRunner } from './runner/evaluator.js';

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,cAAc,YAAY,CAAC;AAG3B,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAGzE,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AAGnE,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAGxD,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC"}

package/dist/{src/index.js → index.js}

@@ -9,7 +9,7 @@ export * from './types.js';
 export { loadExperimentConfig } from './config/loader.js';
 export { loadModules, loadEvaluators } from './config/dynamic-loader.js';
 // Evaluators
-export { baseEvaluationModule } from './evaluators/base-module.js';
+export { baseEvaluationModule } from './base-evaluation-module.js';
 // Runners
 export { DriverManager } from './runner/driver-manager.js';
 export { ExperimentRunner } from './runner/experiment.js';

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,QAAQ;AACR,cAAc,YAAY,CAAC;AAE3B,wBAAwB;AACxB,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAEzE,aAAa;AACb,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AAEnE,UAAU;AACV,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAExD,YAAY;AACZ,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC"}

package/dist/logger.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Experiment package logger
+ */
+import { Logger } from '@modular-prompt/utils';
+/**
+ * Experiment package logger with 'experiment' prefix
+ */
+export declare const logger: Logger;
+//# sourceMappingURL=logger.d.ts.map

package/dist/logger.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../src/logger.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAE/C;;GAEG;AACH,eAAO,MAAM,MAAM,QAAwD,CAAC"}

package/dist/logger.js

@@ -0,0 +1,9 @@
+/**
+ * Experiment package logger
+ */
+import { Logger } from '@modular-prompt/utils';
+/**
+ * Experiment package logger with 'experiment' prefix
+ */
+export const logger = new Logger({ prefix: 'experiment', context: 'main' });
+//# sourceMappingURL=logger.js.map

package/dist/logger.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"logger.js","sourceRoot":"","sources":["../src/logger.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAE/C;;GAEG;AACH,MAAM,CAAC,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC"}

package/dist/reporter/statistics.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"statistics.d.ts","sourceRoot":"","sources":["../../src/reporter/statistics.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAa,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzD,qBAAa,kBAAkB;IACjB,OAAO,CAAC,OAAO;gBAAP,OAAO,EAAE,UAAU,EAAE;IAEzC;;OAEG;IACH,MAAM,IAAI,IAAI;IA2Bd;;OAEG;IACH,OAAO,CAAC,YAAY;IASpB;;OAEG;IACH,OAAO,CAAC,iBAAiB;CAwB1B"}

package/dist/reporter/statistics.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"statistics.js","sourceRoot":"","sources":["../../src/reporter/statistics.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,MAAM,OAAO,kBAAkB;IACT;IAApB,YAAoB,OAAqB;QAArB,YAAO,GAAP,OAAO,CAAc;IAAG,CAAC;IAE7C;;OAEG;IACH,MAAM;QACJ,OAAO,CAAC,GAAG,EAAE,CAAC;QACd,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QAC5B,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;QACrC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QAC5B,OAAO,CAAC,GAAG,EAAE,CAAC;QAEd,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAClC,OAAO,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,QAAQ,MAAM,MAAM,CAAC,KAAK,OAAO,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;YACvF,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;YAE5B,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YACvD,MAAM,WAAW,GAAG,CAAC,WAAW,CAAC,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC;YAEpE,OAAO,CAAC,GAAG,CAAC,iBAAiB,WAAW,CAAC,MAAM,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAEtG,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC3B,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC;gBAC/B,IAAI,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAC;YACtC,CAAC;YAED,OAAO,CAAC,GAAG,EAAE,CAAC;QAChB,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC9B,CAAC;IAED;;OAEG;IACK,YAAY,CAAC,IAAiB;QACpC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QACvC,MAAM,GAAG,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;QAC5D,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC;QAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC;QAE/B,OAAO,CAAC,GAAG,CAAC,uBAAuB,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,GAAG,WAAW,GAAG,IAAI,CAAC,CAAC;IACrF,CAAC;IAED;;OAEG;IACK,iBAAiB,CAAC,IAAiB;QACzC,2BAA2B;QAC3B,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;YAC/B,MAAM,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC;YAC7D,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QACxC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;QAE3B,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7B,OAAO;QACT,CAAC;QAED,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,CAAC;QAC3C,OAAO,CAAC,GAAG,CAAC,uBAAuB,aAAa,CAAC,IAAI,0BAA0B,WAAW,CAAC,MAAM,SAAS,CAAC,CAAC;QAE5G,IAAI,aAAa,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YAC7B,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC;QAC7C,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;YACjC,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE;gBAChD,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;gBAC3D,OAAO,CAAC,GAAG,CAAC,cAAc,GAAG,GAAG,CAAC,KAAK,KAAK,OAAO,MAAM,EAAE,CAAC,CAAC;YAC9D,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;CACF"}

package/dist/{src/run-comparison.d.ts → run-comparison.d.ts}

@@ -17,6 +17,7 @@
  *   --repeat <count>     Number of repetitions (default: 1)
  *   --evaluate           Enable evaluation phase
  *   --evaluators <names> Comma-separated evaluator names (default: all)
+ *   --dry-run            Display execution plan without running the experiment
  */
 export {};
 //# sourceMappingURL=run-comparison.d.ts.map

package/dist/run-comparison.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-comparison.d.ts","sourceRoot":"","sources":["../src/run-comparison.ts"],"names":[],"mappings":";AACA;;;;;;;;;;;;;;;;;;;GAmBG"}