@modular-prompt/experiment 0.1.1

package/dist/src/config/loader.js

@@ -0,0 +1,125 @@
+/**
+ * Configuration loader
+ */
+import { readFileSync } from 'fs';
+import { parse as parseYaml } from 'yaml';
+import { resolve, dirname, extname } from 'path';
+import { createJiti } from 'jiti';
+import { AIService } from '@modular-prompt/driver';
+/**
+ * Resolve path relative to config file directory
+ *
+ * @param configDir - Config file directory
+ * @param path - Path to resolve (can be relative or absolute)
+ * @returns Resolved absolute path
+ */
+function resolveConfigPath(configDir, path) {
+    // If path starts with ~, resolve it as home directory
+    if (path.startsWith('~')) {
+        return path.replace('~', process.env.HOME || '~');
+    }
+    // If path is absolute, return as is
+    if (path.startsWith('/')) {
+        return path;
+    }
+    // Otherwise, resolve relative to config directory
+    return resolve(configDir, path);
+}
+/**
+ * Load experiment configuration
+ *
+ * @param configPath - Path to config file (YAML or TypeScript)
+ * @returns LoadedConfig with all configuration
+ */
+export async function loadExperimentConfig(configPath) {
+    // Get config directory and file extension
+    const configDir = dirname(configPath);
+    const ext = extname(configPath);
+    // Load config based on file type
+    let config;
+    if (ext === '.yaml' || ext === '.yml') {
+        // YAML format
+        const content = readFileSync(configPath, 'utf-8');
+        config = parseYaml(content);
+    }
+    else if (ext === '.ts' || ext === '.js' || ext === '.mjs' || ext === '.cjs') {
+        // TypeScript/JavaScript format (auto-transpile with jiti)
+        const jiti = createJiti(import.meta.url, {
+            interopDefault: true, // Automatically get default export
+            cache: true, // Enable caching for better performance
+            requireCache: false, // Don't use require cache
+        });
+        config = await jiti.import(configPath);
+        if (!config) {
+            throw new Error(`❌ No default export in ${configPath}`);
+        }
+    }
+    else {
+        throw new Error(`❌ Unsupported config file format: ${ext}. Use .yaml, .yml, .ts, .js, .mjs, or .cjs`);
+    }
+    // Extract components
+    const modules = config.modules || [];
+    const testCases = config.testCases || [];
+    const evaluators = config.evaluators || [];
+    // Server config (models, drivers, evaluation, etc.)
+    const serverConfig = {
+        models: config.models,
+        drivers: config.drivers,
+        evaluation: config.evaluation,
+        credentials: config.credentials,
+        selection: config.selection,
+        server: config.server,
+        logging: config.logging,
+    };
+    // Resolve paths in driver configurations relative to config file
+    if (serverConfig.drivers) {
+        for (const driverName in serverConfig.drivers) {
+            const driverConfig = serverConfig.drivers[driverName];
+            // Resolve credentialsPath for vertexai driver
+            if (driverConfig.credentialsPath) {
+                driverConfig.credentialsPath = resolveConfigPath(configDir, driverConfig.credentialsPath);
+            }
+        }
+    }
+    // Setup driver environment variables if specified
+    if (serverConfig.credentials?.googleApplicationCredentials) {
+        const resolvedPath = resolveConfigPath(configDir, serverConfig.credentials.googleApplicationCredentials);
+        process.env.GOOGLE_APPLICATION_CREDENTIALS = resolvedPath;
+        console.log(`Setting GOOGLE_APPLICATION_CREDENTIALS=${resolvedPath}`);
+    }
+    // Validation
+    if (!serverConfig.models || Object.keys(serverConfig.models).length === 0) {
+        throw new Error('❌ No models configured in config file');
+    }
+    // Get model names from object keys
+    const modelNames = new Set(Object.keys(serverConfig.models));
+    // Validate testCase model references
+    for (const testCase of testCases) {
+        if (testCase.models) {
+            for (const modelName of testCase.models) {
+                if (!modelNames.has(modelName)) {
+                    throw new Error(`❌ TestCase '${testCase.name}' references unknown model '${modelName}'`);
+                }
+            }
+        }
+    }
+    // Initialize AIService
+    const aiServiceConfig = {
+        models: serverConfig.models,
+        drivers: serverConfig.drivers || {},
+        defaultOptions: {
+            temperature: 0.7,
+            maxTokens: 2048,
+        },
+    };
+    const aiService = new AIService(aiServiceConfig);
+    return {
+        serverConfig,
+        modules,
+        testCases,
+        evaluators,
+        aiService,
+        configDir
+    };
+}
+//# sourceMappingURL=loader.js.map

package/dist/src/config/loader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"loader.js","sourceRoot":"","sources":["../../../src/config/loader.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AAClC,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAC1C,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AACjD,OAAO,EAAE,UAAU,EAAE,MAAM,MAAM,CAAC;AAClC,OAAO,EAAE,SAAS,EAA0B,MAAM,wBAAwB,CAAC;AAgB3E;;;;;;GAMG;AACH,SAAS,iBAAiB,CAAC,SAAiB,EAAE,IAAY;IACxD,sDAAsD;IACtD,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACzB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC;IACpD,CAAC;IACD,oCAAoC;IACpC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACzB,OAAO,IAAI,CAAC;IACd,CAAC;IACD,kDAAkD;IAClD,OAAO,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;AAClC,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,UAAkB;IAC3D,0CAA0C;IAC1C,MAAM,SAAS,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IACtC,MAAM,GAAG,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IAEhC,iCAAiC;IACjC,IAAI,MAAW,CAAC;IAEhB,IAAI,GAAG,KAAK,OAAO,IAAI,GAAG,KAAK,MAAM,EAAE,CAAC;QACtC,cAAc;QACd,MAAM,OAAO,GAAG,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QAClD,MAAM,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;IAC9B,CAAC;SAAM,IAAI,GAAG,KAAK,KAAK,IAAI,GAAG,KAAK,KAAK,IAAI,GAAG,KAAK,MAAM,IAAI,GAAG,KAAK,MAAM,EAAE,CAAC;QAC9E,0DAA0D;QAC1D,MAAM,IAAI,GAAG,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE;YACvC,cAAc,EAAE,IAAI,EAAG,mCAAmC;YAC1D,KAAK,EAAE,IAAI,EAAY,wCAAwC;YAC/D,YAAY,EAAE,KAAK,EAAI,0BAA0B;SAClD,CAAC,CAAC;QAEH,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAEvC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,0BAA0B,UAAU,EAAE,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CAAC,qCAAqC,GAAG,4CAA4C,CAAC,CAAC;IACxG,CAAC;IAED,qBAAqB;IACrB,MAAM,OAAO,GAAsB,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC;IACxD,MAAM,SAAS,GAAe,MAAM,CAAC,SAAS,IAAI,EAAE,CAAC;IACrD,MAAM,UAAU,GAAyB,MAAM,CAAC,UAAU,IAAI,EAAE,CAAC;IAEjE,oDAAoD;IACpD,MAAM,YAAY,GAAG;QACnB,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,OAAO,EAAE,MAAM,CAAC,OAAO;KACxB,CAAC;IAEF,iEAAiE;IACjE,IAAI,YAAY,CAAC,OAAO,EAAE,CAAC;QACzB,KAAK,MAAM,UAAU,IAAI,YAAY,CAAC,OAAO,EAAE,CAAC;YAC9C,MAAM,YAAY,GAAG,YAAY,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;YAEtD,8CAA8C;YAC9C,IAAI,YAAY,CAAC,eAAe,EAAE,CAAC;gBACjC,YAAY,CAAC,eAAe,GAAG,iBAAiB,CAAC,SAAS,EAAE,YAAY,CAAC,eAAe,CAAC,CAAC;YAC5F,CAAC;QACH,CAAC;IACH,CAAC;IAED,kDAAkD;IAClD,IAAI,YAAY,CAAC,WAAW,EAAE,4BAA4B,EAAE,CAAC;QAC3D,MAAM,YAAY,GAAG,iBAAiB,CAAC,SAAS,EAAE,YAAY,CAAC,WAAW,CAAC,4BAA4B,CAAC,CAAC;QACzG,OAAO,CAAC,GAAG,CAAC,8BAA8B,GAAG,YAAY,CAAC;QAC1D,OAAO,CAAC,GAAG,CAAC,0CAA0C,YAAY,EAAE,CAAC,CAAC;IACxE,CAAC;IAED,aAAa;IACb,IAAI,CAAC,YAAY,CAAC,MAAM,IAAI,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1E,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IAC3D,CAAC;IAED,mCAAmC;IACnC,MAAM,UAAU,GAAG,IAAI,GAAG,CAAS,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC;IAErE,qCAAqC;IACrC,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;YACpB,KAAK,MAAM,SAAS,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;gBACxC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;oBAC/B,MAAM,IAAI,KAAK,CAAC,eAAe,QAAQ,CAAC,IAAI,+BAA+B,SAAS,GAAG,CAAC,CAAC;gBAC3F,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,uBAAuB;IACvB,MAAM,eAAe,GAAsB;QACzC,MAAM,EAAE,YAAY,CAAC,MAAM;QAC3B,OAAO,EAAE,YAAY,CAAC,OAAO,IAAI,EAAE;QACnC,cAAc,EAAE;YACd,WAAW,EAAE,GAAG;YAChB,SAAS,EAAE,IAAI;SAChB;KACF,CAAC;IAEF,MAAM,SAAS,GAAG,IAAI,SAAS,CAAC,eAAe,CAAC,CAAC;IAEjD,OAAO;QACL,YAAY;QACZ,OAAO;QACP,SAAS;QACT,UAAU;QACV,SAAS;QACT,SAAS;KACV,CAAC;AACJ,CAAC"}

package/dist/src/evaluators/base-module.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Base evaluation prompt module
+ *
+ * This module provides the foundation for all evaluation prompts.
+ * It defines how test data is presented to the evaluator.
+ */
+import type { PromptModule } from '@modular-prompt/core';
+import type { EvaluationContext } from '../types.js';
+export declare const baseEvaluationModule: PromptModule<EvaluationContext>;
+//# sourceMappingURL=base-module.d.ts.map

package/dist/src/evaluators/base-module.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"base-module.d.ts","sourceRoot":"","sources":["../../../src/evaluators/base-module.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,YAAY,EAA4B,MAAM,sBAAsB,CAAC;AACnF,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAErD,eAAO,MAAM,oBAAoB,EAAE,YAAY,CAAC,iBAAiB,CAuGhE,CAAC"}

package/dist/src/evaluators/base-module.js

@@ -0,0 +1,103 @@
+/**
+ * Base evaluation prompt module
+ *
+ * This module provides the foundation for all evaluation prompts.
+ * It defines how test data is presented to the evaluator.
+ */
+export const baseEvaluationModule = {
+    createContext: () => ({
+        moduleName: '',
+        prompt: '',
+        runs: [],
+    }),
+    objective: [
+        '- Evaluate the output of a prompt module',
+        '- Provide detailed assessment with scores and reasoning',
+    ],
+    terms: [
+        '- Module: A prompt variation being tested',
+        '- Prompt: The compiled prompt used to generate the output',
+        '- Query Result: The output generated by the LLM',
+        '- Run: A single execution of the prompt',
+    ],
+    instructions: [
+        {
+            type: 'subsection',
+            title: 'Output Format',
+            items: [
+                'Return evaluation in JSON format with the following structure:',
+                '- score: Overall score (0-10)',
+                '- reasoning: Clear explanation of the score',
+                '- details: Object with additional evaluation metrics',
+            ],
+        },
+    ],
+    materials: [
+        {
+            type: 'subsection',
+            title: 'Module Name',
+            items: [
+                (ctx) => ctx.moduleName,
+            ],
+        },
+        {
+            type: 'subsection',
+            title: 'Prompt Used',
+            items: [
+                (ctx) => ({
+                    type: 'text',
+                    content: ctx.prompt,
+                }),
+            ],
+        },
+    ],
+    inputs: [
+        (ctx) => ctx.runs.flatMap((run, idx) => {
+            const result = run.queryResult;
+            const elements = [];
+            // Run number
+            elements.push({
+                type: 'text',
+                content: `Run ${idx + 1}`,
+            });
+            // Output (prefer structuredOutput over content)
+            if (result.structuredOutput) {
+                elements.push({
+                    type: 'json',
+                    content: result.structuredOutput,
+                });
+            }
+            else {
+                elements.push({
+                    type: 'text',
+                    content: result.content,
+                });
+            }
+            return elements;
+        }),
+    ],
+    schema: [
+        {
+            type: 'json',
+            content: {
+                type: 'object',
+                properties: {
+                    score: {
+                        type: 'number',
+                        description: 'Overall score (0-10)',
+                    },
+                    reasoning: {
+                        type: 'string',
+                        description: 'Explanation of the score',
+                    },
+                    details: {
+                        type: 'object',
+                        description: 'Additional evaluation metrics',
+                    },
+                },
+                required: ['score', 'reasoning'],
+            },
+        },
+    ],
+};
+//# sourceMappingURL=base-module.js.map

package/dist/src/evaluators/base-module.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"base-module.js","sourceRoot":"","sources":["../../../src/evaluators/base-module.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,MAAM,CAAC,MAAM,oBAAoB,GAAoC;IACnE,aAAa,EAAE,GAAsB,EAAE,CAAC,CAAC;QACvC,UAAU,EAAE,EAAE;QACd,MAAM,EAAE,EAAE;QACV,IAAI,EAAE,EAAE;KACT,CAAC;IAEF,SAAS,EAAE;QACT,0CAA0C;QAC1C,yDAAyD;KAC1D;IAED,KAAK,EAAE;QACL,2CAA2C;QAC3C,2DAA2D;QAC3D,iDAAiD;QACjD,yCAAyC;KAC1C;IAED,YAAY,EAAE;QACZ;YACE,IAAI,EAAE,YAAY;YAClB,KAAK,EAAE,eAAe;YACtB,KAAK,EAAE;gBACL,gEAAgE;gBAChE,+BAA+B;gBAC/B,6CAA6C;gBAC7C,sDAAsD;aACvD;SACF;KACF;IAED,SAAS,EAAE;QACT;YACE,IAAI,EAAE,YAAY;YAClB,KAAK,EAAE,aAAa;YACpB,KAAK,EAAE;gBACL,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU;aACxB;SACF;QACD;YACE,IAAI,EAAE,YAAY;YAClB,KAAK,EAAE,aAAa;YACpB,KAAK,EAAE;gBACL,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;oBACR,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE,GAAG,CAAC,MAAM;iBACJ,CAAA;aAClB;SACF;KACF;IAED,MAAM,EAAE;QACN,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;YACrC,MAAM,MAAM,GAAG,GAAG,CAAC,WAAW,CAAC;YAC/B,MAAM,QAAQ,GAAqC,EAAE,CAAC;YAEtD,aAAa;YACb,QAAQ,CAAC,IAAI,CAAC;gBACZ,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE,OAAO,GAAG,GAAG,CAAC,EAAE;aAC1B,CAAC,CAAC;YAEH,gDAAgD;YAChD,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,QAAQ,CAAC,IAAI,CAAC;oBACZ,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE,MAAM,CAAC,gBAAgB;iBACjC,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,QAAQ,CAAC,IAAI,CAAC;oBACZ,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE,MAAM,CAAC,OAAO;iBACxB,CAAC,CAAC;YACL,CAAC;YAED,OAAO,QAAQ,CAAC;QAClB,CAAC,CAAC;KACH;IAED,MAAM,EAAE;QACN;YACE,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE;gBACP,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,KAAK,EAAE;wBACL,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,sBAAsB;qBACpC;oBACD,SAAS,EAAE;wBACT,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,0BAA0B;qBACxC;oBACD,OAAO,EAAE;wBACP,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,+BAA+B;qBAC7C;iBACF;gBACD,QAAQ,EAAE,CAAC,OAAO,EAAE,WAAW,CAAC;aACjC;SACF;KACF;CACF,CAAC"}

package/dist/src/evaluators/functional-correctness.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Functional Correctness Evaluator
+ *
+ * Evaluates whether the output meets the functional requirements
+ */
+import type { PromptModule } from '@modular-prompt/core';
+import type { EvaluationContext } from '../types.js';
+declare const _default: {
+    name: string;
+    description: string;
+    module: PromptModule<EvaluationContext>;
+};
+export default _default;
+//# sourceMappingURL=functional-correctness.d.ts.map

package/dist/src/evaluators/functional-correctness.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"functional-correctness.d.ts","sourceRoot":"","sources":["../../../src/evaluators/functional-correctness.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,KAAK,EAAmB,iBAAiB,EAAE,MAAM,aAAa,CAAC;;;;;;AA0FtE,wBAI4B"}

package/dist/src/evaluators/functional-correctness.js

@@ -0,0 +1,95 @@
+/**
+ * Functional Correctness Evaluator
+ *
+ * Evaluates whether the output meets the functional requirements
+ */
+const functionalCorrectnessModule = {
+    createContext: () => ({
+        moduleName: '',
+        prompt: '',
+        runs: [],
+    }),
+    objective: [
+        '- Assess whether the output correctly fulfills the given requirements',
+    ],
+    instructions: [
+        '- Evaluate based on the following criteria:',
+        {
+            type: 'subsection',
+            title: 'Evaluation Criteria',
+            items: [
+                '1. **Requirement Fulfillment**: Does it satisfy the intent described in the prompt?',
+                '2. **Parameter Correctness**: Are all required parameters present and correct?',
+                '3. **Parameter Completeness**: Are optional parameters appropriately used or omitted?',
+                '4. **Logical Consistency**: Is the output logically consistent with the facts?',
+            ],
+        },
+        {
+            type: 'subsection',
+            title: 'Scoring',
+            items: [
+                '- Assign scores (0-10) for each criterion',
+                '- Calculate overall score based on all criteria',
+                '- Provide clear reasoning for each score',
+            ],
+        },
+    ],
+    schema: [
+        {
+            type: 'json',
+            content: {
+                type: 'object',
+                properties: {
+                    score: {
+                        type: 'number',
+                        description: 'Overall score (0-10)',
+                    },
+                    reasoning: {
+                        type: 'string',
+                        description: 'Summary of evaluation',
+                    },
+                    details: {
+                        type: 'object',
+                        properties: {
+                            requirementFulfillment: {
+                                type: 'object',
+                                properties: {
+                                    score: { type: 'number' },
+                                    reasoning: { type: 'string' },
+                                },
+                            },
+                            parameterCorrectness: {
+                                type: 'object',
+                                properties: {
+                                    score: { type: 'number' },
+                                    reasoning: { type: 'string' },
+                                },
+                            },
+                            parameterCompleteness: {
+                                type: 'object',
+                                properties: {
+                                    score: { type: 'number' },
+                                    reasoning: { type: 'string' },
+                                },
+                            },
+                            logicalConsistency: {
+                                type: 'object',
+                                properties: {
+                                    score: { type: 'number' },
+                                    reasoning: { type: 'string' },
+                                },
+                            },
+                        },
+                    },
+                },
+                required: ['score', 'reasoning', 'details'],
+            },
+        },
+    ],
+};
+export default {
+    name: 'Functional Correctness',
+    description: 'Evaluates whether the output meets the functional requirements',
+    module: functionalCorrectnessModule,
+};
+//# sourceMappingURL=functional-correctness.js.map

package/dist/src/evaluators/functional-correctness.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"functional-correctness.js","sourceRoot":"","sources":["../../../src/evaluators/functional-correctness.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAKH,MAAM,2BAA2B,GAAoC;IACnE,aAAa,EAAE,GAAsB,EAAE,CAAC,CAAC;QACvC,UAAU,EAAE,EAAE;QACd,MAAM,EAAE,EAAE;QACV,IAAI,EAAE,EAAE;KACT,CAAC;IAEF,SAAS,EAAE;QACT,uEAAuE;KACxE;IAED,YAAY,EAAE;QACZ,6CAA6C;QAC7C;YACE,IAAI,EAAE,YAAY;YAClB,KAAK,EAAE,qBAAqB;YAC5B,KAAK,EAAE;gBACL,qFAAqF;gBACrF,gFAAgF;gBAChF,uFAAuF;gBACvF,gFAAgF;aACjF;SACF;QACD;YACE,IAAI,EAAE,YAAY;YAClB,KAAK,EAAE,SAAS;YAChB,KAAK,EAAE;gBACL,2CAA2C;gBAC3C,iDAAiD;gBACjD,0CAA0C;aAC3C;SACF;KACF;IAED,MAAM,EAAE;QACN;YACE,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE;gBACP,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,KAAK,EAAE;wBACL,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,sBAAsB;qBACpC;oBACD,SAAS,EAAE;wBACT,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,uBAAuB;qBACrC;oBACD,OAAO,EAAE;wBACP,IAAI,EAAE,QAAQ;wBACd,UAAU,EAAE;4BACV,sBAAsB,EAAE;gCACtB,IAAI,EAAE,QAAQ;gCACd,UAAU,EAAE;oCACV,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;oCACzB,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;iCAC9B;6BACF;4BACD,oBAAoB,EAAE;gCACpB,IAAI,EAAE,QAAQ;gCACd,UAAU,EAAE;oCACV,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;oCACzB,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;iCAC9B;6BACF;4BACD,qBAAqB,EAAE;gCACrB,IAAI,EAAE,QAAQ;gCACd,UAAU,EAAE;oCACV,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;oCACzB,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;iCAC9B;6BACF;4BACD,kBAAkB,EAAE;gCAClB,IAAI,EAAE,QAAQ;gCACd,UAAU,EAAE;oCACV,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;oCACzB,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;iCAC9B;6BACF;yBACF;qBACF;iBACF;gBACD,QAAQ,EAAE,CAAC,OAAO,EAAE,WAAW,EAAE,SAAS,CAAC;aAC5C;SACF;KACF;CACF,CAAC;AAEF,eAAe;IACb,IAAI,EAAE,wBAAwB;IAC9B,WAAW,EAAE,gEAAgE;IAC7E,MAAM,EAAE,2BAA2B;CACV,CAAC"}

package/dist/src/evaluators/json-validator.d.ts

@@ -0,0 +1,13 @@
+/**
+ * JSON Validator Evaluator
+ *
+ * Validates JSON structure in structured output
+ */
+import type { EvaluationContext, EvaluationResult } from '../types.js';
+declare const _default: {
+    name: string;
+    description: string;
+    evaluate(context: EvaluationContext): Promise<EvaluationResult>;
+};
+export default _default;
+//# sourceMappingURL=json-validator.d.ts.map

package/dist/src/evaluators/json-validator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"json-validator.d.ts","sourceRoot":"","sources":["../../../src/evaluators/json-validator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAiB,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;;;;sBAM5D,iBAAiB,GAAG,OAAO,CAAC,gBAAgB,CAAC;;AAJvE,wBAiD0B"}

package/dist/src/evaluators/json-validator.js

@@ -0,0 +1,51 @@
+/**
+ * JSON Validator Evaluator
+ *
+ * Validates JSON structure in structured output
+ */
+export default {
+    name: 'JSON Validator',
+    description: 'Validates JSON structure in output',
+    async evaluate(context) {
+        const errors = [];
+        let validCount = 0;
+        const runDetails = [];
+        for (let i = 0; i < context.runs.length; i++) {
+            const run = context.runs[i];
+            const { structuredOutput } = run.queryResult;
+            if (!structuredOutput) {
+                errors.push(`Run ${i + 1}: No structured output`);
+                runDetails.push({ run: i + 1, valid: false, error: 'No structured output' });
+                continue;
+            }
+            // Basic JSON validation
+            if (typeof structuredOutput === 'object' && structuredOutput !== null) {
+                validCount++;
+                runDetails.push({ run: i + 1, valid: true });
+            }
+            else {
+                const error = 'Invalid JSON structure';
+                errors.push(`Run ${i + 1}: ${error}`);
+                runDetails.push({ run: i + 1, valid: false, error });
+            }
+        }
+        const score = context.runs.length > 0
+            ? (validCount / context.runs.length) * 10
+            : 0;
+        return {
+            evaluator: 'json-validator',
+            moduleName: context.moduleName,
+            score,
+            reasoning: errors.length > 0
+                ? `${validCount}/${context.runs.length} valid outputs. Issues: ${errors.join('; ')}`
+                : `All ${validCount} outputs have valid JSON structure`,
+            details: {
+                validCount,
+                totalCount: context.runs.length,
+                errors,
+                runs: runDetails,
+            },
+        };
+    },
+};
+//# sourceMappingURL=json-validator.js.map

package/dist/src/evaluators/json-validator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"json-validator.js","sourceRoot":"","sources":["../../../src/evaluators/json-validator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAIH,eAAe;IACb,IAAI,EAAE,gBAAgB;IACtB,WAAW,EAAE,oCAAoC;IAEjD,KAAK,CAAC,QAAQ,CAAC,OAA0B;QACvC,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,MAAM,UAAU,GAA2D,EAAE,CAAC;QAE9E,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7C,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC5B,MAAM,EAAE,gBAAgB,EAAE,GAAG,GAAG,CAAC,WAAW,CAAC;YAE7C,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACtB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;gBAClD,UAAU,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,sBAAsB,EAAE,CAAC,CAAC;gBAC7E,SAAS;YACX,CAAC;YAED,wBAAwB;YACxB,IAAI,OAAO,gBAAgB,KAAK,QAAQ,IAAI,gBAAgB,KAAK,IAAI,EAAE,CAAC;gBACtE,UAAU,EAAE,CAAC;gBACb,UAAU,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAC/C,CAAC;iBAAM,CAAC;gBACN,MAAM,KAAK,GAAG,wBAAwB,CAAC;gBACvC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,EAAE,CAAC,CAAC;gBACtC,UAAU,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;YACvD,CAAC;QACH,CAAC;QAED,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC;YACnC,CAAC,CAAC,CAAC,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE;YACzC,CAAC,CAAC,CAAC,CAAC;QAEN,OAAO;YACL,SAAS,EAAE,gBAAgB;YAC3B,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,KAAK;YACL,SAAS,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC;gBAC1B,CAAC,CAAC,GAAG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,2BAA2B,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBACpF,CAAC,CAAC,OAAO,UAAU,oCAAoC;YACzD,OAAO,EAAE;gBACP,UAAU;gBACV,UAAU,EAAE,OAAO,CAAC,IAAI,CAAC,MAAM;gBAC/B,MAAM;gBACN,IAAI,EAAE,UAAU;aACjB;SACF,CAAC;IACJ,CAAC;CACsB,CAAC"}

package/dist/src/index.d.ts

@@ -0,0 +1,14 @@
+/**
+ * @modular-prompt/experiment
+ *
+ * Experiment framework for comparing and evaluating prompt modules
+ */
+export * from './types.js';
+export { loadExperimentConfig } from './config/loader.js';
+export { loadModules, loadEvaluators } from './config/dynamic-loader.js';
+export { baseEvaluationModule } from './evaluators/base-module.js';
+export { DriverManager } from './runner/driver-manager.js';
+export { ExperimentRunner } from './runner/experiment.js';
+export { EvaluatorRunner } from './runner/evaluator.js';
+export { StatisticsReporter } from './reporter/statistics.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/src/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,cAAc,YAAY,CAAC;AAG3B,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAGzE,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AAGnE,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAGxD,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC"}

package/dist/src/index.js

@@ -0,0 +1,19 @@
+/**
+ * @modular-prompt/experiment
+ *
+ * Experiment framework for comparing and evaluating prompt modules
+ */
+// Types
+export * from './types.js';
+// Configuration loaders
+export { loadExperimentConfig } from './config/loader.js';
+export { loadModules, loadEvaluators } from './config/dynamic-loader.js';
+// Evaluators
+export { baseEvaluationModule } from './evaluators/base-module.js';
+// Runners
+export { DriverManager } from './runner/driver-manager.js';
+export { ExperimentRunner } from './runner/experiment.js';
+export { EvaluatorRunner } from './runner/evaluator.js';
+// Reporters
+export { StatisticsReporter } from './reporter/statistics.js';
+//# sourceMappingURL=index.js.map

package/dist/src/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,QAAQ;AACR,cAAc,YAAY,CAAC;AAE3B,wBAAwB;AACxB,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAEzE,aAAa;AACb,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AAEnE,UAAU;AACV,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAExD,YAAY;AACZ,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC"}

package/dist/src/reporter/statistics.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Statistics reporter
+ */
+import type { TestResult } from '../types.js';
+export declare class StatisticsReporter {
+    private results;
+    constructor(results: TestResult[]);
+    /**
+     * Generate and display statistics report
+     */
+    report(): void;
+    /**
+     * Report timing statistics
+     */
+    private reportTiming;
+    /**
+     * Report output consistency
+     */
+    private reportConsistency;
+}
+//# sourceMappingURL=statistics.d.ts.map

package/dist/src/reporter/statistics.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"statistics.d.ts","sourceRoot":"","sources":["../../../src/reporter/statistics.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAa,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzD,qBAAa,kBAAkB;IACjB,OAAO,CAAC,OAAO;gBAAP,OAAO,EAAE,UAAU,EAAE;IAEzC;;OAEG;IACH,MAAM,IAAI,IAAI;IA2Bd;;OAEG;IACH,OAAO,CAAC,YAAY;IASpB;;OAEG;IACH,OAAO,CAAC,iBAAiB;CAwB1B"}

package/dist/src/reporter/statistics.js

@@ -0,0 +1,68 @@
+/**
+ * Statistics reporter
+ */
+export class StatisticsReporter {
+    results;
+    constructor(results) {
+        this.results = results;
+    }
+    /**
+     * Generate and display statistics report
+     */
+    report() {
+        console.log();
+        console.log('='.repeat(80));
+        console.log('📊 Statistics Summary');
+        console.log('='.repeat(80));
+        console.log();
+        for (const result of this.results) {
+            console.log(`${result.testCase} - ${result.model} - [${result.module.toUpperCase()}]`);
+            console.log('─'.repeat(80));
+            const successRuns = result.runs.filter(r => r.success);
+            const successRate = (successRuns.length / result.runs.length) * 100;
+            console.log(`Success rate: ${successRuns.length}/${result.runs.length} (${successRate.toFixed(1)}%)`);
+            if (successRuns.length > 0) {
+                this.reportTiming(successRuns);
+                this.reportConsistency(successRuns);
+            }
+            console.log();
+        }
+        console.log('='.repeat(80));
+    }
+    /**
+     * Report timing statistics
+     */
+    reportTiming(runs) {
+        const times = runs.map(r => r.elapsed);
+        const avg = times.reduce((a, b) => a + b, 0) / times.length;
+        const min = Math.min(...times);
+        const max = Math.max(...times);
+        console.log(`Execution time: avg=${avg.toFixed(0)}ms, min=${min}ms, max=${max}ms`);
+    }
+    /**
+     * Report output consistency
+     */
+    reportConsistency(runs) {
+        // Extract JSON from output
+        const jsonOutputs = runs.map(r => {
+            const match = r.content.match(/```json\s*\n([\s\S]*?)\n```/);
+            return match ? match[1].trim() : null;
+        }).filter(j => j !== null);
+        if (jsonOutputs.length === 0) {
+            return;
+        }
+        const uniqueOutputs = new Set(jsonOutputs);
+        console.log(`Output consistency: ${uniqueOutputs.size} unique output(s) from ${jsonOutputs.length} run(s)`);
+        if (uniqueOutputs.size === 1) {
+            console.log('✅ All outputs are identical');
+        }
+        else {
+            console.log('⚠️  Outputs vary:');
+            Array.from(uniqueOutputs).forEach((output, idx) => {
+                const count = jsonOutputs.filter(j => j === output).length;
+                console.log(`   Variant ${idx + 1} (${count}x): ${output}`);
+            });
+        }
+    }
+}
+//# sourceMappingURL=statistics.js.map

package/dist/src/reporter/statistics.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"statistics.js","sourceRoot":"","sources":["../../../src/reporter/statistics.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,MAAM,OAAO,kBAAkB;IACT;IAApB,YAAoB,OAAqB;QAArB,YAAO,GAAP,OAAO,CAAc;IAAG,CAAC;IAE7C;;OAEG;IACH,MAAM;QACJ,OAAO,CAAC,GAAG,EAAE,CAAC;QACd,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QAC5B,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;QACrC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QAC5B,OAAO,CAAC,GAAG,EAAE,CAAC;QAEd,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAClC,OAAO,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,QAAQ,MAAM,MAAM,CAAC,KAAK,OAAO,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;YACvF,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;YAE5B,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YACvD,MAAM,WAAW,GAAG,CAAC,WAAW,CAAC,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC;YAEpE,OAAO,CAAC,GAAG,CAAC,iBAAiB,WAAW,CAAC,MAAM,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAEtG,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC3B,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC;gBAC/B,IAAI,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAC;YACtC,CAAC;YAED,OAAO,CAAC,GAAG,EAAE,CAAC;QAChB,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC9B,CAAC;IAED;;OAEG;IACK,YAAY,CAAC,IAAiB;QACpC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QACvC,MAAM,GAAG,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;QAC5D,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC;QAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC;QAE/B,OAAO,CAAC,GAAG,CAAC,uBAAuB,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,GAAG,WAAW,GAAG,IAAI,CAAC,CAAC;IACrF,CAAC;IAED;;OAEG;IACK,iBAAiB,CAAC,IAAiB;QACzC,2BAA2B;QAC3B,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;YAC/B,MAAM,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC;YAC7D,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QACxC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;QAE3B,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7B,OAAO;QACT,CAAC;QAED,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,CAAC;QAC3C,OAAO,CAAC,GAAG,CAAC,uBAAuB,aAAa,CAAC,IAAI,0BAA0B,WAAW,CAAC,MAAM,SAAS,CAAC,CAAC;QAE5G,IAAI,aAAa,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YAC7B,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC;QAC7C,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;YACjC,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE;gBAChD,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;gBAC3D,OAAO,CAAC,GAAG,CAAC,cAAc,GAAG,GAAG,CAAC,KAAK,KAAK,OAAO,MAAM,EAAE,CAAC,CAAC;YAC9D,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;CACF"}

package/dist/src/run-comparison.d.ts

@@ -0,0 +1,22 @@
+#!/usr/bin/env node
+/**
+ * Module Comparison Experiment
+ *
+ * Compares the performance and output quality of multiple prompt modules.
+ *
+ * Usage:
+ *   moduler-experiment <config> [options]
+ *
+ * Arguments:
+ *   <config>             Config file path (YAML, TypeScript, or JavaScript)
+ *
+ * Options:
+ *   --test-case <name>   Test case name filter
+ *   --model <provider>   Model provider filter (mlx, vertexai, googlegenai)
+ *   --modules <names>    Comma-separated module names (default: all)
+ *   --repeat <count>     Number of repetitions (default: 1)
+ *   --evaluate           Enable evaluation phase
+ *   --evaluators <names> Comma-separated evaluator names (default: all)
+ */
+export {};
+//# sourceMappingURL=run-comparison.d.ts.map

package/dist/src/run-comparison.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-comparison.d.ts","sourceRoot":"","sources":["../../src/run-comparison.ts"],"names":[],"mappings":";AACA;;;;;;;;;;;;;;;;;;GAkBG"}