@llm-dev-ops/agentics-cli 1.4.26 → 1.4.27

package/dist/bundled-agents/test-bench-agents/dist/core/llm-test-bench.js

@@ -0,0 +1,246 @@
+import { executeCLI, findCLIPath } from '../utils/cli-executor.js';
+import { validateCompletionRequest, validateBenchmarkConfig, validateEvaluationConfig, isValidModel, isValidProvider, } from '../utils/validators.js';
+export class LLMTestBench {
+    cliPath;
+    config;
+    constructor(config = {}) {
+        const resolvedCliPath = config.cliPath ?? findCLIPath();
+        if (!resolvedCliPath) {
+            throw new Error('LLM Test Bench CLI not found. Please install it via cargo or npm, or provide cliPath in config.');
+        }
+        this.cliPath = resolvedCliPath;
+        this.config = {
+            cliPath: resolvedCliPath,
+            workingDir: config.workingDir ?? process.cwd(),
+            verbose: config.verbose ?? false,
+            env: config.env ?? {},
+            timeout: config.timeout ?? 120000,
+        };
+    }
+    async version() {
+        const result = await executeCLI(this.cliPath, {
+            args: ['--version'],
+            cwd: this.config.workingDir,
+            env: this.config.env,
+            timeout: 5000,
+        });
+        if (!result.success) {
+            throw new Error(`Failed to get version: ${result.error}`);
+        }
+        const match = result.stdout.match(/(\d+\.\d+\.\d+)/);
+        const version = match?.[1];
+        return version ?? 'unknown';
+    }
+    async listModels(provider) {
+        const args = ['models', 'list'];
+        if (provider) {
+            if (!isValidProvider(provider)) {
+                throw new Error(`Invalid provider: ${provider}`);
+            }
+            args.push('--provider', provider);
+        }
+        args.push('--json');
+        const result = await executeCLI(this.cliPath, {
+            args,
+            cwd: this.config.workingDir,
+            env: this.config.env,
+            timeout: 10000,
+            parseJson: true,
+        });
+        if (!result.success || !result.data) {
+            throw new Error(`Failed to list models: ${result.error}`);
+        }
+        return result.data.models;
+    }
+    async benchmark(options) {
+        if (!isValidProvider(options.provider)) {
+            throw new Error(`Invalid provider: ${options.provider}`);
+        }
+        if (!isValidModel(options.model)) {
+            throw new Error(`Invalid model: ${options.model}`);
+        }
+        if (!options.prompts || options.prompts.length === 0) {
+            throw new Error('At least one prompt is required');
+        }
+        if (options.config) {
+            validateBenchmarkConfig(options.config);
+        }
+        const args = [
+            'bench',
+            '--provider',
+            options.provider,
+            '--model',
+            options.model,
+            '--json',
+        ];
+        for (const prompt of options.prompts) {
+            args.push('--prompt', prompt);
+        }
+        if (options.config) {
+            if (options.config.concurrency) {
+                args.push('--concurrency', options.config.concurrency.toString());
+            }
+            if (options.config.saveResponses) {
+                args.push('--save-responses');
+            }
+            if (options.config.outputPath) {
+                args.push('--output', options.config.outputPath);
+            }
+            if (options.config.maxDuration) {
+                args.push('--max-duration', options.config.maxDuration.toString());
+            }
+            if (options.config.showProgress === false) {
+                args.push('--no-progress');
+            }
+        }
+        const result = await executeCLI(this.cliPath, {
+            args,
+            cwd: this.config.workingDir,
+            env: this.config.env,
+            timeout: this.config.timeout,
+            parseJson: true,
+        });
+        if (!result.success || !result.data) {
+            throw new Error(`Benchmark failed: ${result.error}`);
+        }
+        return result.data;
+    }
+    async compare(options) {
+        if (!options.models || options.models.length < 2) {
+            throw new Error('At least two models are required for comparison');
+        }
+        if (!options.prompts || options.prompts.length === 0) {
+            throw new Error('At least one prompt is required');
+        }
+        for (const { provider, model } of options.models) {
+            if (!isValidProvider(provider)) {
+                throw new Error(`Invalid provider: ${provider}`);
+            }
+            if (!isValidModel(model)) {
+                throw new Error(`Invalid model: ${model}`);
+            }
+        }
+        if (options.config) {
+            validateBenchmarkConfig(options.config);
+        }
+        const args = ['compare', '--json'];
+        const modelSpecs = options.models.map((m) => `${m.provider}:${m.model}`).join(',');
+        args.push('--models', modelSpecs);
+        for (const prompt of options.prompts) {
+            args.push('--prompt', prompt);
+        }
+        if (options.config) {
+            if (options.config.concurrency) {
+                args.push('--concurrency', options.config.concurrency.toString());
+            }
+            if (options.config.outputPath) {
+                args.push('--output', options.config.outputPath);
+            }
+        }
+        const result = await executeCLI(this.cliPath, {
+            args,
+            cwd: this.config.workingDir,
+            env: this.config.env,
+            timeout: this.config.timeout,
+            parseJson: true,
+        });
+        if (!result.success || !result.data) {
+            throw new Error(`Comparison failed: ${result.error}`);
+        }
+        return result.data;
+    }
+    async evaluate(text, config) {
+        if (!text || text.trim().length === 0) {
+            throw new Error('Text to evaluate cannot be empty');
+        }
+        validateEvaluationConfig(config);
+        const args = ['eval', '--text', text, '--json'];
+        for (const evaluator of config.evaluators) {
+            args.push('--evaluator', evaluator);
+        }
+        if (config.referenceText) {
+            args.push('--reference', config.referenceText);
+        }
+        if (config.context && config.context.length > 0) {
+            for (const ctx of config.context) {
+                args.push('--context', ctx);
+            }
+        }
+        if (config.judgeModel) {
+            args.push('--judge-model', config.judgeModel);
+        }
+        if (config.parallel) {
+            args.push('--parallel');
+        }
+        if (config.cacheResults === false) {
+            args.push('--no-cache');
+        }
+        const result = await executeCLI(this.cliPath, {
+            args,
+            cwd: this.config.workingDir,
+            env: this.config.env,
+            timeout: this.config.timeout,
+            parseJson: true,
+        });
+        if (!result.success || !result.data) {
+            throw new Error(`Evaluation failed: ${result.error}`);
+        }
+        return result.data;
+    }
+    async complete(request) {
+        validateCompletionRequest(request);
+        const args = ['complete', '--model', request.model, '--prompt', request.prompt, '--json'];
+        if (request.maxTokens) {
+            args.push('--max-tokens', request.maxTokens.toString());
+        }
+        if (request.temperature !== undefined) {
+            args.push('--temperature', request.temperature.toString());
+        }
+        if (request.topP !== undefined) {
+            args.push('--top-p', request.topP.toString());
+        }
+        if (request.stop && request.stop.length > 0) {
+            for (const stop of request.stop) {
+                args.push('--stop', stop);
+            }
+        }
+        const result = await executeCLI(this.cliPath, {
+            args,
+            cwd: this.config.workingDir,
+            env: this.config.env,
+            timeout: this.config.timeout,
+            parseJson: true,
+        });
+        if (!result.success || !result.data) {
+            throw new Error(`Completion failed: ${result.error}`);
+        }
+        return result.data;
+    }
+    async optimize(options) {
+        if (!options.prompts || options.prompts.length === 0) {
+            throw new Error('At least one prompt is required');
+        }
+        const args = ['optimize', '--metric', options.metric, '--json'];
+        for (const prompt of options.prompts) {
+            args.push('--prompt', prompt);
+        }
+        if (options.maxCost !== undefined) {
+            args.push('--max-cost', options.maxCost.toString());
+        }
+        if (options.minQuality !== undefined) {
+            args.push('--min-quality', options.minQuality.toString());
+        }
+        const result = await executeCLI(this.cliPath, {
+            args,
+            cwd: this.config.workingDir,
+            env: this.config.env,
+            timeout: this.config.timeout,
+            parseJson: true,
+        });
+        if (!result.success || !result.data) {
+            throw new Error(`Optimization failed: ${result.error}`);
+        }
+        return result.data;
+    }
+}
+//# sourceMappingURL=llm-test-bench.js.map

package/dist/bundled-agents/test-bench-agents/dist/core/llm-test-bench.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-test-bench.js","sourceRoot":"","sources":["../../src/core/llm-test-bench.ts"],"names":[],"mappings":"AAkBA,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AACnE,OAAO,EACL,yBAAyB,EACzB,uBAAuB,EACvB,wBAAwB,EACxB,YAAY,EACZ,eAAe,GAChB,MAAM,wBAAwB,CAAC;AA0BhC,MAAM,OAAO,YAAY;IACf,OAAO,CAAS;IAChB,MAAM,CAAsB;IAQpC,YAAY,SAAoB,EAAE;QAChC,MAAM,eAAe,GAAG,MAAM,CAAC,OAAO,IAAI,WAAW,EAAE,CAAC;QACxD,IAAI,CAAC,eAAe,EAAE,CAAC;YACrB,MAAM,IAAI,KAAK,CACb,iGAAiG,CAClG,CAAC;QACJ,CAAC;QAED,IAAI,CAAC,OAAO,GAAG,eAAe,CAAC;QAC/B,IAAI,CAAC,MAAM,GAAG;YACZ,OAAO,EAAE,eAAe;YACxB,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,OAAO,CAAC,GAAG,EAAE;YAC9C,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,KAAK;YAChC,GAAG,EAAE,MAAM,CAAC,GAAG,IAAI,EAAE;YACrB,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,MAAM;SAClC,CAAC;IACJ,CAAC;IAOD,KAAK,CAAC,OAAO;QACX,MAAM,MAAM,GAAG,MAAM,UAAU,CAAsB,IAAI,CAAC,OAAO,EAAE;YACjE,IAAI,EAAE,CAAC,WAAW,CAAC;YACnB,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU;YAC3B,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG;YACpB,OAAO,EAAE,IAAI;SACd,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,0BAA0B,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;QAC5D,CAAC;QAGD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;QACrD,MAAM,OAAO,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC;QAC3B,OAAO,OAAO,IAAI,SAAS,CAAC;IAC9B,CAAC;IAQD,KAAK,CAAC,UAAU,CAAC,QAAuB;QACtC,MAAM,IAAI,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAChC,IAAI,QAAQ,EAAE,CAAC;YACb,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC/B,MAAM,IAAI,KAAK,CAAC,qBAAqB,QAAQ,EAAE,CAAC,CAAC;YACnD,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;QACpC,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAEpB,MAAM,MAAM,GAAG,MAAM,UAAU,CAA0B,IAAI,CAAC,OAAO,EAAE;YACrE,IAAI;YACJ,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU;YAC3B,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG;YACpB,OAAO,EAAE,KAAK;YACd,SAAS,EAAE,IAAI;SAChB,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACpC,MAAM,IAAI,KAAK,CAAC,0BAA0B,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;QAC5D,CAAC;QAED,OAAO,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC;IAC5B,CAAC;IAQD,KAAK,CAAC,SAAS,CAAC,OAKf;QACC,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;YACvC,MAAM,IAAI,KAAK,CAAC,qBAAqB,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC3D,CAAC;QACD,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CAAC,kBAAkB,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;QACrD,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrD,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;QACrD,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACnB,uBAAuB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC1C,CAAC;QAED,MAAM,IAAI,GAAG;YACX,OAAO;YACP,YAAY;YACZ,OAAO,CAAC,QAAQ;YAChB,SAAS;YACT,OAAO,CAAC,KAAK;YACb,QAAQ;SACT,CAAC;QAGF,KAAK,MAAM,MAAM,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACrC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QAChC,CAAC;QAGD,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACnB,IAAI,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;gBAC/B,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC,CAAC;YACpE,CAAC;YACD,IAAI,OAAO,CAAC,MAAM,CAAC,aAAa,EAAE,CAAC;gBACjC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;YAChC,CAAC;YACD,IAAI,OAAO,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;gBAC9B,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YACnD,CAAC;YACD,IAAI,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;gBAC/B,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC,CAAC;YACrE,CAAC;YACD,IAAI,OAAO,CAAC,MAAM,CAAC,YAAY,KAAK,KAAK,EAAE,CAAC;gBAC1C,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YAC7B,CAAC;QACH,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,UAAU,CAAmB,IAAI,CAAC,OAAO,EAAE;YAC9D,IAAI;YACJ,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU;YAC3B,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG;YACpB,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO;YAC5B,SAAS,EAAE,IAAI;SAChB,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACpC,MAAM,IAAI,KAAK,CAAC,qBAAqB,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;QACvD,CAAC;QAED,OAAO,MAAM,CAAC,IAAI,CAAC;IACrB,CAAC;IAQD,KAAK,CAAC,OAAO,CAAC,OAIb;QACC,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACjD,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;QACrE,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrD,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;QACrD,CAAC;QAED,KAAK,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACjD,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC/B,MAAM,IAAI,KAAK,CAAC,qBAAqB,QAAQ,EAAE,CAAC,CAAC;YACnD,CAAC;YACD,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;gBACzB,MAAM,IAAI,KAAK,CAAC,kBAAkB,KAAK,EAAE,CAAC,CAAC;YAC7C,CAAC;QACH,CAAC;QAED,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACnB,uBAAuB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC1C,CAAC;QAED,MAAM,IAAI,GAAG,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAGnC,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACnF,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;QAGlC,KAAK,MAAM,MAAM,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACrC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QAChC,CAAC;QAGD,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACnB,IAAI,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;gBAC/B,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC,CAAC;YACpE,CAAC;YACD,IAAI,OAAO,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;gBAC9B,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YACnD,CAAC;QACH,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,UAAU,CAAmB,IAAI,CAAC,OAAO,EAAE;YAC9D,IAAI;YACJ,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU;YAC3B,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG;YACpB,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO;YAC5B,SAAS,EAAE,IAAI;SAChB,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACpC,MAAM,IAAI,KAAK,CAAC,sBAAsB,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;QACxD,CAAC;QAED,OAAO,MAAM,CAAC,IAAI,CAAC;IACrB,CAAC;IASD,KAAK,CAAC,QAAQ,CACZ,IAAY,EACZ,MAAwB;QAExB,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtC,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;QACtD,CAAC;QAED,wBAAwB,CAAC,MAAM,CAAC,CAAC;QAEjC,MAAM,IAAI,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC;QAGhD,KAAK,MAAM,SAAS,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;YAC1C,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;QACtC,CAAC;QAGD,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;YACzB,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,MAAM,CAAC,aAAa,CAAC,CAAC;QACjD,CAAC;QACD,IAAI,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChD,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBACjC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC;QACD,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;YACtB,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;QAChD,CAAC;QACD,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YACpB,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAC1B,CAAC;QACD,IAAI,MAAM,CAAC,YAAY,KAAK,KAAK,EAAE,CAAC;YAClC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAC1B,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,UAAU,CAA4B,IAAI,CAAC,OAAO,EAAE;YACvE,IAAI;YACJ,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU;YAC3B,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG;YACpB,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO;YAC5B,SAAS,EAAE,IAAI;SAChB,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACpC,MAAM,IAAI,KAAK,CAAC,sBAAsB,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;QACxD,CAAC;QAED,OAAO,MAAM,CAAC,IAAI,CAAC;IACrB,CAAC;IAQD,KAAK,CAAC,QAAQ,CAAC,OAA0B;QACvC,yBAAyB,CAAC,OAAO,CAAC,CAAC;QAEnC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,SAAS,EAAE,OAAO,CAAC,KAAK,EAAE,UAAU,EAAE,OAAO,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAE1F,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YACtB,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC1D,CAAC;QACD,IAAI,OAAO,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;YACtC,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,OAAO,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC7D,CAAC;QACD,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC/B,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QAChD,CAAC;QACD,IAAI,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5C,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;gBAChC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YAC5B,CAAC;QACH,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,UAAU,CAAqB,IAAI,CAAC,OAAO,EAAE;YAChE,IAAI;YACJ,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU;YAC3B,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG;YACpB,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO;YAC5B,SAAS,EAAE,IAAI;SAChB,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACpC,MAAM,IAAI,KAAK,CAAC,sBAAsB,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;QACxD,CAAC;QAED,OAAO,MAAM,CAAC,IAAI,CAAC;IACrB,CAAC;IAQD,KAAK,CAAC,QAAQ,CAAC,OAKd;QACC,IAAI,CAAC,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrD,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;QACrD,CAAC;QAED,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAEhE,KAAK,MAAM,MAAM,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACrC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QAChC,CAAC;QAED,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YAClC,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,OAAO,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;QACtD,CAAC;QACD,IAAI,OAAO,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;YACrC,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,OAAO,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC5D,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,UAAU,CAI5B,IAAI,CAAC,OAAO,EAAE;YACf,IAAI;YACJ,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU;YAC3B,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG;YACpB,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO;YAC5B,SAAS,EAAE,IAAI;SAChB,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACpC,MAAM,IAAI,KAAK,CAAC,wBAAwB,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;QAC1D,CAAC;QAED,OAAO,MAAM,CAAC,IAAI,CAAC;IACrB,CAAC;CACF"}

package/dist/bundled-agents/test-bench-agents/dist/core/provider-client.d.ts

@@ -0,0 +1,37 @@
+import type { CompletionRequest, CompletionResponse, ProviderName, ModelInfo } from '../types/index.js';
+import { LLMTestBench } from './llm-test-bench.js';
+export declare abstract class ProviderClient {
+    protected ltb: LLMTestBench;
+    protected providerName: ProviderName;
+    constructor(ltb: LLMTestBench, providerName: ProviderName);
+    listModels(): Promise<ModelInfo[]>;
+    complete(prompt: string, options?: Partial<CompletionRequest>): Promise<CompletionResponse>;
+}
+export declare class OpenAIClient extends ProviderClient {
+    constructor(ltb: LLMTestBench);
+    gpt4(prompt: string, options?: Partial<CompletionRequest>): Promise<CompletionResponse>;
+    gpt4Turbo(prompt: string, options?: Partial<CompletionRequest>): Promise<CompletionResponse>;
+    gpt4o(prompt: string, options?: Partial<CompletionRequest>): Promise<CompletionResponse>;
+    gpt35Turbo(prompt: string, options?: Partial<CompletionRequest>): Promise<CompletionResponse>;
+}
+export declare class AnthropicClient extends ProviderClient {
+    constructor(ltb: LLMTestBench);
+    claudeOpus4(prompt: string, options?: Partial<CompletionRequest>): Promise<CompletionResponse>;
+    claudeSonnet45(prompt: string, options?: Partial<CompletionRequest>): Promise<CompletionResponse>;
+    claude35Sonnet(prompt: string, options?: Partial<CompletionRequest>): Promise<CompletionResponse>;
+    claude35Haiku(prompt: string, options?: Partial<CompletionRequest>): Promise<CompletionResponse>;
+}
+export declare class GoogleClient extends ProviderClient {
+    constructor(ltb: LLMTestBench);
+    gemini25Pro(prompt: string, options?: Partial<CompletionRequest>): Promise<CompletionResponse>;
+    gemini15Pro(prompt: string, options?: Partial<CompletionRequest>): Promise<CompletionResponse>;
+    gemini15Flash(prompt: string, options?: Partial<CompletionRequest>): Promise<CompletionResponse>;
+}
+export declare class ProviderClientFactory {
+    private ltb;
+    constructor(ltb: LLMTestBench);
+    openai(): OpenAIClient;
+    anthropic(): AnthropicClient;
+    google(): GoogleClient;
+}
+//# sourceMappingURL=provider-client.d.ts.map

package/dist/bundled-agents/test-bench-agents/dist/core/provider-client.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"provider-client.d.ts","sourceRoot":"","sources":["../../src/core/provider-client.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EACV,iBAAiB,EACjB,kBAAkB,EAClB,YAAY,EACZ,SAAS,EACV,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAKnD,8BAAsB,cAAc;IAClC,SAAS,CAAC,GAAG,EAAE,YAAY,CAAC;IAC5B,SAAS,CAAC,YAAY,EAAE,YAAY,CAAC;gBAEzB,GAAG,EAAE,YAAY,EAAE,YAAY,EAAE,YAAY;IAQnD,UAAU,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;IAOlC,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,iBAAiB,CAAC,GAAG,OAAO,CAAC,kBAAkB,CAAC;CAmBlG;AAKD,qBAAa,YAAa,SAAQ,cAAc;gBAClC,GAAG,EAAE,YAAY;IAOvB,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,iBAAiB,CAAC,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAWvF,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,iBAAiB,CAAC,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAW5F,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,iBAAiB,CAAC,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAWxF,UAAU,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,iBAAiB,CAAC,GAAG,OAAO,CAAC,kBAAkB,CAAC;CAOpG;AAKD,qBAAa,eAAgB,SAAQ,cAAc;gBACrC,GAAG,EAAE,YAAY;IAOvB,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,iBAAiB,CAAC,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAW9F,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,iBAAiB,CAAC,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAWjG,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,iBAAiB,CAAC,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAWjG,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,iBAAiB,CAAC,GAAG,OAAO,CAAC,kBAAkB,CAAC;CAOvG;AAKD,qBAAa,YAAa,SAAQ,cAAc;gBAClC,GAAG,EAAE,YAAY;IAOvB,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,iBAAiB,CAAC,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAW9F,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,iBAAiB,CAAC,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAW9F,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,iBAAiB,CAAC,GAAG,OAAO,CAAC,kBAAkB,CAAC;CAOvG;AAKD,qBAAa,qBAAqB;IACpB,OAAO,CAAC,GAAG;gBAAH,GAAG,EAAE,YAAY;IAKrC,MAAM,IAAI,YAAY;IAOtB,SAAS,IAAI,eAAe;IAO5B,MAAM,IAAI,YAAY;CAGvB"}

package/dist/bundled-agents/test-bench-agents/dist/core/provider-client.js

@@ -0,0 +1,135 @@
+export class ProviderClient {
+    ltb;
+    providerName;
+    constructor(ltb, providerName) {
+        this.ltb = ltb;
+        this.providerName = providerName;
+    }
+    async listModels() {
+        return this.ltb.listModels(this.providerName);
+    }
+    async complete(prompt, options) {
+        const models = await this.listModels();
+        if (models.length === 0) {
+            throw new Error(`No models available for provider: ${this.providerName}`);
+        }
+        const firstModel = models[0];
+        if (!firstModel) {
+            throw new Error(`Unable to retrieve default model for provider: ${this.providerName}`);
+        }
+        const request = {
+            model: firstModel.id,
+            prompt,
+            ...options,
+        };
+        return this.ltb.complete(request);
+    }
+}
+export class OpenAIClient extends ProviderClient {
+    constructor(ltb) {
+        super(ltb, 'openai');
+    }
+    async gpt4(prompt, options) {
+        return this.ltb.complete({
+            model: 'gpt-4',
+            prompt,
+            ...options,
+        });
+    }
+    async gpt4Turbo(prompt, options) {
+        return this.ltb.complete({
+            model: 'gpt-4-turbo',
+            prompt,
+            ...options,
+        });
+    }
+    async gpt4o(prompt, options) {
+        return this.ltb.complete({
+            model: 'gpt-4o',
+            prompt,
+            ...options,
+        });
+    }
+    async gpt35Turbo(prompt, options) {
+        return this.ltb.complete({
+            model: 'gpt-3.5-turbo',
+            prompt,
+            ...options,
+        });
+    }
+}
+export class AnthropicClient extends ProviderClient {
+    constructor(ltb) {
+        super(ltb, 'anthropic');
+    }
+    async claudeOpus4(prompt, options) {
+        return this.ltb.complete({
+            model: 'claude-opus-4',
+            prompt,
+            ...options,
+        });
+    }
+    async claudeSonnet45(prompt, options) {
+        return this.ltb.complete({
+            model: 'claude-sonnet-4.5',
+            prompt,
+            ...options,
+        });
+    }
+    async claude35Sonnet(prompt, options) {
+        return this.ltb.complete({
+            model: 'claude-3-5-sonnet-latest',
+            prompt,
+            ...options,
+        });
+    }
+    async claude35Haiku(prompt, options) {
+        return this.ltb.complete({
+            model: 'claude-3-5-haiku-latest',
+            prompt,
+            ...options,
+        });
+    }
+}
+export class GoogleClient extends ProviderClient {
+    constructor(ltb) {
+        super(ltb, 'google');
+    }
+    async gemini25Pro(prompt, options) {
+        return this.ltb.complete({
+            model: 'gemini-2.5-pro',
+            prompt,
+            ...options,
+        });
+    }
+    async gemini15Pro(prompt, options) {
+        return this.ltb.complete({
+            model: 'gemini-1.5-pro',
+            prompt,
+            ...options,
+        });
+    }
+    async gemini15Flash(prompt, options) {
+        return this.ltb.complete({
+            model: 'gemini-1.5-flash',
+            prompt,
+            ...options,
+        });
+    }
+}
+export class ProviderClientFactory {
+    ltb;
+    constructor(ltb) {
+        this.ltb = ltb;
+    }
+    openai() {
+        return new OpenAIClient(this.ltb);
+    }
+    anthropic() {
+        return new AnthropicClient(this.ltb);
+    }
+    google() {
+        return new GoogleClient(this.ltb);
+    }
+}
+//# sourceMappingURL=provider-client.js.map

package/dist/bundled-agents/test-bench-agents/dist/core/provider-client.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"provider-client.js","sourceRoot":"","sources":["../../src/core/provider-client.ts"],"names":[],"mappings":"AAkBA,MAAM,OAAgB,cAAc;IACxB,GAAG,CAAe;IAClB,YAAY,CAAe;IAErC,YAAY,GAAiB,EAAE,YAA0B;QACvD,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;QACf,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IACnC,CAAC;IAKD,KAAK,CAAC,UAAU;QACd,OAAO,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAChD,CAAC;IAKD,KAAK,CAAC,QAAQ,CAAC,MAAc,EAAE,OAAoC;QACjE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACvC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,qCAAqC,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC;QAC5E,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAC7B,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,kDAAkD,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC;QACzF,CAAC;QAED,MAAM,OAAO,GAAsB;YACjC,KAAK,EAAE,UAAU,CAAC,EAAE;YACpB,MAAM;YACN,GAAG,OAAO;SACX,CAAC;QAEF,OAAO,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IACpC,CAAC;CACF;AAKD,MAAM,OAAO,YAAa,SAAQ,cAAc;IAC9C,YAAY,GAAiB;QAC3B,KAAK,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IACvB,CAAC;IAKD,KAAK,CAAC,IAAI,CAAC,MAAc,EAAE,OAAoC;QAC7D,OAAO,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC;YACvB,KAAK,EAAE,OAAO;YACd,MAAM;YACN,GAAG,OAAO;SACX,CAAC,CAAC;IACL,CAAC;IAKD,KAAK,CAAC,SAAS,CAAC,MAAc,EAAE,OAAoC;QAClE,OAAO,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC;YACvB,KAAK,EAAE,aAAa;YACpB,MAAM;YACN,GAAG,OAAO;SACX,CAAC,CAAC;IACL,CAAC;IAKD,KAAK,CAAC,KAAK,CAAC,MAAc,EAAE,OAAoC;QAC9D,OAAO,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC;YACvB,KAAK,EAAE,QAAQ;YACf,MAAM;YACN,GAAG,OAAO;SACX,CAAC,CAAC;IACL,CAAC;IAKD,KAAK,CAAC,UAAU,CAAC,MAAc,EAAE,OAAoC;QACnE,OAAO,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC;YACvB,KAAK,EAAE,eAAe;YACtB,MAAM;YACN,GAAG,OAAO;SACX,CAAC,CAAC;IACL,CAAC;CACF;AAKD,MAAM,OAAO,eAAgB,SAAQ,cAAc;IACjD,YAAY,GAAiB;QAC3B,KAAK,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;IAC1B,CAAC;IAKD,KAAK,CAAC,WAAW,CAAC,MAAc,EAAE,OAAoC;QACpE,OAAO,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC;YACvB,KAAK,EAAE,eAAe;YACtB,MAAM;YACN,GAAG,OAAO;SACX,CAAC,CAAC;IACL,CAAC;IAKD,KAAK,CAAC,cAAc,CAAC,MAAc,EAAE,OAAoC;QACvE,OAAO,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC;YACvB,KAAK,EAAE,mBAAmB;YAC1B,MAAM;YACN,GAAG,OAAO;SACX,CAAC,CAAC;IACL,CAAC;IAKD,KAAK,CAAC,cAAc,CAAC,MAAc,EAAE,OAAoC;QACvE,OAAO,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC;YACvB,KAAK,EAAE,0BAA0B;YACjC,MAAM;YACN,GAAG,OAAO;SACX,CAAC,CAAC;IACL,CAAC;IAKD,KAAK,CAAC,aAAa,CAAC,MAAc,EAAE,OAAoC;QACtE,OAAO,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC;YACvB,KAAK,EAAE,yBAAyB;YAChC,MAAM;YACN,GAAG,OAAO;SACX,CAAC,CAAC;IACL,CAAC;CACF;AAKD,MAAM,OAAO,YAAa,SAAQ,cAAc;IAC9C,YAAY,GAAiB;QAC3B,KAAK,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IACvB,CAAC;IAKD,KAAK,CAAC,WAAW,CAAC,MAAc,EAAE,OAAoC;QACpE,OAAO,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC;YACvB,KAAK,EAAE,gBAAgB;YACvB,MAAM;YACN,GAAG,OAAO;SACX,CAAC,CAAC;IACL,CAAC;IAKD,KAAK,CAAC,WAAW,CAAC,MAAc,EAAE,OAAoC;QACpE,OAAO,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC;YACvB,KAAK,EAAE,gBAAgB;YACvB,MAAM;YACN,GAAG,OAAO;SACX,CAAC,CAAC;IACL,CAAC;IAKD,KAAK,CAAC,aAAa,CAAC,MAAc,EAAE,OAAoC;QACtE,OAAO,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC;YACvB,KAAK,EAAE,kBAAkB;YACzB,MAAM;YACN,GAAG,OAAO;SACX,CAAC,CAAC;IACL,CAAC;CACF;AAKD,MAAM,OAAO,qBAAqB;IACZ;IAApB,YAAoB,GAAiB;QAAjB,QAAG,GAAH,GAAG,CAAc;IAAG,CAAC;IAKzC,MAAM;QACJ,OAAO,IAAI,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACpC,CAAC;IAKD,SAAS;QACP,OAAO,IAAI,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACvC,CAAC;IAKD,MAAM;QACJ,OAAO,IAAI,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACpC,CAAC;CACF"}

package/dist/bundled-agents/test-bench-agents/dist/evaluators/index.d.ts

@@ -0,0 +1,28 @@
+import { LLMTestBench } from '../core/llm-test-bench.js';
+import type { CombinedEvaluationResults, EvaluatorType } from '../types/index.js';
+export declare class Evaluator {
+    private ltb;
+    constructor(ltb: LLMTestBench);
+    evaluateAll(text: string, options?: {
+        referenceText?: string;
+        context?: string[];
+    }): Promise<CombinedEvaluationResults>;
+    evaluateQuality(text: string): Promise<CombinedEvaluationResults>;
+    evaluateSafety(text: string): Promise<CombinedEvaluationResults>;
+    evaluateAccuracy(text: string, options: {
+        referenceText?: string;
+        context?: string[];
+    }): Promise<CombinedEvaluationResults>;
+    llmAsJudge(text: string, options: {
+        criteria: string[];
+        judgeModel?: string;
+        referenceText?: string;
+    }): Promise<CombinedEvaluationResults>;
+    evaluateWith(text: string, evaluators: EvaluatorType[], options?: {
+        referenceText?: string;
+        context?: string[];
+        judgeModel?: string;
+    }): Promise<CombinedEvaluationResults>;
+}
+export declare function createEvaluator(ltb: LLMTestBench): Evaluator;
+//# sourceMappingURL=index.d.ts.map

package/dist/bundled-agents/test-bench-agents/dist/evaluators/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/evaluators/index.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AACzD,OAAO,KAAK,EAEV,yBAAyB,EACzB,aAAa,EACd,MAAM,mBAAmB,CAAC;AAK3B,qBAAa,SAAS;IACR,OAAO,CAAC,GAAG;gBAAH,GAAG,EAAE,YAAY;IAK/B,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QACxC,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;KACpB,GAAG,OAAO,CAAC,yBAAyB,CAAC;IAuBhC,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,yBAAyB,CAAC;IAYjE,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,yBAAyB,CAAC;IAYhE,gBAAgB,CACpB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE;QACP,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;KACpB,GACA,OAAO,CAAC,yBAAyB,CAAC;IAa/B,UAAU,CACd,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE;QACP,QAAQ,EAAE,MAAM,EAAE,CAAC;QACnB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,aAAa,CAAC,EAAE,MAAM,CAAC;KACxB,GACA,OAAO,CAAC,yBAAyB,CAAC;IAc/B,YAAY,CAChB,IAAI,EAAE,MAAM,EACZ,UAAU,EAAE,aAAa,EAAE,EAC3B,OAAO,CAAC,EAAE;QACR,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;QACnB,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,GACA,OAAO,CAAC,yBAAyB,CAAC;CAStC;AAKD,wBAAgB,eAAe,CAAC,GAAG,EAAE,YAAY,GAAG,SAAS,CAE5D"}

package/dist/bundled-agents/test-bench-agents/dist/evaluators/index.js

@@ -0,0 +1,67 @@
+export class Evaluator {
+    ltb;
+    constructor(ltb) {
+        this.ltb = ltb;
+    }
+    async evaluateAll(text, options) {
+        const config = {
+            evaluators: [
+                'perplexity',
+                'coherence',
+                'relevance',
+                'faithfulness',
+                'readability',
+                'sentiment',
+                'toxicity',
+                'pii-detection',
+            ],
+            ...options,
+            parallel: true,
+            cacheResults: true,
+        };
+        return this.ltb.evaluate(text, config);
+    }
+    async evaluateQuality(text) {
+        const config = {
+            evaluators: ['perplexity', 'coherence', 'readability'],
+            parallel: true,
+        };
+        return this.ltb.evaluate(text, config);
+    }
+    async evaluateSafety(text) {
+        const config = {
+            evaluators: ['toxicity', 'pii-detection'],
+            parallel: true,
+        };
+        return this.ltb.evaluate(text, config);
+    }
+    async evaluateAccuracy(text, options) {
+        const config = {
+            evaluators: ['relevance', 'faithfulness'],
+            ...options,
+            parallel: true,
+        };
+        return this.ltb.evaluate(text, config);
+    }
+    async llmAsJudge(text, options) {
+        const config = {
+            evaluators: ['llm-as-judge'],
+            customCriteria: options.criteria,
+            judgeModel: options.judgeModel ?? 'gpt-4',
+            referenceText: options.referenceText,
+        };
+        return this.ltb.evaluate(text, config);
+    }
+    async evaluateWith(text, evaluators, options) {
+        const config = {
+            evaluators,
+            ...options,
+            parallel: true,
+        };
+        return this.ltb.evaluate(text, config);
+    }
+}
+export function createEvaluator(ltb) {
+    return new Evaluator(ltb);
+}
+//# sourceMappingURL=index.js.map

package/dist/bundled-agents/test-bench-agents/dist/evaluators/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/evaluators/index.ts"],"names":[],"mappings":"AAgBA,MAAM,OAAO,SAAS;IACA;IAApB,YAAoB,GAAiB;QAAjB,QAAG,GAAH,GAAG,CAAc;IAAG,CAAC;IAKzC,KAAK,CAAC,WAAW,CAAC,IAAY,EAAE,OAG/B;QACC,MAAM,MAAM,GAAqB;YAC/B,UAAU,EAAE;gBACV,YAAY;gBACZ,WAAW;gBACX,WAAW;gBACX,cAAc;gBACd,aAAa;gBACb,WAAW;gBACX,UAAU;gBACV,eAAe;aAChB;YACD,GAAG,OAAO;YACV,QAAQ,EAAE,IAAI;YACd,YAAY,EAAE,IAAI;SACnB,CAAC;QAEF,OAAO,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACzC,CAAC;IAKD,KAAK,CAAC,eAAe,CAAC,IAAY;QAChC,MAAM,MAAM,GAAqB;YAC/B,UAAU,EAAE,CAAC,YAAY,EAAE,WAAW,EAAE,aAAa,CAAC;YACtD,QAAQ,EAAE,IAAI;SACf,CAAC;QAEF,OAAO,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACzC,CAAC;IAKD,KAAK,CAAC,cAAc,CAAC,IAAY;QAC/B,MAAM,MAAM,GAAqB;YAC/B,UAAU,EAAE,CAAC,UAAU,EAAE,eAAe,CAAC;YACzC,QAAQ,EAAE,IAAI;SACf,CAAC;QAEF,OAAO,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACzC,CAAC;IAKD,KAAK,CAAC,gBAAgB,CACpB,IAAY,EACZ,OAGC;QAED,MAAM,MAAM,GAAqB;YAC/B,UAAU,EAAE,CAAC,WAAW,EAAE,cAAc,CAAC;YACzC,GAAG,OAAO;YACV,QAAQ,EAAE,IAAI;SACf,CAAC;QAEF,OAAO,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACzC,CAAC;IAKD,KAAK,CAAC,UAAU,CACd,IAAY,EACZ,OAIC;QAED,MAAM,MAAM,GAAqB;YAC/B,UAAU,EAAE,CAAC,cAAc,CAAC;YAC5B,cAAc,EAAE,OAAO,CAAC,QAAQ;YAChC,UAAU,EAAE,OAAO,CAAC,UAAU,IAAI,OAAO;YACzC,aAAa,EAAE,OAAO,CAAC,aAAa;SACrC,CAAC;QAEF,OAAO,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACzC,CAAC;IAKD,KAAK,CAAC,YAAY,CAChB,IAAY,EACZ,UAA2B,EAC3B,OAIC;QAED,MAAM,MAAM,GAAqB;YAC/B,UAAU;YACV,GAAG,OAAO;YACV,QAAQ,EAAE,IAAI;SACf,CAAC;QAEF,OAAO,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACzC,CAAC;CACF;AAKD,MAAM,UAAU,eAAe,CAAC,GAAiB;IAC/C,OAAO,IAAI,SAAS,CAAC,GAAG,CAAC,CAAC;AAC5B,CAAC"}

package/dist/bundled-agents/test-bench-agents/dist/execution/default-policy.d.ts

@@ -0,0 +1,3 @@
+import type { ExecutionPolicy } from './policy.js';
+export declare const DEFAULT_EXECUTION_POLICY: ExecutionPolicy;
+//# sourceMappingURL=default-policy.d.ts.map

package/dist/bundled-agents/test-bench-agents/dist/execution/default-policy.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"default-policy.d.ts","sourceRoot":"","sources":["../../src/execution/default-policy.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAEnD,eAAO,MAAM,wBAAwB,EAAE,eAuFtC,CAAC"}

package/dist/bundled-agents/test-bench-agents/dist/execution/default-policy.js

@@ -0,0 +1,85 @@
+export const DEFAULT_EXECUTION_POLICY = {
+    policy_version: '1.0.0',
+    updated_at: '2026-02-12T00:00:00.000Z',
+    description: 'Default policy: benchmark, compare, evaluate, and all stress test intents ' +
+        'are mandatory execution dependencies. Complete and optimize are recommended.',
+    rules: [
+        {
+            intent: 'benchmark',
+            disposition: 'mandatory',
+            rationale: 'Benchmark results must appear in execution graph for auditability',
+        },
+        {
+            intent: 'compare',
+            disposition: 'mandatory',
+            rationale: 'Model comparisons are decision-critical and must be traced',
+        },
+        {
+            intent: 'evaluate',
+            disposition: 'mandatory',
+            rationale: 'Evaluations are quality gates and must be traced',
+        },
+        {
+            intent: 'complete',
+            disposition: 'recommended',
+            rationale: 'Completions should be traced but may be used for ad-hoc exploration',
+        },
+        {
+            intent: 'optimize',
+            disposition: 'recommended',
+            rationale: 'Optimization is advisory; should be traced but not hard-gated',
+        },
+        {
+            intent: 'load_ramp',
+            disposition: 'mandatory',
+            rationale: 'Load ramp results are critical for capacity planning',
+        },
+        {
+            intent: 'spike',
+            disposition: 'mandatory',
+            rationale: 'Spike test results inform resilience decisions',
+        },
+        {
+            intent: 'soak',
+            disposition: 'mandatory',
+            rationale: 'Soak test results inform stability decisions',
+        },
+        {
+            intent: 'extreme_input',
+            disposition: 'mandatory',
+            rationale: 'Extreme input handling must be auditable',
+        },
+        {
+            intent: 'adversarial',
+            disposition: 'mandatory',
+            rationale: 'Adversarial test results are security-critical',
+        },
+        {
+            intent: 'rate_limit_probe',
+            disposition: 'mandatory',
+            rationale: 'Rate limit probing affects operational parameters',
+        },
+        {
+            intent: 'timeout_boundary',
+            disposition: 'mandatory',
+            rationale: 'Timeout boundary results set SLA parameters',
+        },
+        {
+            intent: 'token_limit',
+            disposition: 'mandatory',
+            rationale: 'Token limit results inform context window policies',
+        },
+        {
+            intent: 'context_overflow',
+            disposition: 'mandatory',
+            rationale: 'Context overflow results are input validation critical',
+        },
+        {
+            intent: 'malformed_request',
+            disposition: 'mandatory',
+            rationale: 'Error handling behavior must be auditable',
+        },
+    ],
+    default_disposition: 'recommended',
+};
+//# sourceMappingURL=default-policy.js.map

package/dist/bundled-agents/test-bench-agents/dist/execution/default-policy.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"default-policy.js","sourceRoot":"","sources":["../../src/execution/default-policy.ts"],"names":[],"mappings":"AAUA,MAAM,CAAC,MAAM,wBAAwB,GAAoB;IACvD,cAAc,EAAE,OAAO;IACvB,UAAU,EAAE,0BAA0B;IACtC,WAAW,EACT,4EAA4E;QAC5E,8EAA8E;IAChF,KAAK,EAAE;QAEL;YACE,MAAM,EAAE,WAAW;YACnB,WAAW,EAAE,WAAW;YACxB,SAAS,EAAE,mEAAmE;SAC/E;QACD;YACE,MAAM,EAAE,SAAS;YACjB,WAAW,EAAE,WAAW;YACxB,SAAS,EAAE,4DAA4D;SACxE;QACD;YACE,MAAM,EAAE,UAAU;YAClB,WAAW,EAAE,WAAW;YACxB,SAAS,EAAE,kDAAkD;SAC9D;QAED;YACE,MAAM,EAAE,UAAU;YAClB,WAAW,EAAE,aAAa;YAC1B,SAAS,EAAE,qEAAqE;SACjF;QACD;YACE,MAAM,EAAE,UAAU;YAClB,WAAW,EAAE,aAAa;YAC1B,SAAS,EAAE,+DAA+D;SAC3E;QAED;YACE,MAAM,EAAE,WAAW;YACnB,WAAW,EAAE,WAAW;YACxB,SAAS,EAAE,sDAAsD;SAClE;QACD;YACE,MAAM,EAAE,OAAO;YACf,WAAW,EAAE,WAAW;YACxB,SAAS,EAAE,gDAAgD;SAC5D;QACD;YACE,MAAM,EAAE,MAAM;YACd,WAAW,EAAE,WAAW;YACxB,SAAS,EAAE,8CAA8C;SAC1D;QACD;YACE,MAAM,EAAE,eAAe;YACvB,WAAW,EAAE,WAAW;YACxB,SAAS,EAAE,0CAA0C;SACtD;QACD;YACE,MAAM,EAAE,aAAa;YACrB,WAAW,EAAE,WAAW;YACxB,SAAS,EAAE,gDAAgD;SAC5D;QACD;YACE,MAAM,EAAE,kBAAkB;YAC1B,WAAW,EAAE,WAAW;YACxB,SAAS,EAAE,mDAAmD;SAC/D;QACD;YACE,MAAM,EAAE,kBAAkB;YAC1B,WAAW,EAAE,WAAW;YACxB,SAAS,EAAE,6CAA6C;SACzD;QACD;YACE,MAAM,EAAE,aAAa;YACrB,WAAW,EAAE,WAAW;YACxB,SAAS,EAAE,oDAAoD;SAChE;QACD;YACE,MAAM,EAAE,kBAAkB;YAC1B,WAAW,EAAE,WAAW;YACxB,SAAS,EAAE,wDAAwD;SACpE;QACD;YACE,MAAM,EAAE,mBAAmB;YAC3B,WAAW,EAAE,WAAW;YACxB,SAAS,EAAE,2CAA2C;SACvD;KACF;IACD,mBAAmB,EAAE,aAAa;CACnC,CAAC"}