@prompd/test 0.5.0-beta.9

package/dist/TestRunner.js

@@ -0,0 +1,463 @@
+"use strict";
+/**
+ * TestRunner - orchestrates the full test lifecycle:
+ * discovery -> compile -> execute -> evaluate -> report
+ *
+ * Consumes @prompd/cli for compilation and execution.
+ * This is the primary public API for @prompd/test.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TestRunner = void 0;
+const path = __importStar(require("path"));
+const fs = __importStar(require("fs"));
+const TestDiscovery_1 = require("./TestDiscovery");
+const EvaluatorEngine_1 = require("./EvaluatorEngine");
+const ConsoleReporter_1 = require("./reporters/ConsoleReporter");
+const JsonReporter_1 = require("./reporters/JsonReporter");
+const JunitReporter_1 = require("./reporters/JunitReporter");
+class TestRunner {
+    /**
+     * @param cli - Optional pre-loaded @prompd/cli module. If provided, skips dynamic import.
+     *              This is the recommended approach when running inside Electron where the CLI
+     *              is already loaded by the main process.
+     */
+    constructor(cli) {
+        this.cliModule = null;
+        this.configLoaded = false;
+        this.discovery = new TestDiscovery_1.TestDiscovery();
+        if (cli) {
+            this.cliModule = cli;
+        }
+    }
+    /**
+     * Ensure CLI config is loaded (API keys, provider settings).
+     * Called once before any execution.
+     */
+    async ensureConfig() {
+        if (this.configLoaded)
+            return;
+        const cli = await this.getCli();
+        try {
+            const configManager = new cli.ConfigManager();
+            console.log('[TestRunner] Loading config...');
+            // loadConfig() is async — must await it
+            if (configManager.loadConfig) {
+                await configManager.loadConfig();
+                // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                const cfg = configManager.config;
+                console.log('[TestRunner] Config loaded:', cfg ? Object.keys(cfg) : 'null');
+                console.log('[TestRunner] API keys:', cfg?.apiKeys ? Object.keys(cfg.apiKeys).filter((k) => cfg.apiKeys[k]) : 'none');
+                console.log('[TestRunner] Default provider:', cfg?.defaultProvider);
+            }
+            else if (configManager.load) {
+                await configManager.load();
+                console.log('[TestRunner] Config loaded via load()');
+            }
+            this.configLoaded = true;
+        }
+        catch (err) {
+            console.error('[TestRunner] Config load failed:', err);
+            // Config may not exist — that's OK for --no-llm runs
+        }
+    }
+    /**
+     * Run tests for a target path (file or directory).
+     * Returns structured results and an exit code (0 = all pass, 1 = failures).
+     */
+    async run(targetPath, options = {}, onProgress) {
+        const startTime = Date.now();
+        // 1. Discovery
+        const { suites, errors: discoveryErrors } = await this.discovery.discover(targetPath);
+        if (discoveryErrors.length > 0 && suites.length === 0) {
+            return this.buildErrorResult(discoveryErrors.map(e => e.message), startTime);
+        }
+        // 2. Run each suite
+        const suiteResults = [];
+        for (const suite of suites) {
+            onProgress?.({ type: 'suite_start', suite: suite.name, testCount: suite.tests.length });
+            const results = await this.runSuite(suite, options, onProgress);
+            suiteResults.push({
+                suite: suite.name,
+                testFilePath: suite.testFilePath,
+                results,
+            });
+            onProgress?.({ type: 'suite_complete', suite: suite.name, results });
+        }
+        // 3. Build summary
+        const summary = this.buildSummary(suiteResults, startTime);
+        return { suites: suiteResults, summary };
+    }
+    /**
+     * Run tests and return formatted output string.
+     */
+    async runAndReport(targetPath, options = {}, onProgress) {
+        const result = await this.run(targetPath, options, onProgress);
+        const reporter = this.getReporter(options);
+        const output = reporter.report(result);
+        const exitCode = (result.summary.failed > 0 || result.summary.errors > 0) ? 1 : 0;
+        return { output, exitCode };
+    }
+    async runSuite(suite, options, onProgress) {
+        const results = [];
+        const allowedEvaluators = this.resolveAllowedEvaluators(options);
+        for (const testCase of suite.tests) {
+            onProgress?.({ type: 'test_start', suite: suite.name, testName: testCase.name });
+            const result = await this.runTestCase(suite, testCase, allowedEvaluators, options, onProgress);
+            results.push(result);
+            onProgress?.({ type: 'test_complete', suite: suite.name, testName: testCase.name, result });
+        }
+        return results;
+    }
+    async runTestCase(suite, testCase, allowedEvaluators, options, onProgress) {
+        const start = Date.now();
+        // Step 1: Compile the target .prmd with test params
+        let compiledOutput;
+        let promptMetadata = {};
+        try {
+            const compileResult = await this.compileTarget(suite.target, testCase.params, options);
+            compiledOutput = compileResult.compiled;
+            promptMetadata = compileResult.metadata;
+        }
+        catch (err) {
+            const errorMessage = err instanceof Error ? err.message : String(err);
+            // If expect_error is set, compilation failure is a PASS
+            if (testCase.expect_error) {
+                return {
+                    suite: suite.name,
+                    testName: testCase.name,
+                    status: 'pass',
+                    duration: Date.now() - start,
+                    assertions: [],
+                    error: `Expected error occurred: ${errorMessage}`,
+                };
+            }
+            return {
+                suite: suite.name,
+                testName: testCase.name,
+                status: 'error',
+                duration: Date.now() - start,
+                assertions: [],
+                error: `Compilation failed: ${errorMessage}`,
+            };
+        }
+        // If expect_error was set but compilation succeeded, that's a failure
+        if (testCase.expect_error) {
+            return {
+                suite: suite.name,
+                testName: testCase.name,
+                status: 'fail',
+                duration: Date.now() - start,
+                assertions: [],
+                compiledInput: compiledOutput,
+                error: 'Expected compilation to fail, but it succeeded',
+            };
+        }
+        // Step 2: Execute against LLM (unless --no-llm)
+        let llmOutput = '';
+        let provider = 'none';
+        let model = 'none';
+        let execDuration = 0;
+        let usage;
+        if (!options.noLlm) {
+            try {
+                const execResult = await this.executePrompt(compiledOutput, promptMetadata, options);
+                llmOutput = execResult.response;
+                provider = execResult.provider;
+                model = execResult.model;
+                execDuration = execResult.duration;
+                usage = execResult.usage;
+            }
+            catch (err) {
+                return {
+                    suite: suite.name,
+                    testName: testCase.name,
+                    status: 'error',
+                    duration: Date.now() - start,
+                    assertions: [],
+                    compiledInput: compiledOutput,
+                    error: `Execution failed: ${err instanceof Error ? err.message : String(err)}`,
+                };
+            }
+        }
+        else {
+            // In --no-llm mode, use the compiled output as the "output" for NLP checks
+            // This enables structural assertions against the compiled prompt itself
+            llmOutput = compiledOutput;
+        }
+        const execution = !options.noLlm ? { provider, model, duration: execDuration, usage } : undefined;
+        // Step 3: Run evaluations
+        if (testCase.assert.length === 0) {
+            return {
+                suite: suite.name,
+                testName: testCase.name,
+                status: 'pass',
+                duration: Date.now() - start,
+                assertions: [],
+                output: llmOutput,
+                compiledInput: compiledOutput,
+                execution,
+            };
+        }
+        const engine = new EvaluatorEngine_1.EvaluatorEngine({
+            testFileDir: path.dirname(suite.testFilePath),
+            evaluatorPrompt: suite.evaluatorPrompt,
+            workspaceRoot: options.workspaceRoot,
+            registryUrl: options.registryUrl,
+            allowedEvaluators,
+            failFast: options.runAll ? false : (options.failFast !== false),
+            cliModule: this.cliModule || undefined,
+            provider: options.provider,
+            model: options.model,
+        });
+        const context = {
+            prompt: compiledOutput,
+            response: llmOutput,
+            params: testCase.params,
+            metadata: { provider, model, duration: execDuration },
+        };
+        const assertions = await engine.evaluate(testCase.assert, context, (assertion) => {
+            onProgress?.({
+                type: 'assertion_complete',
+                suite: suite.name,
+                testName: testCase.name,
+                assertion,
+            });
+        });
+        // Determine overall test status from assertions
+        const hasFailure = assertions.some(a => a.status === 'fail');
+        const hasError = assertions.some(a => a.status === 'error');
+        const status = hasError ? 'error' : hasFailure ? 'fail' : 'pass';
+        return {
+            suite: suite.name,
+            testName: testCase.name,
+            status,
+            duration: Date.now() - start,
+            assertions,
+            output: llmOutput,
+            compiledInput: compiledOutput,
+            execution,
+        };
+    }
+    /**
+     * Compile a .prmd file and return both the compiled text and metadata
+     * (provider, model, temperature, max_tokens from frontmatter).
+     */
+    async compileTarget(targetPath, params, options) {
+        const cli = await this.getCli();
+        const compiler = new cli.PrompdCompiler();
+        if (!fs.existsSync(targetPath)) {
+            throw new Error(`Target prompt file not found: ${targetPath}`);
+        }
+        // Use compileWithContext to get both output and frontmatter metadata
+        const context = await compiler.compileWithContext(targetPath, {
+            outputFormat: 'markdown',
+            parameters: params,
+            filePath: targetPath,
+            workspaceRoot: options.workspaceRoot,
+            registryUrl: options.registryUrl,
+            fileSystem: new cli.NodeFileSystem(),
+        });
+        // compileWithContext may return { compiledResult, metadata } or a string
+        let compiled;
+        let metadata = {};
+        if (typeof context === 'string') {
+            compiled = context;
+        }
+        else if (context && typeof context === 'object') {
+            compiled = context.compiledResult || '';
+            metadata = context.metadata || {};
+        }
+        else {
+            throw new Error('Compilation produced no output');
+        }
+        if (!compiled) {
+            throw new Error('Compilation produced no output');
+        }
+        console.log(`[TestRunner] Compiled ${targetPath}`);
+        console.log(`[TestRunner]   params: ${JSON.stringify(params)}`);
+        console.log(`[TestRunner]   metadata: ${JSON.stringify(metadata)}`);
+        console.log(`[TestRunner]   output (${compiled.length} chars): ${compiled.substring(0, 200)}`);
+        return { compiled, metadata };
+    }
+    /**
+     * Execute compiled prompt text against an LLM using the executor's callLLM directly.
+     * This avoids re-compilation through executeRawText which loses metadata.
+     */
+    async executePrompt(compiled, metadata, runOptions) {
+        await this.ensureConfig();
+        const cli = await this.getCli();
+        const executor = new cli.PrompdExecutor();
+        const start = Date.now();
+        // Resolve provider/model from frontmatter metadata + config defaults
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const configManager = cli.ConfigManager?.getInstance
+            ? cli.ConfigManager.getInstance()
+            : null;
+        const config = configManager?.config || {};
+        // Priority: .prmd frontmatter > test run options (UI selector) > config defaults
+        const provider = String(metadata.provider || runOptions.provider || config.defaultProvider || 'openai');
+        const rawModel = metadata.model || runOptions.model || config.default_model || config.defaultModel || '';
+        // Fall back to a sensible default model if none specified
+        const model = String(rawModel) || this.getDefaultModel(provider);
+        const temperature = Number(metadata.temperature ?? 0.7);
+        const maxTokens = Number(metadata.max_tokens ?? 4096);
+        // Get API key from config
+        const apiKey = configManager?.getApiKey?.(provider, config) || '';
+        console.log(`[TestRunner] Executing: provider=${provider}, model=${model || '(default)'}, tokens=${compiled.length}`);
+        if (!apiKey && provider !== 'ollama') {
+            throw new Error(`No API key configured for provider "${provider}". Check ~/.prompd/config.yaml`);
+        }
+        try {
+            const result = await executor.callLLM(provider, model, compiled, apiKey, temperature, maxTokens);
+            if (!result.success) {
+                throw new Error(result.error || 'LLM execution failed');
+            }
+            return {
+                response: result.response || result.content || '',
+                provider,
+                model,
+                duration: Date.now() - start,
+                usage: result.usage,
+            };
+        }
+        catch (err) {
+            const errMsg = err instanceof Error ? err.message : String(err);
+            console.error(`[TestRunner] callLLM failed: ${errMsg}`);
+            throw new Error(errMsg);
+        }
+    }
+    resolveAllowedEvaluators(options) {
+        if (options.noLlm) {
+            // In --no-llm mode, skip prmd evaluators (they require LLM calls)
+            const base = options.evaluators || ['nlp', 'script'];
+            return base.filter(e => e !== 'prmd');
+        }
+        return options.evaluators || ['nlp', 'script', 'prmd'];
+    }
+    getReporter(options) {
+        switch (options.reporter) {
+            case 'json':
+                return new JsonReporter_1.JsonReporter(options.verbose);
+            case 'junit':
+                return new JunitReporter_1.JunitReporter();
+            case 'console':
+            default:
+                return new ConsoleReporter_1.ConsoleReporter(options.verbose);
+        }
+    }
+    buildSummary(suiteResults, startTime) {
+        let total = 0;
+        let passed = 0;
+        let failed = 0;
+        let errors = 0;
+        let skipped = 0;
+        let totalTokens = 0;
+        const providerSet = new Set();
+        const modelSet = new Set();
+        for (const suite of suiteResults) {
+            for (const result of suite.results) {
+                total++;
+                switch (result.status) {
+                    case 'pass':
+                        passed++;
+                        break;
+                    case 'fail':
+                        failed++;
+                        break;
+                    case 'error':
+                        errors++;
+                        break;
+                    case 'skip':
+                        skipped++;
+                        break;
+                }
+                if (result.execution) {
+                    if (result.execution.provider && result.execution.provider !== 'none') {
+                        providerSet.add(result.execution.provider);
+                    }
+                    if (result.execution.model && result.execution.model !== 'none') {
+                        modelSet.add(result.execution.model);
+                    }
+                    if (result.execution.usage?.totalTokens) {
+                        totalTokens += result.execution.usage.totalTokens;
+                    }
+                }
+            }
+        }
+        return {
+            total,
+            passed,
+            failed,
+            errors,
+            skipped,
+            duration: Date.now() - startTime,
+            totalTokens: totalTokens || undefined,
+            providers: providerSet.size > 0 ? Array.from(providerSet) : undefined,
+            models: modelSet.size > 0 ? Array.from(modelSet) : undefined,
+        };
+    }
+    buildErrorResult(errorMessages, startTime) {
+        return {
+            suites: [],
+            summary: {
+                total: 0,
+                passed: 0,
+                failed: 0,
+                errors: errorMessages.length,
+                skipped: 0,
+                duration: Date.now() - startTime,
+            },
+        };
+    }
+    getDefaultModel(provider) {
+        const defaults = {
+            openai: 'gpt-4o',
+            anthropic: 'claude-sonnet-4-20250514',
+            groq: 'llama-3.1-70b-versatile',
+            google: 'gemini-2.0-flash',
+            mistral: 'mistral-large-latest',
+            deepseek: 'deepseek-chat',
+        };
+        return defaults[provider.toLowerCase()] || 'gpt-4o';
+    }
+    async getCli() {
+        if (!this.cliModule) {
+            throw new Error('@prompd/cli module not provided. Pass it to the TestRunner constructor: new TestRunner(cliModule)');
+        }
+        return this.cliModule;
+    }
+}
+exports.TestRunner = TestRunner;

package/dist/cli-types.d.ts

@@ -0,0 +1,62 @@
+/**
+ * Shared type interfaces for dynamically imported @prompd/cli module.
+ * These mirror the CLI's public API without requiring a compile-time dependency.
+ */
+export interface CompilerModule {
+    PrompdCompiler: new (config?: Record<string, unknown>) => Compiler;
+    PrompdExecutor: new () => Executor;
+    ConfigManager: new () => ConfigManagerInstance;
+    MemoryFileSystem: new (files?: Record<string, string>) => MemoryFileSystemInstance;
+    NodeFileSystem: new () => NodeFileSystemInstance;
+}
+export interface Compiler {
+    compile(sourcePath: string, options: Record<string, unknown>): Promise<string | {
+        output?: string;
+        error?: string;
+        metadata?: Record<string, unknown>;
+    }>;
+    compileWithContext(sourcePath: string, options: Record<string, unknown>): Promise<string | {
+        compiledResult?: string;
+        metadata?: Record<string, unknown>;
+        errors?: unknown[];
+        warnings?: unknown[];
+    }>;
+}
+export interface Executor {
+    execute(filePath: string, options: Record<string, unknown>): Promise<ExecutorResult>;
+    executeRawText(compiledText: string, options: Record<string, unknown>): Promise<ExecutorResult>;
+    callLLM(provider: string, model: string, content: string, apiKey: string, temperature?: number, maxTokens?: number): Promise<{
+        success: boolean;
+        response?: string;
+        content?: string;
+        error?: string;
+        usage?: {
+            promptTokens?: number;
+            completionTokens?: number;
+            totalTokens?: number;
+        };
+    }>;
+}
+export interface ExecutorResult {
+    response?: string;
+    error?: string;
+    usage?: {
+        promptTokens?: number;
+        completionTokens?: number;
+        totalTokens?: number;
+    };
+    metadata?: {
+        provider?: string;
+        model?: string;
+    };
+}
+export interface MemoryFileSystemInstance {
+}
+export interface NodeFileSystemInstance {
+}
+export interface ConfigManagerInstance {
+    loadConfig?(): void;
+    load?(): void;
+    getConfig?(): Record<string, unknown> | null;
+}
+//# sourceMappingURL=cli-types.d.ts.map

package/dist/cli-types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli-types.d.ts","sourceRoot":"","sources":["../src/cli-types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,MAAM,WAAW,cAAc;IAC7B,cAAc,EAAE,KAAK,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,QAAQ,CAAC;IACnE,cAAc,EAAE,UAAU,QAAQ,CAAC;IACnC,aAAa,EAAE,UAAU,qBAAqB,CAAC;IAC/C,gBAAgB,EAAE,KAAK,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,wBAAwB,CAAC;IACnF,cAAc,EAAE,UAAU,sBAAsB,CAAC;CAClD;AAED,MAAM,WAAW,QAAQ;IACvB,OAAO,CACL,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC/B,OAAO,CAAC,MAAM,GAAG;QAClB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACpC,CAAC,CAAC;IAEH,kBAAkB,CAChB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC/B,OAAO,CAAC,MAAM,GAAG;QAClB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACnC,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,QAAQ,CAAC,EAAE,OAAO,EAAE,CAAC;KACtB,CAAC,CAAC;CACJ;AAED,MAAM,WAAW,QAAQ;IACvB,OAAO,CACL,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC/B,OAAO,CAAC,cAAc,CAAC,CAAC;IAE3B,cAAc,CACZ,YAAY,EAAE,MAAM,EACpB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC/B,OAAO,CAAC,cAAc,CAAC,CAAC;IAE3B,OAAO,CACL,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,MAAM,EACd,WAAW,CAAC,EAAE,MAAM,EACpB,SAAS,CAAC,EAAE,MAAM,GACjB,OAAO,CAAC;QACT,OAAO,EAAE,OAAO,CAAC;QACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,KAAK,CAAC,EAAE;YACN,YAAY,CAAC,EAAE,MAAM,CAAC;YACtB,gBAAgB,CAAC,EAAE,MAAM,CAAC;YAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;SACtB,CAAC;KACH,CAAC,CAAC;CACJ;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE;QACN,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB,CAAC;IACF,QAAQ,CAAC,EAAE;QACT,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,CAAC;CACH;AAED,MAAM,WAAW,wBAAwB;CAExC;AAED,MAAM,WAAW,sBAAsB;CAEtC;AAED,MAAM,WAAW,qBAAqB;IACpC,UAAU,CAAC,IAAI,IAAI,CAAC;IACpB,IAAI,CAAC,IAAI,IAAI,CAAC;IACd,SAAS,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;CAC9C"}

package/dist/cli-types.js

@@ -0,0 +1,6 @@
+"use strict";
+/**
+ * Shared type interfaces for dynamically imported @prompd/cli module.
+ * These mirror the CLI's public API without requiring a compile-time dependency.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/evaluators/NlpEvaluator.d.ts

@@ -0,0 +1,26 @@
+/**
+ * NLP Evaluator - local, fast, free, deterministic assertions.
+ *
+ * Checks: contains, not_contains, matches, max_tokens, min_tokens, starts_with, ends_with
+ */
+import type { Evaluator, EvaluatorContext } from './types';
+import type { AssertionDef, AssertionResult } from '../types';
+export declare class NlpEvaluator implements Evaluator {
+    readonly type = "nlp";
+    evaluate(assertion: AssertionDef, context: EvaluatorContext): Promise<AssertionResult>;
+    private runCheck;
+    private checkContains;
+    private checkNotContains;
+    private checkMatches;
+    private checkMaxTokens;
+    private checkMinTokens;
+    private checkStartsWith;
+    private checkEndsWith;
+    /**
+     * Rough token estimation: ~4 characters per token (GPT-family average).
+     * This is intentionally approximate — for precise counting, use a tokenizer.
+     */
+    private estimateTokens;
+    private toStringArray;
+}
+//# sourceMappingURL=NlpEvaluator.d.ts.map

package/dist/evaluators/NlpEvaluator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"NlpEvaluator.d.ts","sourceRoot":"","sources":["../../src/evaluators/NlpEvaluator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAC3D,OAAO,KAAK,EAAE,YAAY,EAAE,eAAe,EAAY,MAAM,UAAU,CAAC;AAExE,qBAAa,YAAa,YAAW,SAAS;IAC5C,QAAQ,CAAC,IAAI,SAAS;IAEhB,QAAQ,CAAC,SAAS,EAAE,YAAY,EAAE,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,eAAe,CAAC;IAwB5F,OAAO,CAAC,QAAQ;IAyBhB,OAAO,CAAC,aAAa;IAiBrB,OAAO,CAAC,gBAAgB;IAiBxB,OAAO,CAAC,YAAY;IAepB,OAAO,CAAC,cAAc;IAetB,OAAO,CAAC,cAAc;IAetB,OAAO,CAAC,eAAe;IAevB,OAAO,CAAC,aAAa;IAerB;;;OAGG;IACH,OAAO,CAAC,cAAc;IAItB,OAAO,CAAC,aAAa;CAKtB"}