@prompd/test 0.5.0-beta.10

package/dist/evaluators/NlpEvaluator.js

@@ -0,0 +1,183 @@
+"use strict";
+/**
+ * NLP Evaluator - local, fast, free, deterministic assertions.
+ *
+ * Checks: contains, not_contains, matches, max_tokens, min_tokens, starts_with, ends_with
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.NlpEvaluator = void 0;
+class NlpEvaluator {
+    constructor() {
+        this.type = 'nlp';
+    }
+    async evaluate(assertion, context) {
+        const start = Date.now();
+        const check = assertion.check;
+        const target = assertion.evaluate || 'response';
+        try {
+            const text = this.resolveTarget(target, context);
+            const targetLabel = target === 'both' ? 'Prompt+Response' : target === 'prompt' ? 'Prompt' : 'Output';
+            const result = this.runCheck(check, assertion.value, text, targetLabel);
+            return {
+                evaluator: 'nlp',
+                check,
+                status: result.pass ? 'pass' : 'fail',
+                reason: result.reason,
+                duration: Date.now() - start,
+            };
+        }
+        catch (err) {
+            return {
+                evaluator: 'nlp',
+                check,
+                status: 'error',
+                reason: err instanceof Error ? err.message : String(err),
+                duration: Date.now() - start,
+            };
+        }
+    }
+    resolveTarget(target, context) {
+        switch (target) {
+            case 'prompt': return context.prompt;
+            case 'both': return `${context.prompt}\n\n${context.response}`;
+            case 'response':
+            default: return context.response;
+        }
+    }
+    runCheck(check, value, output, label = 'Output') {
+        switch (check) {
+            case 'contains':
+                return this.checkContains(value, output, label);
+            case 'not_contains':
+                return this.checkNotContains(value, output, label);
+            case 'matches':
+                return this.checkMatches(value, output, label);
+            case 'max_tokens':
+                return this.checkMaxTokens(value, output);
+            case 'min_tokens':
+                return this.checkMinTokens(value, output);
+            case 'max_words':
+                return this.checkMaxWords(value, output);
+            case 'min_words':
+                return this.checkMinWords(value, output);
+            case 'starts_with':
+                return this.checkStartsWith(value, output, label);
+            case 'ends_with':
+                return this.checkEndsWith(value, output, label);
+            default:
+                return { pass: false, reason: `Unknown NLP check: ${check}` };
+        }
+    }
+    checkContains(value, output, label) {
+        const values = this.toStringArray(value);
+        const lower = output.toLowerCase();
+        const missing = values.filter(v => !lower.includes(v.toLowerCase()));
+        if (missing.length === 0) {
+            return { pass: true, reason: `${label} contains all expected values` };
+        }
+        return {
+            pass: false,
+            reason: `${label} missing: ${missing.map(v => `"${v}"`).join(', ')}`,
+        };
+    }
+    checkNotContains(value, output, label) {
+        const values = this.toStringArray(value);
+        const lower = output.toLowerCase();
+        const found = values.filter(v => lower.includes(v.toLowerCase()));
+        if (found.length === 0) {
+            return { pass: true, reason: `${label} does not contain any excluded values` };
+        }
+        return {
+            pass: false,
+            reason: `${label} contains excluded values: ${found.map(v => `"${v}"`).join(', ')}`,
+        };
+    }
+    checkMatches(value, output, label) {
+        if (typeof value !== 'string') {
+            return { pass: false, reason: '"matches" check requires a string regex pattern' };
+        }
+        const regex = new RegExp(value);
+        if (regex.test(output)) {
+            return { pass: true, reason: `${label} matches pattern /${value}/` };
+        }
+        return { pass: false, reason: `${label} does not match pattern /${value}/` };
+    }
+    checkMaxTokens(value, output) {
+        if (typeof value !== 'number') {
+            return { pass: false, reason: '"max_tokens" check requires a numeric value' };
+        }
+        const tokenCount = this.estimateTokens(output);
+        if (tokenCount <= value) {
+            return { pass: true, reason: `Token count ${tokenCount} <= ${value}` };
+        }
+        return { pass: false, reason: `Token count ${tokenCount} exceeds max ${value}` };
+    }
+    checkMinTokens(value, output) {
+        if (typeof value !== 'number') {
+            return { pass: false, reason: '"min_tokens" check requires a numeric value' };
+        }
+        const tokenCount = this.estimateTokens(output);
+        if (tokenCount >= value) {
+            return { pass: true, reason: `Token count ${tokenCount} >= ${value}` };
+        }
+        return { pass: false, reason: `Token count ${tokenCount} below min ${value}` };
+    }
+    checkStartsWith(value, output, label) {
+        if (typeof value !== 'string') {
+            return { pass: false, reason: '"starts_with" check requires a string value' };
+        }
+        const trimmed = output.trimStart();
+        if (trimmed.toLowerCase().startsWith(value.toLowerCase())) {
+            return { pass: true, reason: `${label} starts with "${value}"` };
+        }
+        return { pass: false, reason: `${label} does not start with "${value}"` };
+    }
+    checkEndsWith(value, output, label) {
+        if (typeof value !== 'string') {
+            return { pass: false, reason: '"ends_with" check requires a string value' };
+        }
+        const trimmed = output.trimEnd();
+        if (trimmed.toLowerCase().endsWith(value.toLowerCase())) {
+            return { pass: true, reason: `${label} ends with "${value}"` };
+        }
+        return { pass: false, reason: `${label} does not end with "${value}"` };
+    }
+    checkMaxWords(value, output) {
+        if (typeof value !== 'number') {
+            return { pass: false, reason: '"max_words" check requires a numeric value' };
+        }
+        const wordCount = this.countWords(output);
+        if (wordCount <= value) {
+            return { pass: true, reason: `Word count ${wordCount} <= ${value}` };
+        }
+        return { pass: false, reason: `Word count ${wordCount} exceeds max ${value}` };
+    }
+    checkMinWords(value, output) {
+        if (typeof value !== 'number') {
+            return { pass: false, reason: '"min_words" check requires a numeric value' };
+        }
+        const wordCount = this.countWords(output);
+        if (wordCount >= value) {
+            return { pass: true, reason: `Word count ${wordCount} >= ${value}` };
+        }
+        return { pass: false, reason: `Word count ${wordCount} below min ${value}` };
+    }
+    countWords(text) {
+        return text.trim().split(/\s+/).filter(w => w.length > 0).length;
+    }
+    /**
+     * Rough token estimation: ~4 characters per token (GPT-family average).
+     * This is intentionally approximate — for precise counting, use a tokenizer.
+     */
+    estimateTokens(text) {
+        return Math.ceil(text.length / 4);
+    }
+    toStringArray(value) {
+        if (value === undefined || value === null)
+            return [];
+        if (Array.isArray(value))
+            return value.map(String);
+        return [String(value)];
+    }
+}
+exports.NlpEvaluator = NlpEvaluator;

package/dist/evaluators/PrmdEvaluator.d.ts

@@ -0,0 +1,42 @@
+/**
+ * Prmd Evaluator - LLM-based evaluation via @prompd/cli.
+ *
+ * Modes:
+ * - prompt: "@scope/pkg@version" -> uses a registry package as the evaluator
+ * - prompt: "./path" -> uses a local .prmd file as the evaluator
+ * - (no prompt field) -> uses the content block of the .test.prmd
+ *
+ * The evaluator prompt receives {{input}}, {{output}}, and {{params}} variables.
+ * Response must start with PASS or FAIL.
+ */
+import type { Evaluator, EvaluatorContext } from './types';
+import type { AssertionDef, AssertionResult } from '../types';
+import type { CompilerModule } from '../cli-types';
+export interface PrmdEvaluatorOptions {
+    testFileDir: string;
+    evaluatorPrompt?: string;
+    workspaceRoot?: string;
+    registryUrl?: string;
+    cliModule?: CompilerModule;
+    provider?: string;
+    model?: string;
+}
+export declare class PrmdEvaluator implements Evaluator {
+    readonly type = "prmd";
+    private options;
+    private cliModule;
+    constructor(options: PrmdEvaluatorOptions);
+    evaluate(assertion: AssertionDef, context: EvaluatorContext): Promise<AssertionResult>;
+    private resolveEvaluatorContent;
+    private resolvePromptTarget;
+    /**
+     * Wrap a registry reference as a minimal .prmd that inherits from the evaluator package.
+     * The compiler handles resolution, download, and caching.
+     */
+    private wrapAsInherits;
+    private compileEvaluator;
+    private parseEvaluatorResponse;
+    private getDefaultModel;
+    private getCli;
+}
+//# sourceMappingURL=PrmdEvaluator.d.ts.map

package/dist/evaluators/PrmdEvaluator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"PrmdEvaluator.d.ts","sourceRoot":"","sources":["../../src/evaluators/PrmdEvaluator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAIH,OAAO,KAAK,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAC3D,OAAO,KAAK,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAC9D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAInD,MAAM,WAAW,oBAAoB;IACnC,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,cAAc,CAAC;IAC3B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,qBAAa,aAAc,YAAW,SAAS;IAC7C,QAAQ,CAAC,IAAI,UAAU;IACvB,OAAO,CAAC,OAAO,CAAuB;IACtC,OAAO,CAAC,SAAS,CAA+B;gBAEpC,OAAO,EAAE,oBAAoB;IAOnC,QAAQ,CAAC,SAAS,EAAE,YAAY,EAAE,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,eAAe,CAAC;YA+F9E,uBAAuB;YAUvB,mBAAmB;IAejC;;;OAGG;IACH,OAAO,CAAC,cAAc;YAgBR,gBAAgB;IAyD9B,OAAO,CAAC,sBAAsB;IAwB9B,OAAO,CAAC,eAAe;YAYT,MAAM;CAQrB"}

package/dist/evaluators/PrmdEvaluator.js

@@ -0,0 +1,265 @@
+"use strict";
+/**
+ * Prmd Evaluator - LLM-based evaluation via @prompd/cli.
+ *
+ * Modes:
+ * - prompt: "@scope/pkg@version" -> uses a registry package as the evaluator
+ * - prompt: "./path" -> uses a local .prmd file as the evaluator
+ * - (no prompt field) -> uses the content block of the .test.prmd
+ *
+ * The evaluator prompt receives {{input}}, {{output}}, and {{params}} variables.
+ * Response must start with PASS or FAIL.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PrmdEvaluator = void 0;
+const path = __importStar(require("path"));
+const fs = __importStar(require("fs"));
+const PASS_FAIL_REGEX = /^(PASS|FAIL)[:\s]*(.*)/i;
+class PrmdEvaluator {
+    constructor(options) {
+        this.type = 'prmd';
+        this.cliModule = null;
+        this.options = options;
+        if (options.cliModule) {
+            this.cliModule = options.cliModule;
+        }
+    }
+    async evaluate(assertion, context) {
+        const start = Date.now();
+        try {
+            const evaluatorContent = await this.resolveEvaluatorContent(assertion);
+            console.log(`[PrmdEvaluator] Resolved evaluator content (${evaluatorContent?.length || 0} chars)`);
+            if (evaluatorContent) {
+                console.log(`[PrmdEvaluator]   source: ${assertion.prompt || 'content block'}`);
+                console.log(`[PrmdEvaluator]   preview: ${evaluatorContent.substring(0, 150)}`);
+            }
+            if (!evaluatorContent) {
+                return {
+                    evaluator: 'prmd',
+                    status: 'error',
+                    reason: 'Could not resolve evaluator prompt content',
+                    duration: Date.now() - start,
+                };
+            }
+            // Compile the evaluator prompt with context as parameters
+            const cli = await this.getCli();
+            const compiled = await this.compileEvaluator(cli, evaluatorContent, context);
+            console.log(`[PrmdEvaluator] Compiled evaluator (${compiled?.length || 0} chars): ${compiled?.substring(0, 150) || 'null'}`);
+            if (!compiled) {
+                return {
+                    evaluator: 'prmd',
+                    status: 'error',
+                    reason: 'Evaluator prompt compilation failed',
+                    duration: Date.now() - start,
+                };
+            }
+            // Execute against LLM using callLLM directly (avoids executeRawText re-compilation)
+            const executor = new cli.PrompdExecutor();
+            // Resolve provider/model/apiKey — same logic as TestRunner
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            const configManager = cli.ConfigManager?.getInstance
+                ? cli.ConfigManager.getInstance()
+                : null;
+            const config = configManager?.config || {};
+            // Priority: assertion-level > run options (UI selector) > config defaults
+            const provider = assertion.provider || this.options.provider || config.defaultProvider || 'openai';
+            const rawModel = assertion.model || this.options.model || config.default_model || config.defaultModel || '';
+            const model = rawModel || this.getDefaultModel(provider);
+            const apiKey = configManager?.getApiKey?.(provider, config) || '';
+            console.log(`[PrmdEvaluator] Executing: provider=${provider}, model=${model}`);
+            if (!apiKey && provider !== 'ollama') {
+                return {
+                    evaluator: 'prmd',
+                    status: 'error',
+                    reason: `No API key configured for provider "${provider}"`,
+                    duration: Date.now() - start,
+                };
+            }
+            const execResult = await executor.callLLM(provider, model, compiled, apiKey);
+            if (!execResult.success) {
+                return {
+                    evaluator: 'prmd',
+                    status: 'error',
+                    reason: execResult.error || 'Evaluator LLM execution failed',
+                    duration: Date.now() - start,
+                };
+            }
+            const response = execResult.response || execResult.content || '';
+            if (!response) {
+                return {
+                    evaluator: 'prmd',
+                    status: 'error',
+                    reason: 'No response from evaluator',
+                    duration: Date.now() - start,
+                };
+            }
+            // Parse PASS/FAIL from response
+            return this.parseEvaluatorResponse(response, Date.now() - start);
+        }
+        catch (err) {
+            return {
+                evaluator: 'prmd',
+                status: 'error',
+                reason: err instanceof Error ? err.message : String(err),
+                duration: Date.now() - start,
+            };
+        }
+    }
+    async resolveEvaluatorContent(assertion) {
+        // If prompt: is specified, resolve it (registry ref, local file)
+        if (assertion.prompt) {
+            return this.resolvePromptTarget(assertion.prompt);
+        }
+        // No prompt: field — use the content block of the .test.prmd
+        return this.options.evaluatorPrompt || null;
+    }
+    async resolvePromptTarget(prompt) {
+        // Registry reference: @scope/package@version
+        if (prompt.startsWith('@')) {
+            return this.wrapAsInherits(prompt);
+        }
+        // Local file path
+        const resolved = path.resolve(this.options.testFileDir, prompt);
+        if (!fs.existsSync(resolved)) {
+            throw new Error(`Evaluator prompt file not found: ${resolved}`);
+        }
+        return fs.readFileSync(resolved, 'utf-8');
+    }
+    /**
+     * Wrap a registry reference as a minimal .prmd that inherits from the evaluator package.
+     * The compiler handles resolution, download, and caching.
+     */
+    wrapAsInherits(registryRef) {
+        return [
+            '---',
+            `inherits: "${registryRef}"`,
+            'parameters:',
+            '  - name: prompt',
+            '    type: string',
+            '  - name: response',
+            '    type: string',
+            '  - name: params',
+            '    type: string',
+            '---',
+            '',
+        ].join('\n');
+    }
+    async compileEvaluator(cli, content, context) {
+        // If content doesn't start with frontmatter, wrap it with minimal frontmatter
+        // so the compiler can process it. Content blocks from .test.prmd are raw markdown.
+        let prmdContent = content;
+        if (!content.trimStart().startsWith('---')) {
+            prmdContent = [
+                '---',
+                'id: evaluator',
+                'name: "Test Evaluator"',
+                'version: 0.0.1',
+                'parameters:',
+                '  - name: prompt',
+                '    type: string',
+                '  - name: response',
+                '    type: string',
+                '  - name: params',
+                '    type: object',
+                '---',
+                '',
+                content,
+            ].join('\n');
+        }
+        const memFs = new cli.MemoryFileSystem({ '/evaluator.prmd': prmdContent });
+        const compiler = new cli.PrompdCompiler();
+        // Inject evaluation context as template variables
+        const parameters = {
+            prompt: context.prompt,
+            response: context.response,
+            params: JSON.stringify(context.params, null, 2),
+        };
+        // Also expose individual params via dot notation
+        for (const [key, value] of Object.entries(context.params)) {
+            parameters[`params.${key}`] = String(value);
+        }
+        const result = await compiler.compile('/evaluator.prmd', {
+            outputFormat: 'markdown',
+            parameters,
+            fileSystem: memFs,
+            workspaceRoot: this.options.workspaceRoot,
+            registryUrl: this.options.registryUrl,
+        });
+        // CLI compile() may return a string directly or an object
+        if (typeof result === 'string') {
+            return result || null;
+        }
+        return result.output || null;
+    }
+    parseEvaluatorResponse(response, duration) {
+        const firstLine = response.trim().split('\n')[0];
+        const match = firstLine.match(PASS_FAIL_REGEX);
+        if (!match) {
+            return {
+                evaluator: 'prmd',
+                status: 'error',
+                reason: `Evaluator response did not start with PASS or FAIL. Got: "${firstLine.substring(0, 100)}"`,
+                duration,
+            };
+        }
+        const verdict = match[1].toUpperCase();
+        const reason = match[2]?.trim() || undefined;
+        return {
+            evaluator: 'prmd',
+            status: verdict === 'PASS' ? 'pass' : 'fail',
+            reason: reason || `Evaluator returned ${verdict}`,
+            duration,
+        };
+    }
+    getDefaultModel(provider) {
+        const defaults = {
+            openai: 'gpt-4o',
+            anthropic: 'claude-sonnet-4-20250514',
+            groq: 'llama-3.1-70b-versatile',
+            google: 'gemini-2.0-flash',
+            mistral: 'mistral-large-latest',
+            deepseek: 'deepseek-chat',
+        };
+        return defaults[provider.toLowerCase()] || 'gpt-4o';
+    }
+    async getCli() {
+        if (!this.cliModule) {
+            throw new Error('@prompd/cli module not provided. Pass it via PrmdEvaluatorOptions.cliModule');
+        }
+        return this.cliModule;
+    }
+}
+exports.PrmdEvaluator = PrmdEvaluator;

package/dist/evaluators/ScriptEvaluator.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Script Evaluator - runs external scripts with stdin/stdout contract.
+ *
+ * Contract:
+ * - Receives JSON on stdin: { input, output, params, metadata }
+ * - Exit code 0 = PASS, 1 = FAIL, other = ERROR
+ * - Stdout = reason (optional)
+ */
+import type { Evaluator, EvaluatorContext } from './types';
+import type { AssertionDef, AssertionResult } from '../types';
+export declare class ScriptEvaluator implements Evaluator {
+    readonly type = "script";
+    private testFileDir;
+    constructor(testFileDir: string);
+    evaluate(assertion: AssertionDef, context: EvaluatorContext): Promise<AssertionResult>;
+    private runScript;
+    private getRunner;
+}
+//# sourceMappingURL=ScriptEvaluator.d.ts.map

package/dist/evaluators/ScriptEvaluator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ScriptEvaluator.d.ts","sourceRoot":"","sources":["../../src/evaluators/ScriptEvaluator.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAKH,OAAO,KAAK,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAC3D,OAAO,KAAK,EAAE,YAAY,EAAE,eAAe,EAAkB,MAAM,UAAU,CAAC;AAI9E,qBAAa,eAAgB,YAAW,SAAS;IAC/C,QAAQ,CAAC,IAAI,YAAY;IACzB,OAAO,CAAC,WAAW,CAAS;gBAEhB,WAAW,EAAE,MAAM;IAIzB,QAAQ,CAAC,SAAS,EAAE,YAAY,EAAE,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,eAAe,CAAC;IAsD5F,OAAO,CAAC,SAAS;IAoDjB,OAAO,CAAC,SAAS;CAoBlB"}

package/dist/evaluators/ScriptEvaluator.js

@@ -0,0 +1,163 @@
+"use strict";
+/**
+ * Script Evaluator - runs external scripts with stdin/stdout contract.
+ *
+ * Contract:
+ * - Receives JSON on stdin: { input, output, params, metadata }
+ * - Exit code 0 = PASS, 1 = FAIL, other = ERROR
+ * - Stdout = reason (optional)
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ScriptEvaluator = void 0;
+const child_process_1 = require("child_process");
+const path = __importStar(require("path"));
+const fs = __importStar(require("fs"));
+const SCRIPT_TIMEOUT_MS = 30000;
+class ScriptEvaluator {
+    constructor(testFileDir) {
+        this.type = 'script';
+        this.testFileDir = testFileDir;
+    }
+    async evaluate(assertion, context) {
+        const start = Date.now();
+        const scriptPath = assertion.run;
+        if (!scriptPath) {
+            return {
+                evaluator: 'script',
+                status: 'error',
+                reason: 'No "run" path specified for script evaluator',
+                duration: Date.now() - start,
+            };
+        }
+        const resolvedPath = path.resolve(this.testFileDir, scriptPath);
+        if (!fs.existsSync(resolvedPath)) {
+            return {
+                evaluator: 'script',
+                status: 'error',
+                reason: `Script not found: ${resolvedPath}`,
+                duration: Date.now() - start,
+            };
+        }
+        // Validate script stays within the test file's directory tree
+        const normalizedScript = path.normalize(resolvedPath);
+        const normalizedBase = path.normalize(this.testFileDir);
+        if (!normalizedScript.startsWith(normalizedBase)) {
+            return {
+                evaluator: 'script',
+                status: 'error',
+                reason: `Script path escapes test directory: ${scriptPath}`,
+                duration: Date.now() - start,
+            };
+        }
+        try {
+            const result = await this.runScript(resolvedPath, context, assertion);
+            return {
+                evaluator: 'script',
+                status: result.exitCode === 0 ? 'pass' : 'fail',
+                reason: result.stdout.trim() || (result.exitCode === 0 ? 'Script passed' : 'Script failed'),
+                duration: Date.now() - start,
+            };
+        }
+        catch (err) {
+            return {
+                evaluator: 'script',
+                status: 'error',
+                reason: err instanceof Error ? err.message : String(err),
+                duration: Date.now() - start,
+            };
+        }
+    }
+    runScript(scriptPath, context, assertion) {
+        return new Promise((resolve, reject) => {
+            const { command, args } = this.getRunner(scriptPath);
+            const child = (0, child_process_1.spawn)(command, args, {
+                cwd: this.testFileDir,
+                timeout: SCRIPT_TIMEOUT_MS,
+                stdio: ['pipe', 'pipe', 'pipe'],
+                shell: process.platform === 'win32',
+            });
+            let stdout = '';
+            let stderr = '';
+            child.stdout.on('data', (data) => {
+                stdout += data.toString();
+            });
+            child.stderr.on('data', (data) => {
+                stderr += data.toString();
+            });
+            child.on('error', (err) => {
+                reject(new Error(`Failed to spawn script: ${err.message}`));
+            });
+            child.on('close', (code) => {
+                if (code === null) {
+                    reject(new Error('Script process was killed (timeout or signal)'));
+                    return;
+                }
+                resolve({ exitCode: code, stdout, stderr });
+            });
+            // Send context as JSON on stdin, include target so script knows what to evaluate
+            const target = assertion.evaluate || 'response';
+            const payload = JSON.stringify({
+                target,
+                prompt: context.prompt,
+                response: context.response,
+                params: context.params,
+                metadata: context.metadata,
+            });
+            child.stdin.write(payload);
+            child.stdin.end();
+        });
+    }
+    getRunner(scriptPath) {
+        const ext = path.extname(scriptPath).toLowerCase();
+        switch (ext) {
+            case '.ts':
+                return { command: 'npx', args: ['tsx', scriptPath] };
+            case '.js':
+            case '.mjs':
+                return { command: 'node', args: [scriptPath] };
+            case '.py':
+                return { command: 'python', args: [scriptPath] };
+            case '.sh':
+                return { command: 'bash', args: [scriptPath] };
+            case '.ps1':
+                return { command: 'powershell', args: ['-File', scriptPath] };
+            default:
+                // For unknown extensions, try running directly (relies on shebang or OS association)
+                return { command: scriptPath, args: [] };
+        }
+    }
+}
+exports.ScriptEvaluator = ScriptEvaluator;