@chanl/eval-cli 0.4.0

package/dist/assertions.js

@@ -0,0 +1,282 @@
+"use strict";
+/**
+ * Assertion engine for `chanl test` — evaluates pass/fail assertions
+ * against scenario execution results.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.extractTranscriptText = extractTranscriptText;
+exports.extractAvgResponseTimeSecs = extractAvgResponseTimeSecs;
+exports.extractToolCalls = extractToolCalls;
+exports.extractOverallScore = extractOverallScore;
+exports.evaluateAssertions = evaluateAssertions;
+/* ------------------------------------------------------------------ */
+/* Transcript helpers                                                  */
+/* ------------------------------------------------------------------ */
+/**
+ * Extract the full transcript text from an execution's stepResults.
+ * Concatenates all actualResponse fields.
+ */
+function extractTranscriptText(execution) {
+    if (!execution.stepResults || !Array.isArray(execution.stepResults)) {
+        return '';
+    }
+    return execution.stepResults
+        .map((step) => step.actualResponse || '')
+        .filter(Boolean)
+        .join('\n');
+}
+/**
+ * Extract average agent response time in seconds from execution data.
+ * Checks execution.metrics first, then computes from stepResults durations.
+ */
+function extractAvgResponseTimeSecs(execution) {
+    // Check for pre-computed latency in metrics
+    if (execution.metrics?.avgResponseTimeMs) {
+        return execution.metrics.avgResponseTimeMs / 1000;
+    }
+    // Compute from stepResults agent durations
+    if (execution.stepResults && Array.isArray(execution.stepResults)) {
+        const agentSteps = execution.stepResults.filter((s) => typeof s.stepId === 'string' &&
+            s.stepId.includes('agent') &&
+            s.duration > 0);
+        if (agentSteps.length > 0) {
+            const total = agentSteps.reduce((sum, s) => sum + s.duration, 0);
+            return total / agentSteps.length / 1000;
+        }
+    }
+    // Fall back to overall duration if available
+    if (execution.duration) {
+        return execution.duration / 1000;
+    }
+    return null;
+}
+/**
+ * Extract tool names mentioned in an execution.
+ * Checks stepResults metadata and falls back to transcript keyword search.
+ */
+function extractToolCalls(execution) {
+    const tools = new Set();
+    if (execution.stepResults && Array.isArray(execution.stepResults)) {
+        for (const step of execution.stepResults) {
+            // Check step metadata for tool call info
+            if (step.toolCalls && Array.isArray(step.toolCalls)) {
+                for (const tc of step.toolCalls) {
+                    if (tc.name)
+                        tools.add(tc.name);
+                }
+            }
+            if (step.metadata?.toolCalls && Array.isArray(step.metadata.toolCalls)) {
+                for (const tc of step.metadata.toolCalls) {
+                    if (tc.name)
+                        tools.add(tc.name);
+                }
+            }
+        }
+    }
+    // Check execution-level metrics
+    if (execution.metrics?.toolCalls && Array.isArray(execution.metrics.toolCalls)) {
+        for (const tc of execution.metrics.toolCalls) {
+            if (typeof tc === 'string')
+                tools.add(tc);
+            else if (tc.name)
+                tools.add(tc.name);
+        }
+    }
+    return Array.from(tools);
+}
+/**
+ * Extract the overall score from an execution, if present.
+ */
+function extractOverallScore(execution) {
+    if (typeof execution.overallScore === 'number') {
+        return execution.overallScore;
+    }
+    if (typeof execution.metadata?.scorecardResult?.overallScore === 'number') {
+        return execution.metadata.scorecardResult.overallScore;
+    }
+    return null;
+}
+/* ------------------------------------------------------------------ */
+/* Assertion evaluators                                                */
+/* ------------------------------------------------------------------ */
+function evaluateKeyword(assertion, execution) {
+    const transcript = extractTranscriptText(execution).toLowerCase();
+    const description = assertion.description || 'Keyword check';
+    if (assertion.must_include && assertion.must_include.length > 0) {
+        const missing = assertion.must_include.filter((kw) => !transcript.includes(kw.toLowerCase()));
+        if (missing.length > 0) {
+            return {
+                type: 'keyword',
+                description,
+                passed: false,
+                actual: `missing: ${missing.join(', ')}`,
+                expected: `must include: ${assertion.must_include.join(', ')}`,
+                reason: `Keywords not found in transcript: ${missing.join(', ')}`,
+            };
+        }
+    }
+    if (assertion.must_not_include && assertion.must_not_include.length > 0) {
+        const found = assertion.must_not_include.filter((kw) => transcript.includes(kw.toLowerCase()));
+        if (found.length > 0) {
+            return {
+                type: 'keyword',
+                description,
+                passed: false,
+                actual: `found: ${found.join(', ')}`,
+                expected: `must not include: ${assertion.must_not_include.join(', ')}`,
+                reason: `Forbidden keywords found in transcript: ${found.join(', ')}`,
+            };
+        }
+    }
+    return {
+        type: 'keyword',
+        description,
+        passed: true,
+        actual: 'all keywords matched',
+        expected: formatKeywordExpected(assertion),
+        reason: 'All keyword checks passed',
+    };
+}
+function formatKeywordExpected(assertion) {
+    const parts = [];
+    if (assertion.must_include?.length) {
+        parts.push(`include: ${assertion.must_include.join(', ')}`);
+    }
+    if (assertion.must_not_include?.length) {
+        parts.push(`exclude: ${assertion.must_not_include.join(', ')}`);
+    }
+    return parts.join('; ');
+}
+function evaluateResponseTime(assertion, execution) {
+    const description = assertion.description || 'Response time check';
+    const actualSecs = extractAvgResponseTimeSecs(execution);
+    if (actualSecs === null) {
+        return {
+            type: 'response_time',
+            description,
+            passed: false,
+            actual: 'no timing data',
+            expected: `<= ${assertion.max_seconds}s`,
+            reason: 'No response time data available in execution results',
+        };
+    }
+    const passed = actualSecs <= assertion.max_seconds;
+    return {
+        type: 'response_time',
+        description,
+        passed,
+        actual: `${actualSecs.toFixed(2)}s`,
+        expected: `<= ${assertion.max_seconds}s`,
+        reason: passed
+            ? `Response time ${actualSecs.toFixed(2)}s within limit`
+            : `Response time ${actualSecs.toFixed(2)}s exceeds ${assertion.max_seconds}s limit`,
+    };
+}
+function evaluatePrompt(assertion, execution) {
+    const description = assertion.description || 'Prompt-based check';
+    const score = extractOverallScore(execution);
+    if (score === null) {
+        return {
+            type: 'prompt',
+            description,
+            passed: false,
+            actual: 'no scorecard score',
+            expected: `>= ${assertion.min_score}/10`,
+            reason: 'No scorecard score available. Ensure the scenario has a scorecard configured.',
+        };
+    }
+    // Normalize: if score is 0-100, scale to 0-10 for comparison
+    const normalizedScore = score > 10 ? score / 10 : score;
+    const passed = normalizedScore >= assertion.min_score;
+    return {
+        type: 'prompt',
+        description,
+        passed,
+        actual: `${normalizedScore.toFixed(1)}/10`,
+        expected: `>= ${assertion.min_score}/10`,
+        reason: passed
+            ? `Score ${normalizedScore.toFixed(1)} meets minimum ${assertion.min_score}`
+            : `Score ${normalizedScore.toFixed(1)} below minimum ${assertion.min_score}`,
+    };
+}
+function evaluateToolCall(assertion, execution) {
+    const description = assertion.description || 'Tool call check';
+    const actualTools = extractToolCalls(execution);
+    // If no tools found via structured data, fall back to transcript search
+    let effectiveTools = actualTools;
+    if (actualTools.length === 0) {
+        const transcript = extractTranscriptText(execution).toLowerCase();
+        effectiveTools = assertion.expected.filter((tool) => transcript.includes(tool.toLowerCase()));
+    }
+    const missing = assertion.expected.filter((tool) => !effectiveTools.some((actual) => actual.toLowerCase() === tool.toLowerCase()));
+    const passed = missing.length === 0;
+    return {
+        type: 'tool_call',
+        description,
+        passed,
+        actual: effectiveTools.length > 0 ? effectiveTools.join(', ') : 'none',
+        expected: assertion.expected.join(', '),
+        reason: passed
+            ? `All expected tools were called: ${assertion.expected.join(', ')}`
+            : `Missing tool calls: ${missing.join(', ')}`,
+    };
+}
+function evaluateScore(assertion, execution) {
+    const description = assertion.description || 'Score check';
+    const score = extractOverallScore(execution);
+    if (score === null) {
+        return {
+            type: 'score',
+            description,
+            passed: false,
+            actual: 'no score',
+            expected: `>= ${assertion.min_score}`,
+            reason: 'No overall score available in execution results',
+        };
+    }
+    const passed = score >= assertion.min_score;
+    return {
+        type: 'score',
+        description,
+        passed,
+        actual: String(score),
+        expected: `>= ${assertion.min_score}`,
+        reason: passed
+            ? `Score ${score} meets minimum ${assertion.min_score}`
+            : `Score ${score} below minimum ${assertion.min_score}`,
+    };
+}
+/* ------------------------------------------------------------------ */
+/* Public API                                                          */
+/* ------------------------------------------------------------------ */
+/**
+ * Evaluate all assertions against an execution result.
+ */
+function evaluateAssertions(assertions, execution) {
+    return assertions.map((assertion) => {
+        switch (assertion.type) {
+            case 'keyword':
+                return evaluateKeyword(assertion, execution);
+            case 'response_time':
+                return evaluateResponseTime(assertion, execution);
+            case 'prompt':
+                return evaluatePrompt(assertion, execution);
+            case 'tool_call':
+                return evaluateToolCall(assertion, execution);
+            case 'score':
+                return evaluateScore(assertion, execution);
+            default: {
+                const unknown = assertion;
+                return {
+                    type: (typeof unknown.type === 'string' ? unknown.type : 'unknown'),
+                    description: (typeof unknown.description === 'string' ? unknown.description : 'Unknown assertion type'),
+                    passed: false,
+                    actual: 'N/A',
+                    expected: 'N/A',
+                    reason: `Unknown assertion type: ${typeof unknown.type === 'string' ? unknown.type : 'unknown'}`,
+                };
+            }
+        }
+    });
+}
+//# sourceMappingURL=assertions.js.map

package/dist/assertions.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"assertions.js","sourceRoot":"","sources":["../src/assertions.ts"],"names":[],"mappings":";AAAA;;;GAGG;;AAwEH,sDAQC;AAMD,gEA6BC;AAMD,4CA4BC;AAKD,kDAQC;AA2MD,gDAiCC;AA9UD,wEAAwE;AACxE,yEAAyE;AACzE,wEAAwE;AAExE;;;GAGG;AACH,SAAgB,qBAAqB,CAAC,SAAc;IAClD,IAAI,CAAC,SAAS,CAAC,WAAW,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,WAAW,CAAC,EAAE,CAAC;QACpE,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,OAAO,SAAS,CAAC,WAAW;SACzB,GAAG,CAAC,CAAC,IAAS,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,IAAI,EAAE,CAAC;SAC7C,MAAM,CAAC,OAAO,CAAC;SACf,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,SAAgB,0BAA0B,CAAC,SAAc;IACvD,4CAA4C;IAC5C,IAAI,SAAS,CAAC,OAAO,EAAE,iBAAiB,EAAE,CAAC;QACzC,OAAO,SAAS,CAAC,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC;IACpD,CAAC;IAED,2CAA2C;IAC3C,IAAI,SAAS,CAAC,WAAW,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,WAAW,CAAC,EAAE,CAAC;QAClE,MAAM,UAAU,GAAG,SAAS,CAAC,WAAW,CAAC,MAAM,CAC7C,CAAC,CAAM,EAAE,EAAE,CACT,OAAO,CAAC,CAAC,MAAM,KAAK,QAAQ;YAC5B,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC;YAC1B,CAAC,CAAC,QAAQ,GAAG,CAAC,CACjB,CAAC;QACF,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,MAAM,KAAK,GAAG,UAAU,CAAC,MAAM,CAC7B,CAAC,GAAW,EAAE,CAAM,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,EACzC,CAAC,CACF,CAAC;YACF,OAAO,KAAK,GAAG,UAAU,CAAC,MAAM,GAAG,IAAI,CAAC;QAC1C,CAAC;IACH,CAAC;IAED,6CAA6C;IAC7C,IAAI,SAAS,CAAC,QAAQ,EAAE,CAAC;QACvB,OAAO,SAAS,CAAC,QAAQ,GAAG,IAAI,CAAC;IACnC,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,SAAgB,gBAAgB,CAAC,SAAc;IAC7C,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAEhC,IAAI,SAAS,CAAC,WAAW,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,WAAW,CAAC,EAAE,CAAC;QAClE,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,WAAW,EAAE,CAAC;YACzC,yCAAyC;YACzC,IAAI,IAAI,CAAC,SAAS,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;gBACpD,KAAK,MAAM,EAAE,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;oBAChC,IAAI,EAAE,CAAC,IAAI;wBAAE,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;gBAClC,CAAC;YACH,CAAC;YACD,IAAI,IAAI,CAAC,QAAQ,EAAE,SAAS,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;gBACvE,KAAK,MAAM,EAAE,IAAI,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;oBACzC,IAAI,EAAE,CAAC,IAAI;wBAAE,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;gBAClC,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,gCAAgC;IAChC,IAAI,SAAS,CAAC,OAAO,EAAE,SAAS,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;QAC/E,KAAK,MAAM,EAAE,IAAI,SAAS,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC;YAC7C,IAAI,OAAO,EAAE,KAAK,QAAQ;gBAAE,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;iBACrC,IAAI,EAAE,CAAC,IAAI;gBAAE,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;QACvC,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AAC3B,CAAC;AAED;;GAEG;AACH,SAAgB,mBAAmB,CAAC,SAAc;IAChD,IAAI,OAAO,SAAS,CAAC,YAAY,KAAK,QAAQ,EAAE,CAAC;QAC/C,OAAO,SAAS,CAAC,YAAY,CAAC;IAChC,CAAC;IACD,IAAI,OAAO,SAAS,CAAC,QAAQ,EAAE,eAAe,EAAE,YAAY,KAAK,QAAQ,EAAE,CAAC;QAC1E,OAAO,SAAS,CAAC,QAAQ,CAAC,eAAe,CAAC,YAAY,CAAC;IACzD,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,wEAAwE;AACxE,yEAAyE;AACzE,wEAAwE;AAExE,SAAS,eAAe,CAAC,SAA2B,EAAE,SAAc;IAClE,MAAM,UAAU,GAAG,qBAAqB,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC;IAClE,MAAM,WAAW,GAAG,SAAS,CAAC,WAAW,IAAI,eAAe,CAAC;IAE7D,IAAI,SAAS,CAAC,YAAY,IAAI,SAAS,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAChE,MAAM,OAAO,GAAG,SAAS,CAAC,YAAY,CAAC,MAAM,CAC3C,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC,CAC/C,CAAC;QACF,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,WAAW;gBACX,MAAM,EAAE,KAAK;gBACb,MAAM,EAAE,YAAY,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBACxC,QAAQ,EAAE,iBAAiB,SAAS,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBAC9D,MAAM,EAAE,qCAAqC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;aAClE,CAAC;QACJ,CAAC;IACH,CAAC;IAED,IAAI,SAAS,CAAC,gBAAgB,IAAI,SAAS,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxE,MAAM,KAAK,GAAG,SAAS,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CACrD,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC,CACtC,CAAC;QACF,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrB,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,WAAW;gBACX,MAAM,EAAE,KAAK;gBACb,MAAM,EAAE,UAAU,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBACpC,QAAQ,EAAE,qBAAqB,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBACtE,MAAM,EAAE,2CAA2C,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;aACtE,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO;QACL,IAAI,EAAE,SAAS;QACf,WAAW;QACX,MAAM,EAAE,IAAI;QACZ,MAAM,EAAE,sBAAsB;QAC9B,QAAQ,EAAE,qBAAqB,CAAC,SAAS,CAAC;QAC1C,MAAM,EAAE,2BAA2B;KACpC,CAAC;AACJ,CAAC;AAED,SAAS,qBAAqB,CAAC,SAA2B;IACxD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,SAAS,CAAC,YAAY,EAAE,MAAM,EAAE,CAAC;QACnC,KAAK,CAAC,IAAI,CAAC,YAAY,SAAS,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC9D,CAAC;IACD,IAAI,SAAS,CAAC,gBAAgB,EAAE,MAAM,EAAE,CAAC;QACvC,KAAK,CAAC,IAAI,CAAC,YAAY,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAClE,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,oBAAoB,CAC3B,SAAgC,EAChC,SAAc;IAEd,MAAM,WAAW,GAAG,SAAS,CAAC,WAAW,IAAI,qBAAqB,CAAC;IACnE,MAAM,UAAU,GAAG,0BAA0B,CAAC,SAAS,CAAC,CAAC;IAEzD,IAAI,UAAU,KAAK,IAAI,EAAE,CAAC;QACxB,OAAO;YACL,IAAI,EAAE,eAAe;YACrB,WAAW;YACX,MAAM,EAAE,KAAK;YACb,MAAM,EAAE,gBAAgB;YACxB,QAAQ,EAAE,MAAM,SAAS,CAAC,WAAW,GAAG;YACxC,MAAM,EAAE,sDAAsD;SAC/D,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,UAAU,IAAI,SAAS,CAAC,WAAW,CAAC;IACnD,OAAO;QACL,IAAI,EAAE,eAAe;QACrB,WAAW;QACX,MAAM;QACN,MAAM,EAAE,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;QACnC,QAAQ,EAAE,MAAM,SAAS,CAAC,WAAW,GAAG;QACxC,MAAM,EAAE,MAAM;YACZ,CAAC,CAAC,iBAAiB,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,gBAAgB;YACxD,CAAC,CAAC,iBAAiB,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,SAAS,CAAC,WAAW,SAAS;KACtF,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CACrB,SAA0B,EAC1B,SAAc;IAEd,MAAM,WAAW,GAAG,SAAS,CAAC,WAAW,IAAI,oBAAoB,CAAC;IAClE,MAAM,KAAK,GAAG,mBAAmB,CAAC,SAAS,CAAC,CAAC;IAE7C,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;QACnB,OAAO;YACL,IAAI,EAAE,QAAQ;YACd,WAAW;YACX,MAAM,EAAE,KAAK;YACb,MAAM,EAAE,oBAAoB;YAC5B,QAAQ,EAAE,MAAM,SAAS,CAAC,SAAS,KAAK;YACxC,MAAM,EACJ,+EAA+E;SAClF,CAAC;IACJ,CAAC;IAED,6DAA6D;IAC7D,MAAM,eAAe,GAAG,KAAK,GAAG,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;IACxD,MAAM,MAAM,GAAG,eAAe,IAAI,SAAS,CAAC,SAAS,CAAC;IAEtD,OAAO;QACL,IAAI,EAAE,QAAQ;QACd,WAAW;QACX,MAAM;QACN,MAAM,EAAE,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK;QAC1C,QAAQ,EAAE,MAAM,SAAS,CAAC,SAAS,KAAK;QACxC,MAAM,EAAE,MAAM;YACZ,CAAC,CAAC,SAAS,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,kBAAkB,SAAS,CAAC,SAAS,EAAE;YAC5E,CAAC,CAAC,SAAS,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,kBAAkB,SAAS,CAAC,SAAS,EAAE;KAC/E,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CACvB,SAA4B,EAC5B,SAAc;IAEd,MAAM,WAAW,GAAG,SAAS,CAAC,WAAW,IAAI,iBAAiB,CAAC;IAC/D,MAAM,WAAW,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAC;IAEhD,wEAAwE;IACxE,IAAI,cAAc,GAAG,WAAW,CAAC;IACjC,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,MAAM,UAAU,GAAG,qBAAqB,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC;QAClE,cAAc,GAAG,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAClD,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CACxC,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAG,SAAS,CAAC,QAAQ,CAAC,MAAM,CACvC,CAAC,IAAI,EAAE,EAAE,CACP,CAAC,cAAc,CAAC,IAAI,CAClB,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,IAAI,CAAC,WAAW,EAAE,CACxD,CACJ,CAAC;IAEF,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC;IACpC,OAAO;QACL,IAAI,EAAE,WAAW;QACjB,WAAW;QACX,MAAM;QACN,MAAM,EAAE,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM;QACtE,QAAQ,EAAE,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC;QACvC,MAAM,EAAE,MAAM;YACZ,CAAC,CAAC,mCAAmC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;YACpE,CAAC,CAAC,uBAAuB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;KAChD,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CACpB,SAAyB,EACzB,SAAc;IAEd,MAAM,WAAW,GAAG,SAAS,CAAC,WAAW,IAAI,aAAa,CAAC;IAC3D,MAAM,KAAK,GAAG,mBAAmB,CAAC,SAAS,CAAC,CAAC;IAE7C,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;QACnB,OAAO;YACL,IAAI,EAAE,OAAO;YACb,WAAW;YACX,MAAM,EAAE,KAAK;YACb,MAAM,EAAE,UAAU;YAClB,QAAQ,EAAE,MAAM,SAAS,CAAC,SAAS,EAAE;YACrC,MAAM,EAAE,iDAAiD;SAC1D,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,KAAK,IAAI,SAAS,CAAC,SAAS,CAAC;IAC5C,OAAO;QACL,IAAI,EAAE,OAAO;QACb,WAAW;QACX,MAAM;QACN,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC;QACrB,QAAQ,EAAE,MAAM,SAAS,CAAC,SAAS,EAAE;QACrC,MAAM,EAAE,MAAM;YACZ,CAAC,CAAC,SAAS,KAAK,kBAAkB,SAAS,CAAC,SAAS,EAAE;YACvD,CAAC,CAAC,SAAS,KAAK,kBAAkB,SAAS,CAAC,SAAS,EAAE;KAC1D,CAAC;AACJ,CAAC;AAED,wEAAwE;AACxE,yEAAyE;AACzE,wEAAwE;AAExE;;GAEG;AACH,SAAgB,kBAAkB,CAChC,UAAuB,EACvB,SAAc;IAEd,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE;QAClC,QAAQ,SAAS,CAAC,IAAI,EAAE,CAAC;YACvB,KAAK,SAAS;gBACZ,OAAO,eAAe,CAAC,SAA6B,EAAE,SAAS,CAAC,CAAC;YACnE,KAAK,eAAe;gBAClB,OAAO,oBAAoB,CACzB,SAAkC,EAClC,SAAS,CACV,CAAC;YACJ,KAAK,QAAQ;gBACX,OAAO,cAAc,CAAC,SAA4B,EAAE,SAAS,CAAC,CAAC;YACjE,KAAK,WAAW;gBACd,OAAO,gBAAgB,CAAC,SAA8B,EAAE,SAAS,CAAC,CAAC;YACrE,KAAK,OAAO;gBACV,OAAO,aAAa,CAAC,SAA2B,EAAE,SAAS,CAAC,CAAC;YAC/D,OAAO,CAAC,CAAC,CAAC;gBACR,MAAM,OAAO,GAAG,SAAoC,CAAC;gBACrD,OAAO;oBACL,IAAI,EAAE,CAAC,OAAO,OAAO,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;oBACnE,WAAW,EACT,CAAC,OAAO,OAAO,CAAC,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,wBAAwB,CAAC;oBAC5F,MAAM,EAAE,KAAK;oBACb,MAAM,EAAE,KAAK;oBACb,QAAQ,EAAE,KAAK;oBACf,MAAM,EAAE,2BAA2B,OAAO,OAAO,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,EAAE;iBACjG,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/baseline.d.ts

@@ -0,0 +1,100 @@
+/**
+ * Baseline management for `chanl test --save-baseline` / `--baseline`.
+ *
+ * Saves test results as a JSON snapshot and compares future runs
+ * against the snapshot to detect regressions.
+ */
+export interface BaselineAssertionEntry {
+    [assertionKey: string]: boolean;
+}
+export interface BaselineTestEntry {
+    score: number | null;
+    assertions: Record<string, boolean>;
+}
+export interface BaselineFile {
+    version: 1;
+    savedAt: string;
+    results: Record<string, BaselineTestEntry>;
+}
+export type ComparisonStatus = 'improved' | 'regressed' | 'unchanged' | 'added' | 'removed';
+export interface AssertionComparison {
+    name: string;
+    status: ComparisonStatus;
+    before: boolean | null;
+    after: boolean | null;
+}
+export interface TestComparison {
+    file: string;
+    status: ComparisonStatus;
+    scoreBefore: number | null;
+    scoreAfter: number | null;
+    scoreDelta: number | null;
+    assertions: AssertionComparison[];
+}
+export interface BaselineComparison {
+    tests: TestComparison[];
+    summary: {
+        improved: number;
+        regressed: number;
+        unchanged: number;
+        added: number;
+        removed: number;
+    };
+    hasRegressions: boolean;
+}
+/**
+ * Build a stable key for an assertion result.
+ * Uses the description if available, otherwise falls back to type + index.
+ */
+export declare function assertionKey(result: {
+    type: string;
+    description?: string;
+}, index: number): string;
+/**
+ * Build a BaselineFile from test results (same shape as test.ts TestFileResult[]).
+ */
+export declare function buildBaseline(results: Array<{
+    file: string;
+    scenarioName: string;
+    assertions: Array<{
+        type: string;
+        description: string;
+        passed: boolean;
+    }>;
+    error?: string;
+}>): BaselineFile;
+/**
+ * Save baseline results to a JSON file.
+ */
+export declare function saveBaseline(results: Array<{
+    file: string;
+    scenarioName: string;
+    assertions: Array<{
+        type: string;
+        description: string;
+        passed: boolean;
+    }>;
+    error?: string;
+}>, filePath?: string): string;
+/**
+ * Load a baseline file. Returns null if the file does not exist.
+ */
+export declare function loadBaseline(filePath?: string): BaselineFile | null;
+/**
+ * Compare current test results against a saved baseline.
+ */
+export declare function compareWithBaseline(currentResults: Array<{
+    file: string;
+    scenarioName: string;
+    assertions: Array<{
+        type: string;
+        description: string;
+        passed: boolean;
+    }>;
+    error?: string;
+}>, baseline: BaselineFile): BaselineComparison;
+/**
+ * Print a human-readable baseline comparison to stdout.
+ */
+export declare function printBaselineComparison(comparison: BaselineComparison): void;
+//# sourceMappingURL=baseline.d.ts.map

package/dist/baseline.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"baseline.d.ts","sourceRoot":"","sources":["../src/baseline.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAUH,MAAM,WAAW,sBAAsB;IACrC,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC;CACjC;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACrC;AAED,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,CAAC,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,iBAAiB,CAAC,CAAC;CAC5C;AAED,MAAM,MAAM,gBAAgB,GAAG,UAAU,GAAG,WAAW,GAAG,WAAW,GAAG,OAAO,GAAG,SAAS,CAAC;AAE5F,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,gBAAgB,CAAC;IACzB,MAAM,EAAE,OAAO,GAAG,IAAI,CAAC;IACvB,KAAK,EAAE,OAAO,GAAG,IAAI,CAAC;CACvB;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,gBAAgB,CAAC;IACzB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,UAAU,EAAE,mBAAmB,EAAE,CAAC;CACnC;AAED,MAAM,WAAW,kBAAkB;IACjC,KAAK,EAAE,cAAc,EAAE,CAAC;IACxB,OAAO,EAAE;QACP,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,MAAM,CAAC;QAClB,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;IACF,cAAc,EAAE,OAAO,CAAC;CACzB;AAMD;;;GAGG;AACH,wBAAgB,YAAY,CAC1B,MAAM,EAAE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,EAC9C,KAAK,EAAE,MAAM,GACZ,MAAM,CAKR;AAQD;;GAEG;AACH,wBAAgB,aAAa,CAC3B,OAAO,EAAE,KAAK,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;IAC1E,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC,GACD,YAAY,CA2Bd;AAED;;GAEG;AACH,wBAAgB,YAAY,CAC1B,OAAO,EAAE,KAAK,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;IAC1E,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC,EACF,QAAQ,CAAC,EAAE,MAAM,GAChB,MAAM,CAKR;AAED;;GAEG;AACH,wBAAgB,YAAY,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,YAAY,GAAG,IAAI,CAiBnE;AAMD;;GAEG;AACH,wBAAgB,mBAAmB,CACjC,cAAc,EAAE,KAAK,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;IAC1E,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC,EACF,QAAQ,EAAE,YAAY,GACrB,kBAAkB,CAmIpB;AAmCD;;GAEG;AACH,wBAAgB,uBAAuB,CAAC,UAAU,EAAE,kBAAkB,GAAG,IAAI,CAwD5E"}