@skillmark/cli 0.1.0

package/dist/engine/concept-accuracy-scorer.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Concept accuracy scorer - calculates how well responses match expected concepts
+ */
+import type { TestDefinition, TestResult, BenchmarkMetrics } from '../types/index.js';
+/** Scoring options */
+export interface ScoringOptions {
+    /** Minimum similarity threshold for fuzzy matching (0-1) */
+    fuzzyThreshold?: number;
+    /** Whether to use case-insensitive matching */
+    caseInsensitive?: boolean;
+}
+/**
+ * Score a response against expected concepts
+ */
+export declare function scoreResponse(test: TestDefinition, response: string, metrics: BenchmarkMetrics, options?: ScoringOptions): TestResult;
+/**
+ * Aggregate metrics from multiple test results
+ */
+export declare function aggregateMetrics(results: TestResult[]): BenchmarkMetrics;
+/**
+ * Calculate pass rate from test results
+ */
+export declare function calculatePassRate(results: TestResult[]): number;
+//# sourceMappingURL=concept-accuracy-scorer.d.ts.map

package/dist/engine/concept-accuracy-scorer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"concept-accuracy-scorer.d.ts","sourceRoot":"","sources":["../../src/engine/concept-accuracy-scorer.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,KAAK,EAAE,cAAc,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAEtF,sBAAsB;AACtB,MAAM,WAAW,cAAc;IAC7B,4DAA4D;IAC5D,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,+CAA+C;IAC/C,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B;AAOD;;GAEG;AACH,wBAAgB,aAAa,CAC3B,IAAI,EAAE,cAAc,EACpB,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,gBAAgB,EACzB,OAAO,GAAE,cAAmB,GAC3B,UAAU,CA4DZ;AAmFD;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,gBAAgB,CA6CxE;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,MAAM,CAI/D"}

package/dist/engine/concept-accuracy-scorer.js

@@ -0,0 +1,186 @@
+const DEFAULT_OPTIONS = {
+    fuzzyThreshold: 0.8,
+    caseInsensitive: true,
+};
+/**
+ * Score a response against expected concepts
+ */
+export function scoreResponse(test, response, metrics, options = {}) {
+    const opts = { ...DEFAULT_OPTIONS, ...options };
+    const normalizedResponse = opts.caseInsensitive ? response.toLowerCase() : response;
+    const matchedConcepts = [];
+    const missedConcepts = [];
+    // Check each expected concept
+    for (const concept of test.concepts) {
+        const normalizedConcept = opts.caseInsensitive ? concept.toLowerCase() : concept;
+        if (conceptMatches(normalizedResponse, normalizedConcept, opts.fuzzyThreshold)) {
+            matchedConcepts.push(concept);
+        }
+        else {
+            missedConcepts.push(concept);
+        }
+    }
+    // Also check expected patterns from test definition
+    for (const expected of test.expected) {
+        // Parse checkbox items like "- [ ] Pattern to check"
+        const checkboxMatch = expected.match(/^-\s*\[[\sx]\]\s*(.+)$/i);
+        const pattern = checkboxMatch ? checkboxMatch[1] : expected;
+        const normalizedPattern = opts.caseInsensitive ? pattern.toLowerCase() : pattern;
+        // Skip if already in concepts
+        if (test.concepts.some((c) => c.toLowerCase() === normalizedPattern)) {
+            continue;
+        }
+        if (conceptMatches(normalizedResponse, normalizedPattern, opts.fuzzyThreshold)) {
+            if (!matchedConcepts.includes(pattern)) {
+                matchedConcepts.push(pattern);
+            }
+        }
+        else {
+            if (!missedConcepts.includes(pattern)) {
+                missedConcepts.push(pattern);
+            }
+        }
+    }
+    // Calculate accuracy
+    const totalConcepts = matchedConcepts.length + missedConcepts.length;
+    const accuracy = totalConcepts > 0 ? (matchedConcepts.length / totalConcepts) * 100 : 0;
+    // Update metrics with calculated accuracy
+    const scoredMetrics = {
+        ...metrics,
+        accuracy,
+    };
+    return {
+        test,
+        metrics: scoredMetrics,
+        matchedConcepts,
+        missedConcepts,
+        response,
+        timestamp: new Date().toISOString(),
+        passed: accuracy >= 70, // Default passing threshold
+    };
+}
+/**
+ * Check if a concept is present in the response
+ */
+function conceptMatches(response, concept, fuzzyThreshold) {
+    // Direct substring match
+    if (response.includes(concept)) {
+        return true;
+    }
+    // Word-by-word match for multi-word concepts
+    const conceptWords = concept.split(/\s+/).filter((w) => w.length > 2);
+    if (conceptWords.length > 1) {
+        const matchedWords = conceptWords.filter((word) => response.includes(word));
+        const ratio = matchedWords.length / conceptWords.length;
+        if (ratio >= fuzzyThreshold) {
+            return true;
+        }
+    }
+    // Check for synonyms/variations
+    const variations = generateVariations(concept);
+    for (const variation of variations) {
+        if (response.includes(variation)) {
+            return true;
+        }
+    }
+    return false;
+}
+/**
+ * Generate common variations of a concept
+ */
+function generateVariations(concept) {
+    const variations = [];
+    // Hyphenated vs spaced
+    if (concept.includes('-')) {
+        variations.push(concept.replace(/-/g, ' '));
+    }
+    if (concept.includes(' ')) {
+        variations.push(concept.replace(/\s+/g, '-'));
+    }
+    // Plural/singular
+    if (concept.endsWith('s') && concept.length > 3) {
+        variations.push(concept.slice(0, -1));
+    }
+    else {
+        variations.push(concept + 's');
+    }
+    // Common abbreviations
+    const abbreviations = {
+        'context': ['ctx'],
+        'configuration': ['config', 'cfg'],
+        'documentation': ['docs', 'doc'],
+        'application': ['app'],
+        'authentication': ['auth'],
+        'authorization': ['authz'],
+        'database': ['db'],
+        'message': ['msg'],
+        'response': ['resp'],
+        'request': ['req'],
+    };
+    for (const [full, abbrs] of Object.entries(abbreviations)) {
+        if (concept.includes(full)) {
+            for (const abbr of abbrs) {
+                variations.push(concept.replace(full, abbr));
+            }
+        }
+        for (const abbr of abbrs) {
+            if (concept.includes(abbr)) {
+                variations.push(concept.replace(abbr, full));
+            }
+        }
+    }
+    return variations;
+}
+/**
+ * Aggregate metrics from multiple test results
+ */
+export function aggregateMetrics(results) {
+    if (results.length === 0) {
+        return {
+            accuracy: 0,
+            tokensTotal: 0,
+            tokensInput: 0,
+            tokensOutput: 0,
+            durationMs: 0,
+            toolCount: 0,
+            costUsd: 0,
+        };
+    }
+    const sum = results.reduce((acc, r) => ({
+        accuracy: acc.accuracy + r.metrics.accuracy,
+        tokensTotal: acc.tokensTotal + r.metrics.tokensTotal,
+        tokensInput: acc.tokensInput + r.metrics.tokensInput,
+        tokensOutput: acc.tokensOutput + r.metrics.tokensOutput,
+        durationMs: acc.durationMs + r.metrics.durationMs,
+        toolCount: acc.toolCount + r.metrics.toolCount,
+        costUsd: acc.costUsd + r.metrics.costUsd,
+    }), {
+        accuracy: 0,
+        tokensTotal: 0,
+        tokensInput: 0,
+        tokensOutput: 0,
+        durationMs: 0,
+        toolCount: 0,
+        costUsd: 0,
+    });
+    const count = results.length;
+    return {
+        accuracy: sum.accuracy / count,
+        tokensTotal: Math.round(sum.tokensTotal / count),
+        tokensInput: Math.round(sum.tokensInput / count),
+        tokensOutput: Math.round(sum.tokensOutput / count),
+        durationMs: Math.round(sum.durationMs / count),
+        toolCount: Math.round(sum.toolCount / count),
+        costUsd: sum.costUsd / count,
+    };
+}
+/**
+ * Calculate pass rate from test results
+ */
+export function calculatePassRate(results) {
+    if (results.length === 0)
+        return 0;
+    const passed = results.filter((r) => r.passed).length;
+    return (passed / results.length) * 100;
+}
+//# sourceMappingURL=concept-accuracy-scorer.js.map

package/dist/engine/concept-accuracy-scorer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"concept-accuracy-scorer.js","sourceRoot":"","sources":["../../src/engine/concept-accuracy-scorer.ts"],"names":[],"mappings":"AAaA,MAAM,eAAe,GAA6B;IAChD,cAAc,EAAE,GAAG;IACnB,eAAe,EAAE,IAAI;CACtB,CAAC;AAEF;;GAEG;AACH,MAAM,UAAU,aAAa,CAC3B,IAAoB,EACpB,QAAgB,EAChB,OAAyB,EACzB,UAA0B,EAAE;IAE5B,MAAM,IAAI,GAAG,EAAE,GAAG,eAAe,EAAE,GAAG,OAAO,EAAE,CAAC;IAChD,MAAM,kBAAkB,GAAG,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC;IAEpF,MAAM,eAAe,GAAa,EAAE,CAAC;IACrC,MAAM,cAAc,GAAa,EAAE,CAAC;IAEpC,8BAA8B;IAC9B,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QACpC,MAAM,iBAAiB,GAAG,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;QAEjF,IAAI,cAAc,CAAC,kBAAkB,EAAE,iBAAiB,EAAE,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC;YAC/E,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAChC,CAAC;aAAM,CAAC;YACN,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IAED,oDAAoD;IACpD,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QACrC,qDAAqD;QACrD,MAAM,aAAa,GAAG,QAAQ,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAChE,MAAM,OAAO,GAAG,aAAa,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;QAC5D,MAAM,iBAAiB,GAAG,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;QAEjF,8BAA8B;QAC9B,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,KAAK,iBAAiB,CAAC,EAAE,CAAC;YACrE,SAAS;QACX,CAAC;QAED,IAAI,cAAc,CAAC,kBAAkB,EAAE,iBAAiB,EAAE,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC;YAC/E,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;gBACvC,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAChC,CAAC;QACH,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;gBACtC,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC;IACH,CAAC;IAED,qBAAqB;IACrB,MAAM,aAAa,GAAG,eAAe,CAAC,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC;IACrE,MAAM,QAAQ,GAAG,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,MAAM,GAAG,aAAa,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAExF,0CAA0C;IAC1C,MAAM,aAAa,GAAqB;QACtC,GAAG,OAAO;QACV,QAAQ;KACT,CAAC;IAEF,OAAO;QACL,IAAI;QACJ,OAAO,EAAE,aAAa;QACtB,eAAe;QACf,cAAc;QACd,QAAQ;QACR,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,MAAM,EAAE,QAAQ,IAAI,EAAE,EAAE,4BAA4B;KACrD,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CAAC,QAAgB,EAAE,OAAe,EAAE,cAAsB;IAC/E,yBAAyB;IACzB,IAAI,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;QAC/B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,6CAA6C;IAC7C,MAAM,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACtE,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,MAAM,YAAY,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;QAC5E,MAAM,KAAK,GAAG,YAAY,CAAC,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;QACxD,IAAI,KAAK,IAAI,cAAc,EAAE,CAAC;YAC5B,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,gCAAgC;IAChC,MAAM,UAAU,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAC;IAC/C,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,IAAI,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YACjC,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,SAAS,kBAAkB,CAAC,OAAe;IACzC,MAAM,UAAU,GAAa,EAAE,CAAC;IAEhC,uBAAuB;IACvB,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC1B,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC;IAC9C,CAAC;IACD,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC1B,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC;IAChD,CAAC;IAED,kBAAkB;IAClB,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAChD,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACxC,CAAC;SAAM,CAAC;QACN,UAAU,CAAC,IAAI,CAAC,OAAO,GAAG,GAAG,CAAC,CAAC;IACjC,CAAC;IAED,uBAAuB;IACvB,MAAM,aAAa,GAA6B;QAC9C,SAAS,EAAE,CAAC,KAAK,CAAC;QAClB,eAAe,EAAE,CAAC,QAAQ,EAAE,KAAK,CAAC;QAClC,eAAe,EAAE,CAAC,MAAM,EAAE,KAAK,CAAC;QAChC,aAAa,EAAE,CAAC,KAAK,CAAC;QACtB,gBAAgB,EAAE,CAAC,MAAM,CAAC;QAC1B,eAAe,EAAE,CAAC,OAAO,CAAC;QAC1B,UAAU,EAAE,CAAC,IAAI,CAAC;QAClB,SAAS,EAAE,CAAC,KAAK,CAAC;QAClB,UAAU,EAAE,CAAC,MAAM,CAAC;QACpB,SAAS,EAAE,CAAC,KAAK,CAAC;KACnB,CAAC;IAEF,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,CAAC;QAC1D,IAAI,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;YAC/C,CAAC;QACH,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC3B,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;YAC/C,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAqB;IACpD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO;YACL,QAAQ,EAAE,CAAC;YACX,WAAW,EAAE,CAAC;YACd,WAAW,EAAE,CAAC;YACd,YAAY,EAAE,CAAC;YACf,UAAU,EAAE,CAAC;YACb,SAAS,EAAE,CAAC;YACZ,OAAO,EAAE,CAAC;SACX,CAAC;IACJ,CAAC;IAED,MAAM,GAAG,GAAG,OAAO,CAAC,MAAM,CACxB,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QACX,QAAQ,EAAE,GAAG,CAAC,QAAQ,GAAG,CAAC,CAAC,OAAO,CAAC,QAAQ;QAC3C,WAAW,EAAE,GAAG,CAAC,WAAW,GAAG,CAAC,CAAC,OAAO,CAAC,WAAW;QACpD,WAAW,EAAE,GAAG,CAAC,WAAW,GAAG,CAAC,CAAC,OAAO,CAAC,WAAW;QACpD,YAAY,EAAE,GAAG,CAAC,YAAY,GAAG,CAAC,CAAC,OAAO,CAAC,YAAY;QACvD,UAAU,EAAE,GAAG,CAAC,UAAU,GAAG,CAAC,CAAC,OAAO,CAAC,UAAU;QACjD,SAAS,EAAE,GAAG,CAAC,SAAS,GAAG,CAAC,CAAC,OAAO,CAAC,SAAS;QAC9C,OAAO,EAAE,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,OAAO;KACzC,CAAC,EACF;QACE,QAAQ,EAAE,CAAC;QACX,WAAW,EAAE,CAAC;QACd,WAAW,EAAE,CAAC;QACd,YAAY,EAAE,CAAC;QACf,UAAU,EAAE,CAAC;QACb,SAAS,EAAE,CAAC;QACZ,OAAO,EAAE,CAAC;KACX,CACF,CAAC;IAEF,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC;IAE7B,OAAO;QACL,QAAQ,EAAE,GAAG,CAAC,QAAQ,GAAG,KAAK;QAC9B,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,WAAW,GAAG,KAAK,CAAC;QAChD,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,WAAW,GAAG,KAAK,CAAC;QAChD,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,YAAY,GAAG,KAAK,CAAC;QAClD,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,GAAG,KAAK,CAAC;QAC9C,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,GAAG,KAAK,CAAC;QAC5C,OAAO,EAAE,GAAG,CAAC,OAAO,GAAG,KAAK;KAC7B,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,OAAqB;IACrD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACnC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IACtD,OAAO,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC;AACzC,CAAC"}

package/dist/engine/concept-accuracy-scorer.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=concept-accuracy-scorer.test.d.ts.map

package/dist/engine/concept-accuracy-scorer.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"concept-accuracy-scorer.test.d.ts","sourceRoot":"","sources":["../../src/engine/concept-accuracy-scorer.test.ts"],"names":[],"mappings":""}

package/dist/engine/concept-accuracy-scorer.test.js

@@ -0,0 +1,230 @@
+/**
+ * Tests for concept-accuracy-scorer.ts
+ * Verifies scoring logic, fuzzy matching, and metric aggregation
+ */
+import { describe, it, expect } from 'vitest';
+import { scoreResponse, aggregateMetrics, calculatePassRate, } from './concept-accuracy-scorer.js';
+// Helper to create test definition
+function createTestDefinition(overrides = {}) {
+    return {
+        name: 'test-concept-matching',
+        type: 'knowledge',
+        concepts: ['orchestrator', 'context isolation', 'consensus'],
+        timeout: 600,
+        prompt: 'How do multi-agent systems work?',
+        expected: ['coordination pattern', 'message passing'],
+        sourcePath: '/test/test.md',
+        ...overrides,
+    };
+}
+// Helper to create metrics
+function createMetrics(overrides = {}) {
+    return {
+        accuracy: 0,
+        tokensTotal: 1000,
+        tokensInput: 800,
+        tokensOutput: 200,
+        durationMs: 5000,
+        toolCount: 3,
+        costUsd: 0.01,
+        ...overrides,
+    };
+}
+describe('scoreResponse', () => {
+    describe('exact matching', () => {
+        it('matches concepts that appear exactly in response', () => {
+            const test = createTestDefinition({
+                concepts: ['orchestrator', 'consensus'],
+                expected: [],
+            });
+            const response = 'The orchestrator pattern uses consensus for decisions';
+            const metrics = createMetrics();
+            const result = scoreResponse(test, response, metrics);
+            expect(result.matchedConcepts).toContain('orchestrator');
+            expect(result.matchedConcepts).toContain('consensus');
+            expect(result.missedConcepts).toHaveLength(0);
+            expect(result.metrics.accuracy).toBe(100);
+        });
+        it('tracks missed concepts', () => {
+            const test = createTestDefinition({
+                concepts: ['orchestrator', 'blockchain', 'consensus'],
+                expected: [],
+            });
+            const response = 'The orchestrator pattern coordinates agents';
+            const metrics = createMetrics();
+            const result = scoreResponse(test, response, metrics);
+            expect(result.matchedConcepts).toContain('orchestrator');
+            expect(result.missedConcepts).toContain('blockchain');
+            expect(result.missedConcepts).toContain('consensus');
+        });
+    });
+    describe('case insensitive matching', () => {
+        it('matches regardless of case by default', () => {
+            const test = createTestDefinition({
+                concepts: ['Orchestrator', 'CONSENSUS'],
+                expected: [],
+            });
+            const response = 'The orchestrator uses consensus mechanisms';
+            const metrics = createMetrics();
+            const result = scoreResponse(test, response, metrics);
+            expect(result.matchedConcepts).toHaveLength(2);
+            expect(result.metrics.accuracy).toBe(100);
+        });
+    });
+    describe('fuzzy matching', () => {
+        it('matches hyphenated vs spaced variations', () => {
+            const test = createTestDefinition({
+                concepts: ['context-isolation'],
+                expected: [],
+            });
+            const response = 'We use context isolation for safety';
+            const metrics = createMetrics();
+            const result = scoreResponse(test, response, metrics);
+            expect(result.matchedConcepts).toContain('context-isolation');
+        });
+        it('matches plurals', () => {
+            const test = createTestDefinition({
+                concepts: ['agents'],
+                expected: [],
+            });
+            const response = 'Each agent has its own context';
+            const metrics = createMetrics();
+            const result = scoreResponse(test, response, metrics);
+            expect(result.matchedConcepts).toContain('agents');
+        });
+        it('matches common abbreviations', () => {
+            const test = createTestDefinition({
+                concepts: ['configuration'],
+                expected: [],
+            });
+            const response = 'The config file specifies settings';
+            const metrics = createMetrics();
+            const result = scoreResponse(test, response, metrics);
+            expect(result.matchedConcepts).toContain('configuration');
+        });
+    });
+    describe('expected patterns', () => {
+        it('parses checkbox items from expected', () => {
+            const test = createTestDefinition({
+                concepts: [],
+                expected: ['- [ ] Uses message passing', '- [x] Handles failures'],
+            });
+            const response = 'The system uses message passing and handles failures gracefully';
+            const metrics = createMetrics();
+            const result = scoreResponse(test, response, metrics);
+            expect(result.matchedConcepts).toContain('Uses message passing');
+            expect(result.matchedConcepts).toContain('Handles failures');
+        });
+    });
+    describe('accuracy calculation', () => {
+        it('calculates accuracy as matched / total * 100', () => {
+            const test = createTestDefinition({
+                concepts: ['orchestrator', 'consensus', 'blockchain', 'kubernetes'],
+                expected: [],
+            });
+            // Only 'orchestrator' and 'consensus' appear in response
+            const response = 'The orchestrator uses consensus for coordination';
+            const metrics = createMetrics();
+            const result = scoreResponse(test, response, metrics);
+            expect(result.metrics.accuracy).toBe(50); // 2/4 * 100
+        });
+        it('returns 0 accuracy when no concepts', () => {
+            const test = createTestDefinition({
+                concepts: [],
+                expected: [],
+            });
+            const response = 'Some response';
+            const metrics = createMetrics();
+            const result = scoreResponse(test, response, metrics);
+            expect(result.metrics.accuracy).toBe(0);
+        });
+    });
+    describe('pass/fail threshold', () => {
+        it('passes when accuracy >= 70%', () => {
+            const test = createTestDefinition({
+                concepts: ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j'],
+                expected: [],
+            });
+            // Match 7 out of 10 = 70%
+            const response = 'a b c d e f g';
+            const metrics = createMetrics();
+            const result = scoreResponse(test, response, metrics);
+            expect(result.passed).toBe(true);
+        });
+        it('fails when accuracy < 70%', () => {
+            const test = createTestDefinition({
+                concepts: ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j'],
+                expected: [],
+            });
+            // Match 6 out of 10 = 60%
+            const response = 'a b c d e f';
+            const metrics = createMetrics();
+            const result = scoreResponse(test, response, metrics);
+            expect(result.passed).toBe(false);
+        });
+    });
+});
+describe('aggregateMetrics', () => {
+    it('returns zero metrics for empty results', () => {
+        const result = aggregateMetrics([]);
+        expect(result.accuracy).toBe(0);
+        expect(result.tokensTotal).toBe(0);
+        expect(result.durationMs).toBe(0);
+        expect(result.costUsd).toBe(0);
+    });
+    it('averages metrics across multiple results', () => {
+        const results = [
+            {
+                test: createTestDefinition(),
+                metrics: createMetrics({ accuracy: 80, tokensTotal: 1000, costUsd: 0.10 }),
+                matchedConcepts: [],
+                missedConcepts: [],
+                response: '',
+                timestamp: new Date().toISOString(),
+                passed: true,
+            },
+            {
+                test: createTestDefinition(),
+                metrics: createMetrics({ accuracy: 60, tokensTotal: 2000, costUsd: 0.20 }),
+                matchedConcepts: [],
+                missedConcepts: [],
+                response: '',
+                timestamp: new Date().toISOString(),
+                passed: false,
+            },
+        ];
+        const aggregated = aggregateMetrics(results);
+        expect(aggregated.accuracy).toBe(70); // (80 + 60) / 2
+        expect(aggregated.tokensTotal).toBe(1500); // (1000 + 2000) / 2
+        expect(aggregated.costUsd).toBeCloseTo(0.15, 10); // (0.10 + 0.20) / 2
+    });
+});
+describe('calculatePassRate', () => {
+    it('returns 0 for empty results', () => {
+        expect(calculatePassRate([])).toBe(0);
+    });
+    it('calculates percentage of passed tests', () => {
+        const results = [
+            { passed: true },
+            { passed: true },
+            { passed: false },
+            { passed: true },
+        ];
+        expect(calculatePassRate(results)).toBe(75); // 3/4 * 100
+    });
+    it('returns 100 when all tests pass', () => {
+        const results = [
+            { passed: true },
+            { passed: true },
+        ];
+        expect(calculatePassRate(results)).toBe(100);
+    });
+    it('returns 0 when all tests fail', () => {
+        const results = [
+            { passed: false },
+            { passed: false },
+        ];
+        expect(calculatePassRate(results)).toBe(0);
+    });
+});
+//# sourceMappingURL=concept-accuracy-scorer.test.js.map

package/dist/engine/concept-accuracy-scorer.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"concept-accuracy-scorer.test.js","sourceRoot":"","sources":["../../src/engine/concept-accuracy-scorer.test.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EACL,aAAa,EACb,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,8BAA8B,CAAC;AAGtC,mCAAmC;AACnC,SAAS,oBAAoB,CAAC,YAAqC,EAAE;IACnE,OAAO;QACL,IAAI,EAAE,uBAAuB;QAC7B,IAAI,EAAE,WAAW;QACjB,QAAQ,EAAE,CAAC,cAAc,EAAE,mBAAmB,EAAE,WAAW,CAAC;QAC5D,OAAO,EAAE,GAAG;QACZ,MAAM,EAAE,kCAAkC;QAC1C,QAAQ,EAAE,CAAC,sBAAsB,EAAE,iBAAiB,CAAC;QACrD,UAAU,EAAE,eAAe;QAC3B,GAAG,SAAS;KACb,CAAC;AACJ,CAAC;AAED,2BAA2B;AAC3B,SAAS,aAAa,CAAC,YAAuC,EAAE;IAC9D,OAAO;QACL,QAAQ,EAAE,CAAC;QACX,WAAW,EAAE,IAAI;QACjB,WAAW,EAAE,GAAG;QAChB,YAAY,EAAE,GAAG;QACjB,UAAU,EAAE,IAAI;QAChB,SAAS,EAAE,CAAC;QACZ,OAAO,EAAE,IAAI;QACb,GAAG,SAAS;KACb,CAAC;AACJ,CAAC;AAED,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;IAC7B,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;QAC9B,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;YAC1D,MAAM,IAAI,GAAG,oBAAoB,CAAC;gBAChC,QAAQ,EAAE,CAAC,cAAc,EAAE,WAAW,CAAC;gBACvC,QAAQ,EAAE,EAAE;aACb,CAAC,CAAC;YACH,MAAM,QAAQ,GAAG,uDAAuD,CAAC;YACzE,MAAM,OAAO,GAAG,aAAa,EAAE,CAAC;YAEhC,MAAM,MAAM,GAAG,aAAa,CAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;YAEtD,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;YACzD,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;YACtD,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAC9C,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC5C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wBAAwB,EAAE,GAAG,EAAE;YAChC,MAAM,IAAI,GAAG,oBAAoB,CAAC;gBAChC,QAAQ,EAAE,CAAC,cAAc,EAAE,YAAY,EAAE,WAAW,CAAC;gBACrD,QAAQ,EAAE,EAAE;aACb,CAAC,CAAC;YACH,MAAM,QAAQ,GAAG,6CAA6C,CAAC;YAC/D,MAAM,OAAO,GAAG,aAAa,EAAE,CAAC;YAEhC,MAAM,MAAM,GAAG,aAAa,CAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;YAEtD,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;YACzD,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;YACtD,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,2BAA2B,EAAE,GAAG,EAAE;QACzC,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,MAAM,IAAI,GAAG,oBAAoB,CAAC;gBAChC,QAAQ,EAAE,CAAC,cAAc,EAAE,WAAW,CAAC;gBACvC,QAAQ,EAAE,EAAE;aACb,CAAC,CAAC;YACH,MAAM,QAAQ,GAAG,4CAA4C,CAAC;YAC9D,MAAM,OAAO,GAAG,aAAa,EAAE,CAAC;YAEhC,MAAM,MAAM,GAAG,aAAa,CAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;YAEtD,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAC/C,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC5C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;QAC9B,EAAE,CAAC,yCAAyC,EAAE,GAAG,EAAE;YACjD,MAAM,IAAI,GAAG,oBAAoB,CAAC;gBAChC,QAAQ,EAAE,CAAC,mBAAmB,CAAC;gBAC/B,QAAQ,EAAE,EAAE;aACb,CAAC,CAAC;YACH,MAAM,QAAQ,GAAG,qCAAqC,CAAC;YACvD,MAAM,OAAO,GAAG,aAAa,EAAE,CAAC;YAEhC,MAAM,MAAM,GAAG,aAAa,CAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;YAEtD,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,SAAS,CAAC,mBAAmB,CAAC,CAAC;QAChE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iBAAiB,EAAE,GAAG,EAAE;YACzB,MAAM,IAAI,GAAG,oBAAoB,CAAC;gBAChC,QAAQ,EAAE,CAAC,QAAQ,CAAC;gBACpB,QAAQ,EAAE,EAAE;aACb,CAAC,CAAC;YACH,MAAM,QAAQ,GAAG,gCAAgC,CAAC;YAClD,MAAM,OAAO,GAAG,aAAa,EAAE,CAAC;YAEhC,MAAM,MAAM,GAAG,aAAa,CAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;YAEtD,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;QACrD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8BAA8B,EAAE,GAAG,EAAE;YACtC,MAAM,IAAI,GAAG,oBAAoB,CAAC;gBAChC,QAAQ,EAAE,CAAC,eAAe,CAAC;gBAC3B,QAAQ,EAAE,EAAE;aACb,CAAC,CAAC;YACH,MAAM,QAAQ,GAAG,oCAAoC,CAAC;YACtD,MAAM,OAAO,GAAG,aAAa,EAAE,CAAC;YAEhC,MAAM,MAAM,GAAG,aAAa,CAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;YAEtD,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC;QAC5D,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;QACjC,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;YAC7C,MAAM,IAAI,GAAG,oBAAoB,CAAC;gBAChC,QAAQ,EAAE,EAAE;gBACZ,QAAQ,EAAE,CAAC,4BAA4B,EAAE,wBAAwB,CAAC;aACnE,CAAC,CAAC;YACH,MAAM,QAAQ,GAAG,iEAAiE,CAAC;YACnF,MAAM,OAAO,GAAG,aAAa,EAAE,CAAC;YAEhC,MAAM,MAAM,GAAG,aAAa,CAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;YAEtD,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,SAAS,CAAC,sBAAsB,CAAC,CAAC;YACjE,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC;QAC/D,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,sBAAsB,EAAE,GAAG,EAAE;QACpC,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;YACtD,MAAM,IAAI,GAAG,oBAAoB,CAAC;gBAChC,QAAQ,EAAE,CAAC,cAAc,EAAE,WAAW,EAAE,YAAY,EAAE,YAAY,CAAC;gBACnE,QAAQ,EAAE,EAAE;aACb,CAAC,CAAC;YACH,yDAAyD;YACzD,MAAM,QAAQ,GAAG,kDAAkD,CAAC;YACpE,MAAM,OAAO,GAAG,aAAa,EAAE,CAAC;YAEhC,MAAM,MAAM,GAAG,aAAa,CAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;YAEtD,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,YAAY;QACxD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;YAC7C,MAAM,IAAI,GAAG,oBAAoB,CAAC;gBAChC,QAAQ,EAAE,EAAE;gBACZ,QAAQ,EAAE,EAAE;aACb,CAAC,CAAC;YACH,MAAM,QAAQ,GAAG,eAAe,CAAC;YACjC,MAAM,OAAO,GAAG,aAAa,EAAE,CAAC;YAEhC,MAAM,MAAM,GAAG,aAAa,CAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;YAEtD,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;QACnC,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;YACrC,MAAM,IAAI,GAAG,oBAAoB,CAAC;gBAChC,QAAQ,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC;gBAC5D,QAAQ,EAAE,EAAE;aACb,CAAC,CAAC;YACH,0BAA0B;YAC1B,MAAM,QAAQ,GAAG,eAAe,CAAC;YACjC,MAAM,OAAO,GAAG,aAAa,EAAE,CAAC;YAEhC,MAAM,MAAM,GAAG,aAAa,CAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;YAEtD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2BAA2B,EAAE,GAAG,EAAE;YACnC,MAAM,IAAI,GAAG,oBAAoB,CAAC;gBAChC,QAAQ,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC;gBAC5D,QAAQ,EAAE,EAAE;aACb,CAAC,CAAC;YACH,0BAA0B;YAC1B,MAAM,QAAQ,GAAG,aAAa,CAAC;YAC/B,MAAM,OAAO,GAAG,aAAa,EAAE,CAAC;YAEhC,MAAM,MAAM,GAAG,aAAa,CAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;YAEtD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;IAChC,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;QAChD,MAAM,MAAM,GAAG,gBAAgB,CAAC,EAAE,CAAC,CAAC;QAEpC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACnC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;QAClD,MAAM,OAAO,GAAiB;YAC5B;gBACE,IAAI,EAAE,oBAAoB,EAAE;gBAC5B,OAAO,EAAE,aAAa,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;gBAC1E,eAAe,EAAE,EAAE;gBACnB,cAAc,EAAE,EAAE;gBAClB,QAAQ,EAAE,EAAE;gBACZ,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,MAAM,EAAE,IAAI;aACb;YACD;gBACE,IAAI,EAAE,oBAAoB,EAAE;gBAC5B,OAAO,EAAE,aAAa,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;gBAC1E,eAAe,EAAE,EAAE;gBACnB,cAAc,EAAE,EAAE;gBAClB,QAAQ,EAAE,EAAE;gBACZ,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,MAAM,EAAE,KAAK;aACd;SACF,CAAC;QAEF,MAAM,UAAU,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;QAE7C,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,gBAAgB;QACtD,MAAM,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,oBAAoB;QAC/D,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,WAAW,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,oBAAoB;IACxE,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;IACjC,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;QACrC,MAAM,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACxC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;QAC/C,MAAM,OAAO,GAAiB;YAC5B,EAAE,MAAM,EAAE,IAAI,EAAgB;YAC9B,EAAE,MAAM,EAAE,IAAI,EAAgB;YAC9B,EAAE,MAAM,EAAE,KAAK,EAAgB;YAC/B,EAAE,MAAM,EAAE,IAAI,EAAgB;SAC/B,CAAC;QAEF,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,YAAY;IAC3D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;QACzC,MAAM,OAAO,GAAiB;YAC5B,EAAE,MAAM,EAAE,IAAI,EAAgB;YAC9B,EAAE,MAAM,EAAE,IAAI,EAAgB;SAC/B,CAAC;QAEF,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;QACvC,MAAM,OAAO,GAAiB;YAC5B,EAAE,MAAM,EAAE,KAAK,EAAgB;YAC/B,EAAE,MAAM,EAAE,KAAK,EAAgB;SAChC,CAAC;QAEF,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/engine/enhanced-test-prompt-builder.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Enhanced test prompt builder for skill test generation.
+ *
+ * Builds enriched prompts by combining skill content with analysis
+ * from skill-creator (capabilities, concepts, edge cases, testing patterns).
+ */
+import type { SkillAnalysis } from './skill-creator-invoker.js';
+/**
+ * Build enhanced prompt with skill analysis context.
+ *
+ * When analysis is available, enriches the prompt with:
+ * - Extracted capabilities and concepts
+ * - Edge cases to consider
+ * - Claude Code testing patterns
+ * - Expected tool invocations
+ *
+ * When analysis is null (graceful degradation), builds basic prompt.
+ *
+ * @param skillContent - Formatted skill content from SkillContentCollector
+ * @param analysis - SkillAnalysis from skill-creator, or null for basic mode
+ * @returns Enhanced prompt string for test generation
+ */
+export declare function buildEnhancedTestPrompt(skillContent: string, analysis: SkillAnalysis | null): string;
+/**
+ * Extract concepts from analysis for test definition.
+ *
+ * Merges key concepts from analysis with any existing concepts.
+ */
+export declare function mergeConceptsFromAnalysis(existingConcepts: string[], analysis: SkillAnalysis | null): string[];
+//# sourceMappingURL=enhanced-test-prompt-builder.d.ts.map

package/dist/engine/enhanced-test-prompt-builder.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"enhanced-test-prompt-builder.d.ts","sourceRoot":"","sources":["../../src/engine/enhanced-test-prompt-builder.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAuDhE;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,uBAAuB,CACrC,YAAY,EAAE,MAAM,EACpB,QAAQ,EAAE,aAAa,GAAG,IAAI,GAC7B,MAAM,CA8CR;AAuBD;;;;GAIG;AACH,wBAAgB,yBAAyB,CACvC,gBAAgB,EAAE,MAAM,EAAE,EAC1B,QAAQ,EAAE,aAAa,GAAG,IAAI,GAC7B,MAAM,EAAE,CAOV"}

package/dist/engine/enhanced-test-prompt-builder.js

@@ -0,0 +1,134 @@
+/**
+ * Format skill analysis section for the prompt.
+ */
+function formatSkillAnalysis(analysis) {
+    const sections = [];
+    if (analysis.capabilities.length > 0) {
+        sections.push('### Capabilities');
+        for (const cap of analysis.capabilities) {
+            sections.push(`- ${cap}`);
+        }
+    }
+    if (analysis.keyConcepts.length > 0) {
+        sections.push('\n### Key Concepts');
+        for (const concept of analysis.keyConcepts) {
+            sections.push(`- ${concept}`);
+        }
+    }
+    if (analysis.edgeCases.length > 0) {
+        sections.push('\n### Edge Cases to Test');
+        for (const edge of analysis.edgeCases) {
+            sections.push(`- ${edge}`);
+        }
+    }
+    return sections.join('\n');
+}
+/**
+ * Format Claude Code testing context section.
+ */
+function formatTestingContext(analysis) {
+    const sections = [];
+    if (analysis.testingPatterns.length > 0) {
+        sections.push('### Testing Patterns');
+        for (const pattern of analysis.testingPatterns) {
+            sections.push(`- ${pattern}`);
+        }
+    }
+    if (analysis.toolInvocationExpectations.length > 0) {
+        sections.push('\n### Expected Tool Invocations');
+        for (const tool of analysis.toolInvocationExpectations) {
+            sections.push(`- ${tool}`);
+        }
+    }
+    return sections.join('\n');
+}
+/**
+ * Build enhanced prompt with skill analysis context.
+ *
+ * When analysis is available, enriches the prompt with:
+ * - Extracted capabilities and concepts
+ * - Edge cases to consider
+ * - Claude Code testing patterns
+ * - Expected tool invocations
+ *
+ * When analysis is null (graceful degradation), builds basic prompt.
+ *
+ * @param skillContent - Formatted skill content from SkillContentCollector
+ * @param analysis - SkillAnalysis from skill-creator, or null for basic mode
+ * @returns Enhanced prompt string for test generation
+ */
+export function buildEnhancedTestPrompt(skillContent, analysis) {
+    const hasAnalysis = analysis && (analysis.capabilities.length > 0 ||
+        analysis.keyConcepts.length > 0 ||
+        analysis.edgeCases.length > 0);
+    if (!hasAnalysis) {
+        // Fallback to basic prompt (same as original TEST_GENERATION_PROMPT)
+        return buildBasicTestPrompt(skillContent);
+    }
+    const skillAnalysisSection = formatSkillAnalysis(analysis);
+    const testingContextSection = formatTestingContext(analysis);
+    return `You must respond with ONLY a JSON object. No explanation, no markdown code blocks, just raw JSON.
+
+## Skill Analysis
+${skillAnalysisSection}
+
+## Claude Code Testing Context
+${testingContextSection}
+
+## Skill Content
+${skillContent}
+
+## Output Format
+{"skill_name":"<name>","tests":[{
+  "name":"<skill>-<topic>",
+  "test_type":"knowledge"|"task",
+  "concepts":["extracted from analysis above"],
+  "timeout":600|1800,
+  "prompt":"...",
+  "expected_items":["..."]
+}]}
+
+## Rules
+- Generate 2-4 tests: at least 1 knowledge + 1 task test
+- Use concepts from the Skill Analysis section above
+- Consider edge cases listed above when designing tests
+- Include expected tool invocations in task test expected_items
+- timeout: 600 (knowledge/10min), 1800 (task/30min)
+- 4-8 expected_items per test
+- Test names should be descriptive: <skill-name>-<topic>
+
+JSON:`;
+}
+/**
+ * Build basic test prompt without analysis (graceful degradation).
+ */
+function buildBasicTestPrompt(skillContent) {
+    return `You must respond with ONLY a JSON object. No explanation, no markdown code blocks, just raw JSON.
+
+Generate tests for this skill. Output format:
+{"skill_name":"<name>","tests":[{"name":"<skill>-<topic>","test_type":"knowledge"|"task","concepts":["..."],"timeout":600|1800,"prompt":"...","expected_items":["..."]}]}
+
+Rules:
+- 2-4 tests, at least 1 knowledge + 1 task
+- Extract concepts from Key Concepts Index or section headers
+- timeout: 600 (knowledge/10min), 1800 (task/30min)
+- 4-8 expected_items per test
+
+Skill content:
+${skillContent}
+
+JSON:`;
+}
+/**
+ * Extract concepts from analysis for test definition.
+ *
+ * Merges key concepts from analysis with any existing concepts.
+ */
+export function mergeConceptsFromAnalysis(existingConcepts, analysis) {
+    if (!analysis || analysis.keyConcepts.length === 0) {
+        return existingConcepts;
+    }
+    const merged = new Set([...existingConcepts, ...analysis.keyConcepts]);
+    return Array.from(merged);
+}
+//# sourceMappingURL=enhanced-test-prompt-builder.js.map

package/dist/engine/enhanced-test-prompt-builder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"enhanced-test-prompt-builder.js","sourceRoot":"","sources":["../../src/engine/enhanced-test-prompt-builder.ts"],"names":[],"mappings":"AAQA;;GAEG;AACH,SAAS,mBAAmB,CAAC,QAAuB;IAClD,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,IAAI,QAAQ,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrC,QAAQ,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAClC,KAAK,MAAM,GAAG,IAAI,QAAQ,CAAC,YAAY,EAAE,CAAC;YACxC,QAAQ,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,IAAI,QAAQ,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpC,QAAQ,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;QACpC,KAAK,MAAM,OAAO,IAAI,QAAQ,CAAC,WAAW,EAAE,CAAC;YAC3C,QAAQ,CAAC,IAAI,CAAC,KAAK,OAAO,EAAE,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;IAED,IAAI,QAAQ,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClC,QAAQ,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QAC1C,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,SAAS,EAAE,CAAC;YACtC,QAAQ,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;QAC7B,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC7B,CAAC;AAED;;GAEG;AACH,SAAS,oBAAoB,CAAC,QAAuB;IACnD,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,IAAI,QAAQ,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxC,QAAQ,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;QACtC,KAAK,MAAM,OAAO,IAAI,QAAQ,CAAC,eAAe,EAAE,CAAC;YAC/C,QAAQ,CAAC,IAAI,CAAC,KAAK,OAAO,EAAE,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;IAED,IAAI,QAAQ,CAAC,0BAA0B,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACnD,QAAQ,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;QACjD,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,0BAA0B,EAAE,CAAC;YACvD,QAAQ,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;QAC7B,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC7B,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,uBAAuB,CACrC,YAAoB,EACpB,QAA8B;IAE9B,MAAM,WAAW,GAAG,QAAQ,IAAI,CAC9B,QAAQ,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC;QAChC,QAAQ,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC;QAC/B,QAAQ,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAC9B,CAAC;IAEF,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,qEAAqE;QACrE,OAAO,oBAAoB,CAAC,YAAY,CAAC,CAAC;IAC5C,CAAC;IAED,MAAM,oBAAoB,GAAG,mBAAmB,CAAC,QAAS,CAAC,CAAC;IAC5D,MAAM,qBAAqB,GAAG,oBAAoB,CAAC,QAAS,CAAC,CAAC;IAE9D,OAAO;;;EAGP,oBAAoB;;;EAGpB,qBAAqB;;;EAGrB,YAAY;;;;;;;;;;;;;;;;;;;;;MAqBR,CAAC;AACP,CAAC;AAED;;GAEG;AACH,SAAS,oBAAoB,CAAC,YAAoB;IAChD,OAAO;;;;;;;;;;;;EAYP,YAAY;;MAER,CAAC;AACP,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,yBAAyB,CACvC,gBAA0B,EAC1B,QAA8B;IAE9B,IAAI,CAAC,QAAQ,IAAI,QAAQ,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnD,OAAO,gBAAgB,CAAC;IAC1B,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,gBAAgB,EAAE,GAAG,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC;IACvE,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC5B,CAAC"}