@skilljack/evals 1.0.0

package/dist/src/report/report.js

@@ -0,0 +1,216 @@
+/**
+ * Report generation for skill evaluation results.
+ *
+ * Generates markdown and JSON reports from combined evaluation scores.
+ */
+import * as fs from 'fs/promises';
+import * as path from 'path';
+import { loadConfigSync } from '../config.js';
+/**
+ * Generate a markdown report from evaluation results.
+ */
+export async function generateReport(evaluation, results, scores, outputPath, metadata) {
+    const config = loadConfigSync();
+    const totalTasks = evaluation.tasks.length;
+    const summary = computeSummary(results, scores);
+    const failureBreakdown = computeFailureBreakdown(scores);
+    // Determine pass/fail
+    const discoveryPassed = summary.discoveryAccuracy >= config.discoveryThreshold;
+    const scorePassed = summary.avgAdherence >= config.scoreThreshold && summary.avgOutputQuality >= config.scoreThreshold;
+    const passed = discoveryPassed && scorePassed;
+    // Build metadata section
+    let metaSection = '';
+    if (metadata) {
+        const metaLines = [`**Skill Path:** \`${metadata.skillPath}\``];
+        if (metadata.gitCommit) {
+            metaLines.push(`**Git:** ${metadata.gitBranch}@${metadata.gitCommit}`);
+        }
+        if (metadata.version)
+            metaLines.push(`**Version:** ${metadata.version}`);
+        metaLines.push(`**Agent Model:** ${metadata.agentModel}`);
+        metaLines.push(`**Judge Model:** ${metadata.judgeModel}`);
+        metaSection = metaLines.join('\n') + '\n';
+    }
+    // Build report
+    let report = `# Skill Evaluation Report: ${evaluation.skillName}
+
+**Generated:** ${new Date().toISOString()}
+**Total Tasks:** ${totalTasks}
+**Result:** ${passed ? 'PASS' : 'FAIL'}
+${metaSection}
+---
+
+## Summary
+
+| Metric | Value | Threshold | Status |
+|--------|-------|-----------|--------|
+| **Discovery Accuracy** | ${(summary.discoveryAccuracy * 100).toFixed(1)}% | ${(config.discoveryThreshold * 100).toFixed(0)}% | ${discoveryPassed ? 'PASS' : 'FAIL'} |
+| **Avg Adherence Score** | ${summary.avgAdherence.toFixed(2)}/5.0 | ${config.scoreThreshold.toFixed(1)} | ${summary.avgAdherence >= config.scoreThreshold ? 'PASS' : 'FAIL'} |
+| **Avg Output Quality** | ${summary.avgOutputQuality.toFixed(2)}/5.0 | ${config.scoreThreshold.toFixed(1)} | ${summary.avgOutputQuality >= config.scoreThreshold ? 'PASS' : 'FAIL'} |
+| **Avg Weighted Score** | ${summary.avgWeightedScore.toFixed(2)} | | |
+| **Total Duration** | ${(summary.totalDurationMs / 1000).toFixed(1)}s | | |
+| **Total Cost** | $${summary.totalCostUsd.toFixed(4)} | | |
+
+## Failure Analysis
+
+| Category | Count | Percentage |
+|----------|-------|------------|
+`;
+    for (const fb of failureBreakdown) {
+        const displayCat = fb.category === 'none' ? 'No Failure' : formatCategory(fb.category);
+        report += `| ${displayCat} | ${fb.count} | ${fb.percentage.toFixed(1)}% |\n`;
+    }
+    report += `\n---\n\n## Task Details\n\n`;
+    for (let i = 0; i < evaluation.tasks.length; i++) {
+        const task = evaluation.tasks[i];
+        const result = results[i];
+        const score = scores[i];
+        const loadedSkills = result.skillLoads.length > 0
+            ? result.skillLoads.map((s) => `\`${s}\``).join(', ')
+            : 'None';
+        report += `### Task ${i + 1}: ${task.id}
+
+**Prompt:** ${task.prompt}
+
+**Expected Skill:** \`${task.expectedSkillLoad}\`
+**Loaded Skills:** ${loadedSkills}
+
+#### Scores
+
+| Dimension | Score | Status |
+|-----------|-------|--------|
+| Discovery | ${Math.round(score.discovery)} | ${score.discovery >= 1 ? 'PASS' : 'FAIL'} |
+| Adherence | ${score.adherence}/5 | ${score.adherence >= 4 ? 'PASS' : 'FAIL'} |
+| Output Quality | ${score.outputQuality}/5 | ${score.outputQuality >= 4 ? 'PASS' : 'FAIL'} |
+| **Weighted** | **${score.weightedScore.toFixed(2)}** | |
+
+**Failure Category:** ${formatCategory(score.failureCategory)}
+`;
+        // Show deterministic results if available
+        if (score.deterministic) {
+            report += `\n**Deterministic Check:** ${score.deterministic.passed ? 'PASS' : 'FAIL'}\n`;
+            for (const detail of score.deterministic.details) {
+                report += `- ${detail}\n`;
+            }
+        }
+        report += `\n**Reasoning:** ${score.reasoning || 'No reasoning provided'}
+
+<details>
+<summary>Agent Output (click to expand)</summary>
+
+\`\`\`
+${result.output.slice(0, config.reportOutputTruncation) || '(no output)'}
+\`\`\`
+
+</details>
+
+**Metrics:** Duration: ${(result.durationMs / 1000).toFixed(1)}s | Turns: ${result.numTurns} | Cost: $${result.costUsd.toFixed(4)}
+
+---
+
+`;
+    }
+    if (outputPath) {
+        await fs.mkdir(path.dirname(outputPath), { recursive: true });
+        await fs.writeFile(outputPath, report);
+        console.log(`Report saved to: ${outputPath}`);
+    }
+    return report;
+}
+/**
+ * Generate JSON report for programmatic analysis.
+ */
+export async function generateJsonResults(evaluation, results, scores, outputPath, metadata) {
+    const config = loadConfigSync();
+    const summary = computeSummary(results, scores);
+    const failureBreakdown = computeFailureBreakdown(scores);
+    const discoveryPassed = summary.discoveryAccuracy >= config.discoveryThreshold;
+    const scorePassed = summary.avgAdherence >= config.scoreThreshold && summary.avgOutputQuality >= config.scoreThreshold;
+    const passed = discoveryPassed && scorePassed;
+    const failureReasons = [];
+    if (!discoveryPassed) {
+        failureReasons.push(`Discovery rate ${(summary.discoveryAccuracy * 100).toFixed(1)}% below threshold ${(config.discoveryThreshold * 100).toFixed(0)}%`);
+    }
+    if (summary.avgAdherence < config.scoreThreshold) {
+        failureReasons.push(`Avg adherence ${summary.avgAdherence.toFixed(2)} below threshold ${config.scoreThreshold}`);
+    }
+    if (summary.avgOutputQuality < config.scoreThreshold) {
+        failureReasons.push(`Avg output quality ${summary.avgOutputQuality.toFixed(2)} below threshold ${config.scoreThreshold}`);
+    }
+    const report = {
+        skillName: evaluation.skillName,
+        timestamp: new Date().toISOString(),
+        passed,
+        failureReasons,
+        metadata: metadata ? {
+            skillPath: metadata.skillPath,
+            gitCommit: metadata.gitCommit,
+            gitBranch: metadata.gitBranch,
+            version: metadata.version,
+            agentModel: metadata.agentModel,
+            judgeModel: metadata.judgeModel,
+        } : undefined,
+        summary,
+        failureBreakdown,
+        tasks: evaluation.tasks.map((task, i) => ({
+            task,
+            result: results[i],
+            score: scores[i],
+        })),
+    };
+    if (outputPath) {
+        await fs.mkdir(path.dirname(outputPath), { recursive: true });
+        await fs.writeFile(outputPath, JSON.stringify(report, null, 2));
+        console.log(`JSON results saved to: ${outputPath}`);
+    }
+    return report;
+}
+/**
+ * Compute summary statistics from combined scores.
+ */
+export function computeSummary(results, scores) {
+    const totalTasks = scores.length;
+    const discoveryCorrect = scores.filter((s) => s.discovery >= 1).length;
+    return {
+        totalTasks,
+        discoveryAccuracy: totalTasks > 0 ? discoveryCorrect / totalTasks : 0,
+        avgAdherence: totalTasks > 0
+            ? scores.reduce((sum, s) => sum + s.adherence, 0) / totalTasks
+            : 0,
+        avgOutputQuality: totalTasks > 0
+            ? scores.reduce((sum, s) => sum + s.outputQuality, 0) / totalTasks
+            : 0,
+        avgWeightedScore: totalTasks > 0
+            ? scores.reduce((sum, s) => sum + s.weightedScore, 0) / totalTasks
+            : 0,
+        totalDurationMs: results.reduce((sum, r) => sum + r.durationMs, 0),
+        totalCostUsd: results.reduce((sum, r) => sum + r.costUsd, 0),
+    };
+}
+/**
+ * Compute failure category breakdown.
+ */
+export function computeFailureBreakdown(scores) {
+    const counts = new Map();
+    for (const score of scores) {
+        const cat = score.failureCategory || 'none';
+        counts.set(cat, (counts.get(cat) || 0) + 1);
+    }
+    const total = scores.length;
+    return Array.from(counts.entries())
+        .map(([category, count]) => ({
+        category: category,
+        count,
+        percentage: total > 0 ? (count / total) * 100 : 0,
+    }))
+        .sort((a, b) => b.count - a.count);
+}
+function formatCategory(cat) {
+    if (cat === 'none')
+        return 'No Failure';
+    return cat
+        .split('_')
+        .map((w) => w.charAt(0).toUpperCase() + w.slice(1))
+        .join(' ');
+}
+//# sourceMappingURL=report.js.map

package/dist/src/report/report.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"report.js","sourceRoot":"","sources":["../../../src/report/report.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,MAAM,aAAa,CAAC;AAClC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAW7B,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAE9C;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,UAA2B,EAC3B,OAAqB,EACrB,MAAuB,EACvB,UAAmB,EACnB,QAAyB;IAEzB,MAAM,MAAM,GAAG,cAAc,EAAE,CAAC;IAChC,MAAM,UAAU,GAAG,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC;IAC3C,MAAM,OAAO,GAAG,cAAc,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAChD,MAAM,gBAAgB,GAAG,uBAAuB,CAAC,MAAM,CAAC,CAAC;IAEzD,sBAAsB;IACtB,MAAM,eAAe,GAAG,OAAO,CAAC,iBAAiB,IAAI,MAAM,CAAC,kBAAkB,CAAC;IAC/E,MAAM,WAAW,GAAG,OAAO,CAAC,YAAY,IAAI,MAAM,CAAC,cAAc,IAAI,OAAO,CAAC,gBAAgB,IAAI,MAAM,CAAC,cAAc,CAAC;IACvH,MAAM,MAAM,GAAG,eAAe,IAAI,WAAW,CAAC;IAE9C,yBAAyB;IACzB,IAAI,WAAW,GAAG,EAAE,CAAC;IACrB,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,SAAS,GAAG,CAAC,qBAAqB,QAAQ,CAAC,SAAS,IAAI,CAAC,CAAC;QAChE,IAAI,QAAQ,CAAC,SAAS,EAAE,CAAC;YACvB,SAAS,CAAC,IAAI,CAAC,YAAY,QAAQ,CAAC,SAAS,IAAI,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC;QACzE,CAAC;QACD,IAAI,QAAQ,CAAC,OAAO;YAAE,SAAS,CAAC,IAAI,CAAC,gBAAgB,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC;QACzE,SAAS,CAAC,IAAI,CAAC,oBAAoB,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;QAC1D,SAAS,CAAC,IAAI,CAAC,oBAAoB,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;QAC1D,WAAW,GAAG,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAC5C,CAAC;IAED,eAAe;IACf,IAAI,MAAM,GAAG,8BAA8B,UAAU,CAAC,SAAS;;iBAEhD,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;mBACtB,UAAU;cACf,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;EACpC,WAAW;;;;;;;6BAOgB,CAAC,OAAO,CAAC,iBAAiB,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,kBAAkB,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;8BACtI,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,MAAM,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,OAAO,CAAC,YAAY,IAAI,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;6BAC/I,OAAO,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,MAAM,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,OAAO,CAAC,gBAAgB,IAAI,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;6BACtJ,OAAO,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC;yBACvC,CAAC,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;sBAC9C,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC;;;;;;CAMpD,CAAC;IAEA,KAAK,MAAM,EAAE,IAAI,gBAAgB,EAAE,CAAC;QAClC,MAAM,UAAU,GAAG,EAAE,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,cAAc,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC;QACvF,MAAM,IAAI,KAAK,UAAU,MAAM,EAAE,CAAC,KAAK,MAAM,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC;IAC/E,CAAC;IAED,MAAM,IAAI,8BAA8B,CAAC;IAEzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACjD,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACjC,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAC1B,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAExB,MAAM,YAAY,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC;YAC/C,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;YACrD,CAAC,CAAC,MAAM,CAAC;QAEX,MAAM,IAAI,YAAY,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,EAAE;;cAE7B,IAAI,CAAC,MAAM;;wBAED,IAAI,CAAC,iBAAiB;qBACzB,YAAY;;;;;;gBAMjB,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,SAAS,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;gBACvE,KAAK,CAAC,SAAS,QAAQ,KAAK,CAAC,SAAS,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;qBACxD,KAAK,CAAC,aAAa,QAAQ,KAAK,CAAC,aAAa,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;qBACrE,KAAK,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC;;wBAE3B,cAAc,CAAC,KAAK,CAAC,eAAe,CAAC;CAC5D,CAAC;QAEE,0CAA0C;QAC1C,IAAI,KAAK,CAAC,aAAa,EAAE,CAAC;YACxB,MAAM,IAAI,8BAA8B,KAAK,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC;YACzF,KAAK,MAAM,MAAM,IAAI,KAAK,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC;gBACjD,MAAM,IAAI,KAAK,MAAM,IAAI,CAAC;YAC5B,CAAC;QACH,CAAC;QAED,MAAM,IAAI,oBAAoB,KAAK,CAAC,SAAS,IAAI,uBAAuB;;;;;;EAM1E,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,sBAAsB,CAAC,IAAI,aAAa;;;;;yBAK/C,CAAC,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,cAAc,MAAM,CAAC,QAAQ,aAAa,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;;;;CAIhI,CAAC;IACA,CAAC;IAED,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9D,MAAM,EAAE,CAAC,SAAS,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QACvC,OAAO,CAAC,GAAG,CAAC,oBAAoB,UAAU,EAAE,CAAC,CAAC;IAChD,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,UAA2B,EAC3B,OAAqB,EACrB,MAAuB,EACvB,UAAmB,EACnB,QAAyB;IAEzB,MAAM,MAAM,GAAG,cAAc,EAAE,CAAC;IAChC,MAAM,OAAO,GAAG,cAAc,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAChD,MAAM,gBAAgB,GAAG,uBAAuB,CAAC,MAAM,CAAC,CAAC;IAEzD,MAAM,eAAe,GAAG,OAAO,CAAC,iBAAiB,IAAI,MAAM,CAAC,kBAAkB,CAAC;IAC/E,MAAM,WAAW,GAAG,OAAO,CAAC,YAAY,IAAI,MAAM,CAAC,cAAc,IAAI,OAAO,CAAC,gBAAgB,IAAI,MAAM,CAAC,cAAc,CAAC;IACvH,MAAM,MAAM,GAAG,eAAe,IAAI,WAAW,CAAC;IAE9C,MAAM,cAAc,GAAa,EAAE,CAAC;IACpC,IAAI,CAAC,eAAe,EAAE,CAAC;QACrB,cAAc,CAAC,IAAI,CACjB,kBAAkB,CAAC,OAAO,CAAC,iBAAiB,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,qBAAqB,CAAC,MAAM,CAAC,kBAAkB,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CACnI,CAAC;IACJ,CAAC;IACD,IAAI,OAAO,CAAC,YAAY,GAAG,MAAM,CAAC,cAAc,EAAE,CAAC;QACjD,cAAc,CAAC,IAAI,CACjB,iBAAiB,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,oBAAoB,MAAM,CAAC,cAAc,EAAE,CAC5F,CAAC;IACJ,CAAC;IACD,IAAI,OAAO,CAAC,gBAAgB,GAAG,MAAM,CAAC,cAAc,EAAE,CAAC;QACrD,cAAc,CAAC,IAAI,CACjB,sBAAsB,OAAO,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC,oBAAoB,MAAM,CAAC,cAAc,EAAE,CACrG,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAqB;QAC/B,SAAS,EAAE,UAAU,CAAC,SAAS;QAC/B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,MAAM;QACN,cAAc;QACd,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;YACnB,SAAS,EAAE,QAAQ,CAAC,SAAS;YAC7B,SAAS,EAAE,QAAQ,CAAC,SAAS;YAC7B,SAAS,EAAE,QAAQ,CAAC,SAAS;YAC7B,OAAO,EAAE,QAAQ,CAAC,OAAO;YACzB,UAAU,EAAE,QAAQ,CAAC,UAAU;YAC/B,UAAU,EAAE,QAAQ,CAAC,UAAU;SAChC,CAAC,CAAC,CAAC,SAAS;QACb,OAAO;QACP,gBAAgB;QAChB,KAAK,EAAE,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YACxC,IAAI;YACJ,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;YAClB,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;SACjB,CAAC,CAAC;KACJ,CAAC;IAEF,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9D,MAAM,EAAE,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAChE,OAAO,CAAC,GAAG,CAAC,0BAA0B,UAAU,EAAE,CAAC,CAAC;IACtD,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAC5B,OAAqB,EACrB,MAAuB;IAEvB,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC;IACjC,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;IAEvE,OAAO;QACL,UAAU;QACV,iBAAiB,EAAE,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,gBAAgB,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;QACrE,YAAY,EAAE,UAAU,GAAG,CAAC;YAC1B,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,GAAG,UAAU;YAC9D,CAAC,CAAC,CAAC;QACL,gBAAgB,EAAE,UAAU,GAAG,CAAC;YAC9B,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,aAAa,EAAE,CAAC,CAAC,GAAG,UAAU;YAClE,CAAC,CAAC,CAAC;QACL,gBAAgB,EAAE,UAAU,GAAG,CAAC;YAC9B,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,aAAa,EAAE,CAAC,CAAC,GAAG,UAAU;YAClE,CAAC,CAAC,CAAC;QACL,eAAe,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC;QAClE,YAAY,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;KAC7D,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,uBAAuB,CAAC,MAAuB;IAC7D,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,GAAG,GAAG,KAAK,CAAC,eAAe,IAAI,MAAM,CAAC;QAC5C,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC9C,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC;IAC5B,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;SAChC,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC;QAC3B,QAAQ,EAAE,QAA2B;QACrC,KAAK;QACL,UAAU,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;KAClD,CAAC,CAAC;SACF,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;AACvC,CAAC;AAED,SAAS,cAAc,CAAC,GAAW;IACjC,IAAI,GAAG,KAAK,MAAM;QAAE,OAAO,YAAY,CAAC;IACxC,OAAO,GAAG;SACP,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;SAClD,IAAI,CAAC,GAAG,CAAC,CAAC;AACf,CAAC"}

package/dist/src/runner/runner.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Skill Evaluation Runner - Claude Agent SDK
+ *
+ * Runs evaluation tasks against an agent using the Claude Agent SDK.
+ * Supports local skill delivery (.claude/skills/) with both Anthropic API
+ * and Bedrock (via CLAUDE_CODE_USE_BEDROCK=1 env var).
+ *
+ * Security: Uses permissionMode 'bypassPermissions' for automated execution,
+ * with file writes restricted via canUseTool callback to allowedWriteDirs only.
+ */
+import type { EvalTask, SkillEvaluation, TaskResult, RunnerOptions } from '../types.js';
+import type { SessionLogger } from '../session/session-logger.js';
+export declare class SkillEvalRunner {
+    private options;
+    constructor(options?: RunnerOptions);
+    /**
+     * Execute a single evaluation task.
+     */
+    runTask(task: EvalTask, logger?: SessionLogger): Promise<TaskResult>;
+    /**
+     * Execute a task with timeout protection.
+     */
+    runTaskWithTimeout(task: EvalTask, timeoutMs?: number, logger?: SessionLogger): Promise<TaskResult>;
+    /**
+     * Run all tasks in an evaluation suite.
+     */
+    runAll(evaluation: SkillEvaluation, createLogger?: (task: EvalTask) => SessionLogger): Promise<TaskResult[]>;
+}
+//# sourceMappingURL=runner.d.ts.map

package/dist/src/runner/runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../../src/runner/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAGH,OAAO,KAAK,EACV,QAAQ,EACR,eAAe,EAEf,UAAU,EACV,aAAa,EACd,MAAM,aAAa,CAAC;AASrB,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAC;AAElE,qBAAa,eAAe;IAC1B,OAAO,CAAC,OAAO,CAA0B;gBAE7B,OAAO,GAAE,aAAkB;IAavC;;OAEG;IACG,OAAO,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,UAAU,CAAC;IA4H1E;;OAEG;IACG,kBAAkB,CACtB,IAAI,EAAE,QAAQ,EACd,SAAS,CAAC,EAAE,MAAM,EAClB,MAAM,CAAC,EAAE,aAAa,GACrB,OAAO,CAAC,UAAU,CAAC;IAgCtB;;OAEG;IACG,MAAM,CACV,UAAU,EAAE,eAAe,EAC3B,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE,QAAQ,KAAK,aAAa,GAC/C,OAAO,CAAC,UAAU,EAAE,CAAC;CA6CzB"}

package/dist/src/runner/runner.js

@@ -0,0 +1,211 @@
+/**
+ * Skill Evaluation Runner - Claude Agent SDK
+ *
+ * Runs evaluation tasks against an agent using the Claude Agent SDK.
+ * Supports local skill delivery (.claude/skills/) with both Anthropic API
+ * and Bedrock (via CLAUDE_CODE_USE_BEDROCK=1 env var).
+ *
+ * Security: Uses permissionMode 'bypassPermissions' for automated execution,
+ * with file writes restricted via canUseTool callback to allowedWriteDirs only.
+ */
+import { query } from '@anthropic-ai/claude-agent-sdk';
+import { isAssistantMessage, isResultMessage, isTextBlock, isToolUseBlock, } from '../types.js';
+import { createToolPolicy } from './security.js';
+import { loadConfigSync } from '../config.js';
+export class SkillEvalRunner {
+    options;
+    constructor(options = {}) {
+        const config = loadConfigSync();
+        this.options = {
+            cwd: options.cwd ?? process.cwd(),
+            parallel: options.parallel ?? false,
+            model: options.model ?? config.defaultAgentModel,
+            settingSources: options.settingSources ?? ['project'],
+            countReadAsFallback: options.countReadAsFallback ?? false,
+            allowedWriteDirs: options.allowedWriteDirs ?? config.allowedWriteDirs,
+        };
+    }
+    /**
+     * Execute a single evaluation task.
+     */
+    async runTask(task, logger) {
+        const skillLoads = [];
+        const toolCalls = [];
+        const startTime = Date.now();
+        try {
+            let resultOutput = '';
+            let resultDurationMs = 0;
+            let resultNumTurns = 0;
+            let resultCostUsd = 0;
+            const toolPolicy = createToolPolicy(this.options.allowedWriteDirs, this.options.cwd);
+            const q = query({
+                prompt: task.prompt,
+                options: {
+                    cwd: this.options.cwd,
+                    model: this.options.model,
+                    systemPrompt: { type: 'preset', preset: 'claude_code' },
+                    settingSources: this.options.settingSources,
+                    allowedTools: [
+                        'Read', 'Write', 'Edit',
+                        'Glob', 'Grep', 'Bash',
+                        'Skill', 'Task',
+                    ],
+                    permissionMode: 'bypassPermissions',
+                    canUseTool: toolPolicy,
+                },
+            });
+            for await (const message of q) {
+                // Process assistant messages
+                if (isAssistantMessage(message)) {
+                    const content = message.message.content;
+                    logger?.addAssistantMessage(content);
+                    for (const block of content) {
+                        if (isTextBlock(block)) {
+                            resultOutput += block.text;
+                            logger?.addTextMessage(block.text);
+                        }
+                        if (isToolUseBlock(block)) {
+                            const toolName = block.name;
+                            const toolInput = block.input;
+                            toolCalls.push({
+                                tool: toolName,
+                                toolUseId: block.id,
+                                timestamp: Date.now(),
+                                input: toolInput,
+                            });
+                            logger?.addToolUse(toolName, toolInput);
+                            // Detect skill loading via Skill tool
+                            if (toolName === 'Skill') {
+                                const skillName = toolInput.skill || '';
+                                if (skillName) {
+                                    skillLoads.push(skillName);
+                                }
+                            }
+                            // Optionally detect via Read calls to SKILL.md
+                            if (this.options.countReadAsFallback && toolName === 'Read') {
+                                const filePath = toolInput.file_path || '';
+                                if (filePath.includes('SKILL.md') || filePath.includes('/skills/')) {
+                                    const match = filePath.match(/skills\/([^/]+)/);
+                                    if (match) {
+                                        skillLoads.push(match[1]);
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+                // Capture final metrics from result message
+                if (isResultMessage(message)) {
+                    resultDurationMs = message.duration_ms ?? 0;
+                    resultNumTurns = message.num_turns ?? 0;
+                    resultCostUsd = message.total_cost_usd ?? 0;
+                    if (message.result) {
+                        resultOutput = message.result;
+                    }
+                }
+            }
+            return {
+                taskId: task.id,
+                prompt: task.prompt,
+                output: resultOutput,
+                durationMs: resultDurationMs || (Date.now() - startTime),
+                numTurns: resultNumTurns,
+                costUsd: resultCostUsd,
+                skillLoads: [...new Set(skillLoads)],
+                toolCalls,
+                isError: false,
+                errorMessage: '',
+            };
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            logger?.markAsError(errorMessage);
+            return {
+                taskId: task.id,
+                prompt: task.prompt,
+                output: '',
+                durationMs: Date.now() - startTime,
+                numTurns: 0,
+                costUsd: 0,
+                skillLoads: [],
+                toolCalls: [],
+                isError: true,
+                errorMessage,
+            };
+        }
+    }
+    /**
+     * Execute a task with timeout protection.
+     */
+    async runTaskWithTimeout(task, timeoutMs, logger) {
+        const config = loadConfigSync();
+        const timeout = timeoutMs ?? config.taskTimeoutMs;
+        const timeoutPromise = new Promise((_, reject) => {
+            setTimeout(() => reject(new Error(`Task ${task.id} timed out after ${timeout}ms`)), timeout);
+        });
+        try {
+            return await Promise.race([this.runTask(task, logger), timeoutPromise]);
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            logger?.markAsError(errorMessage);
+            return {
+                taskId: task.id,
+                prompt: task.prompt,
+                output: '',
+                durationMs: timeout,
+                numTurns: 0,
+                costUsd: 0,
+                skillLoads: [],
+                toolCalls: [],
+                isError: true,
+                errorMessage,
+            };
+        }
+    }
+    /**
+     * Run all tasks in an evaluation suite.
+     */
+    async runAll(evaluation, createLogger) {
+        if (this.options.parallel) {
+            const results = await Promise.allSettled(evaluation.tasks.map((task) => {
+                const logger = createLogger?.(task);
+                return this.runTaskWithTimeout(task, undefined, logger);
+            }));
+            return results.map((result, i) => {
+                if (result.status === 'fulfilled') {
+                    return result.value;
+                }
+                const task = evaluation.tasks[i];
+                return {
+                    taskId: task.id,
+                    prompt: task.prompt,
+                    output: '',
+                    durationMs: 0,
+                    numTurns: 0,
+                    costUsd: 0,
+                    skillLoads: [],
+                    toolCalls: [],
+                    isError: true,
+                    errorMessage: result.reason?.message || 'Unknown error',
+                };
+            });
+        }
+        const results = [];
+        for (const task of evaluation.tasks) {
+            console.log(`Running task ${task.id}: ${task.prompt.slice(0, 60)}...`);
+            const logger = createLogger?.(task);
+            const result = await this.runTaskWithTimeout(task, undefined, logger);
+            results.push(result);
+            if (result.isError) {
+                console.error(`  ERROR: ${result.errorMessage}`);
+            }
+            else {
+                console.log(`  Skills loaded: ${result.skillLoads.join(', ') || 'none'}`);
+                console.log(`  Duration: ${(result.durationMs / 1000).toFixed(1)}s | Cost: $${result.costUsd.toFixed(4)}`);
+            }
+        }
+        return results;
+    }
+}
+//# sourceMappingURL=runner.js.map

package/dist/src/runner/runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../../src/runner/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AAQvD,OAAO,EACL,kBAAkB,EAClB,eAAe,EACf,WAAW,EACX,cAAc,GACf,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAG9C,MAAM,OAAO,eAAe;IAClB,OAAO,CAA0B;IAEzC,YAAY,UAAyB,EAAE;QACrC,MAAM,MAAM,GAAG,cAAc,EAAE,CAAC;QAEhC,IAAI,CAAC,OAAO,GAAG;YACb,GAAG,EAAE,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE;YACjC,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,KAAK;YACnC,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,MAAM,CAAC,iBAAiB;YAChD,cAAc,EAAE,OAAO,CAAC,cAAc,IAAI,CAAC,SAAS,CAAC;YACrD,mBAAmB,EAAE,OAAO,CAAC,mBAAmB,IAAI,KAAK;YACzD,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,IAAI,MAAM,CAAC,gBAAgB;SACtE,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO,CAAC,IAAc,EAAE,MAAsB;QAClD,MAAM,UAAU,GAAa,EAAE,CAAC;QAChC,MAAM,SAAS,GAAqB,EAAE,CAAC;QACvC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE7B,IAAI,CAAC;YACH,IAAI,YAAY,GAAG,EAAE,CAAC;YACtB,IAAI,gBAAgB,GAAG,CAAC,CAAC;YACzB,IAAI,cAAc,GAAG,CAAC,CAAC;YACvB,IAAI,aAAa,GAAG,CAAC,CAAC;YAEtB,MAAM,UAAU,GAAG,gBAAgB,CACjC,IAAI,CAAC,OAAO,CAAC,gBAAgB,EAC7B,IAAI,CAAC,OAAO,CAAC,GAAG,CACjB,CAAC;YAEF,MAAM,CAAC,GAAG,KAAK,CAAC;gBACd,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,OAAO,EAAE;oBACP,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG;oBACrB,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK;oBACzB,YAAY,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE;oBACvD,cAAc,EAAE,IAAI,CAAC,OAAO,CAAC,cAAc;oBAC3C,YAAY,EAAE;wBACZ,MAAM,EAAE,OAAO,EAAE,MAAM;wBACvB,MAAM,EAAE,MAAM,EAAE,MAAM;wBACtB,OAAO,EAAE,MAAM;qBAChB;oBACD,cAAc,EAAE,mBAAmB;oBACnC,UAAU,EAAE,UAAU;iBACvB;aACF,CAAC,CAAC;YAEH,IAAI,KAAK,EAAE,MAAM,OAAO,IAAI,CAAC,EAAE,CAAC;gBAC9B,6BAA6B;gBAC7B,IAAI,kBAAkB,CAAC,OAAO,CAAC,EAAE,CAAC;oBAChC,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC;oBAExC,MAAM,EAAE,mBAAmB,CAAC,OAAoB,CAAC,CAAC;oBAElD,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;wBAC5B,IAAI,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC;4BACvB,YAAY,IAAI,KAAK,CAAC,IAAI,CAAC;4BAC3B,MAAM,EAAE,cAAc,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;wBACrC,CAAC;wBAED,IAAI,cAAc,CAAC,KAAK,CAAC,EAAE,CAAC;4BAC1B,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC;4BAC5B,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC;4BAE9B,SAAS,CAAC,IAAI,CAAC;gCACb,IAAI,EAAE,QAAQ;gCACd,SAAS,EAAE,KAAK,CAAC,EAAE;gCACnB,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;gCACrB,KAAK,EAAE,SAAS;6BACjB,CAAC,CAAC;4BAEH,MAAM,EAAE,UAAU,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;4BAExC,sCAAsC;4BACtC,IAAI,QAAQ,KAAK,OAAO,EAAE,CAAC;gCACzB,MAAM,SAAS,GAAI,SAAS,CAAC,KAAgB,IAAI,EAAE,CAAC;gCACpD,IAAI,SAAS,EAAE,CAAC;oCACd,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gCAC7B,CAAC;4BACH,CAAC;4BAED,+CAA+C;4BAC/C,IAAI,IAAI,CAAC,OAAO,CAAC,mBAAmB,IAAI,QAAQ,KAAK,MAAM,EAAE,CAAC;gCAC5D,MAAM,QAAQ,GAAI,SAAS,CAAC,SAAoB,IAAI,EAAE,CAAC;gCACvD,IAAI,QAAQ,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;oCACnE,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;oCAChD,IAAI,KAAK,EAAE,CAAC;wCACV,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;oCAC5B,CAAC;gCACH,CAAC;4BACH,CAAC;wBACH,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,4CAA4C;gBAC5C,IAAI,eAAe,CAAC,OAAO,CAAC,EAAE,CAAC;oBAC7B,gBAAgB,GAAG,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC;oBAC5C,cAAc,GAAG,OAAO,CAAC,SAAS,IAAI,CAAC,CAAC;oBACxC,aAAa,GAAG,OAAO,CAAC,cAAc,IAAI,CAAC,CAAC;oBAE5C,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;wBACnB,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC;oBAChC,CAAC;gBACH,CAAC;YACH,CAAC;YAED,OAAO;gBACL,MAAM,EAAE,IAAI,CAAC,EAAE;gBACf,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,MAAM,EAAE,YAAY;gBACpB,UAAU,EAAE,gBAAgB,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;gBACxD,QAAQ,EAAE,cAAc;gBACxB,OAAO,EAAE,aAAa;gBACtB,UAAU,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC;gBACpC,SAAS;gBACT,OAAO,EAAE,KAAK;gBACd,YAAY,EAAE,EAAE;aACjB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC5E,MAAM,EAAE,WAAW,CAAC,YAAY,CAAC,CAAC;YAElC,OAAO;gBACL,MAAM,EAAE,IAAI,CAAC,EAAE;gBACf,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,MAAM,EAAE,EAAE;gBACV,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;gBAClC,QAAQ,EAAE,CAAC;gBACX,OAAO,EAAE,CAAC;gBACV,UAAU,EAAE,EAAE;gBACd,SAAS,EAAE,EAAE;gBACb,OAAO,EAAE,IAAI;gBACb,YAAY;aACb,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,kBAAkB,CACtB,IAAc,EACd,SAAkB,EAClB,MAAsB;QAEtB,MAAM,MAAM,GAAG,cAAc,EAAE,CAAC;QAChC,MAAM,OAAO,GAAG,SAAS,IAAI,MAAM,CAAC,aAAa,CAAC;QAElD,MAAM,cAAc,GAAG,IAAI,OAAO,CAAa,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;YAC3D,UAAU,CACR,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,QAAQ,IAAI,CAAC,EAAE,oBAAoB,OAAO,IAAI,CAAC,CAAC,EACvE,OAAO,CACR,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE,cAAc,CAAC,CAAC,CAAC;QAC1E,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC5E,MAAM,EAAE,WAAW,CAAC,YAAY,CAAC,CAAC;YAElC,OAAO;gBACL,MAAM,EAAE,IAAI,CAAC,EAAE;gBACf,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,MAAM,EAAE,EAAE;gBACV,UAAU,EAAE,OAAO;gBACnB,QAAQ,EAAE,CAAC;gBACX,OAAO,EAAE,CAAC;gBACV,UAAU,EAAE,EAAE;gBACd,SAAS,EAAE,EAAE;gBACb,OAAO,EAAE,IAAI;gBACb,YAAY;aACb,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CACV,UAA2B,EAC3B,YAAgD;QAEhD,IAAI,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;YAC1B,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CACtC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;gBAC5B,MAAM,MAAM,GAAG,YAAY,EAAE,CAAC,IAAI,CAAC,CAAC;gBACpC,OAAO,IAAI,CAAC,kBAAkB,CAAC,IAAI,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;YAC1D,CAAC,CAAC,CACH,CAAC;YAEF,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBAC/B,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;oBAClC,OAAO,MAAM,CAAC,KAAK,CAAC;gBACtB,CAAC;gBACD,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBACjC,OAAO;oBACL,MAAM,EAAE,IAAI,CAAC,EAAE;oBACf,MAAM,EAAE,IAAI,CAAC,MAAM;oBACnB,MAAM,EAAE,EAAE;oBACV,UAAU,EAAE,CAAC;oBACb,QAAQ,EAAE,CAAC;oBACX,OAAO,EAAE,CAAC;oBACV,UAAU,EAAE,EAAE;oBACd,SAAS,EAAE,EAAE;oBACb,OAAO,EAAE,IAAI;oBACb,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,IAAI,eAAe;iBACxD,CAAC;YACJ,CAAC,CAAC,CAAC;QACL,CAAC;QAED,MAAM,OAAO,GAAiB,EAAE,CAAC;QACjC,KAAK,MAAM,IAAI,IAAI,UAAU,CAAC,KAAK,EAAE,CAAC;YACpC,OAAO,CAAC,GAAG,CAAC,gBAAgB,IAAI,CAAC,EAAE,KAAK,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;YACvE,MAAM,MAAM,GAAG,YAAY,EAAE,CAAC,IAAI,CAAC,CAAC;YACpC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,IAAI,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;YACtE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAErB,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBACnB,OAAO,CAAC,KAAK,CAAC,YAAY,MAAM,CAAC,YAAY,EAAE,CAAC,CAAC;YACnD,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,GAAG,CAAC,oBAAoB,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,MAAM,EAAE,CAAC,CAAC;gBAC1E,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,cAAc,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC7G,CAAC;QACH,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;CACF"}

package/dist/src/runner/security.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Security policies for the evaluation runner.
+ *
+ * Restricts file writes to allowed directories via the Agent SDK's
+ * canUseTool callback.
+ */
+/**
+ * Create a canUseTool callback that restricts Write/Edit to allowed directories.
+ *
+ * Matches the Agent SDK's CanUseTool signature:
+ *   (toolName, input, options) => Promise<PermissionResult>
+ */
+export declare function createToolPolicy(allowedWriteDirs: string[], cwd: string): (toolName: string, input: Record<string, unknown>, _options: {
+    signal: AbortSignal;
+    toolUseID: string;
+    [key: string]: unknown;
+}) => Promise<{
+    behavior: "allow";
+    updatedInput: Record<string, unknown>;
+    message?: undefined;
+} | {
+    behavior: "deny";
+    message: string;
+    updatedInput?: undefined;
+}>;
+//# sourceMappingURL=security.d.ts.map

package/dist/src/runner/security.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"security.d.ts","sourceRoot":"","sources":["../../../src/runner/security.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAC9B,gBAAgB,EAAE,MAAM,EAAE,EAC1B,GAAG,EAAE,MAAM,IAOT,UAAU,MAAM,EAChB,OAAO,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC9B,UAAU;IAAE,MAAM,EAAE,WAAW,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE;;;;;;;;GAwB/E"}

package/dist/src/runner/security.js

@@ -0,0 +1,34 @@
+/**
+ * Security policies for the evaluation runner.
+ *
+ * Restricts file writes to allowed directories via the Agent SDK's
+ * canUseTool callback.
+ */
+import * as path from 'path';
+/**
+ * Create a canUseTool callback that restricts Write/Edit to allowed directories.
+ *
+ * Matches the Agent SDK's CanUseTool signature:
+ *   (toolName, input, options) => Promise<PermissionResult>
+ */
+export function createToolPolicy(allowedWriteDirs, cwd) {
+    const resolvedDirs = allowedWriteDirs.map((dir) => path.resolve(cwd, dir));
+    return async (toolName, input, _options) => {
+        // Allow all non-write tools
+        if (!['Write', 'Edit'].includes(toolName)) {
+            return { behavior: 'allow', updatedInput: input };
+        }
+        // Check if file path is in allowed directories
+        const filePath = input.file_path || '';
+        const resolvedPath = path.resolve(cwd, filePath);
+        const isAllowed = resolvedDirs.some((dir) => resolvedPath.startsWith(dir));
+        if (isAllowed) {
+            return { behavior: 'allow', updatedInput: input };
+        }
+        return {
+            behavior: 'deny',
+            message: `Write denied: ${filePath} is outside allowed directories: ${allowedWriteDirs.join(', ')}`,
+        };
+    };
+}
+//# sourceMappingURL=security.js.map

package/dist/src/runner/security.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"security.js","sourceRoot":"","sources":["../../../src/runner/security.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB,CAC9B,gBAA0B,EAC1B,GAAW;IAEX,MAAM,YAAY,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAChD,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CACvB,CAAC;IAEF,OAAO,KAAK,EACV,QAAgB,EAChB,KAA8B,EAC9B,QAA4E,EAC5E,EAAE;QACF,4BAA4B;QAC5B,IAAI,CAAC,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC1C,OAAO,EAAE,QAAQ,EAAE,OAAgB,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC;QAC7D,CAAC;QAED,+CAA+C;QAC/C,MAAM,QAAQ,GAAI,KAAK,CAAC,SAAoB,IAAI,EAAE,CAAC;QACnD,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;QAEjD,MAAM,SAAS,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAC1C,YAAY,CAAC,UAAU,CAAC,GAAG,CAAC,CAC7B,CAAC;QAEF,IAAI,SAAS,EAAE,CAAC;YACd,OAAO,EAAE,QAAQ,EAAE,OAAgB,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC;QAC7D,CAAC;QAED,OAAO;YACL,QAAQ,EAAE,MAAe;YACzB,OAAO,EAAE,iBAAiB,QAAQ,oCAAoC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;SACpG,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC"}

package/dist/src/runner/skill-setup.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Local skill setup and cleanup for evaluation runs.
+ *
+ * Copies skill directories into .claude/skills/ within the working directory
+ * so the Agent SDK can discover them via settingSources: ['project'].
+ */
+/**
+ * Copy skills from a source directory to .claude/skills/ in the working directory.
+ *
+ * @param skillsSourceDir - Directory containing skill folders (each with SKILL.md)
+ * @param cwd - Working directory where .claude/skills/ should be created
+ * @returns List of skill names that were set up
+ */
+export declare function setupLocalSkills(skillsSourceDir: string, cwd: string): Promise<string[]>;
+/**
+ * Remove .claude/skills/ from the working directory.
+ */
+export declare function cleanupLocalSkills(cwd: string): Promise<void>;
+//# sourceMappingURL=skill-setup.d.ts.map

package/dist/src/runner/skill-setup.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"skill-setup.d.ts","sourceRoot":"","sources":["../../../src/runner/skill-setup.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH;;;;;;GAMG;AACH,wBAAsB,gBAAgB,CACpC,eAAe,EAAE,MAAM,EACvB,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,MAAM,EAAE,CAAC,CAkCnB;AAED;;GAEG;AACH,wBAAsB,kBAAkB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAOnE"}

package/dist/src/runner/skill-setup.js

@@ -0,0 +1,72 @@
+/**
+ * Local skill setup and cleanup for evaluation runs.
+ *
+ * Copies skill directories into .claude/skills/ within the working directory
+ * so the Agent SDK can discover them via settingSources: ['project'].
+ */
+import * as fs from 'fs/promises';
+import * as path from 'path';
+/**
+ * Copy skills from a source directory to .claude/skills/ in the working directory.
+ *
+ * @param skillsSourceDir - Directory containing skill folders (each with SKILL.md)
+ * @param cwd - Working directory where .claude/skills/ should be created
+ * @returns List of skill names that were set up
+ */
+export async function setupLocalSkills(skillsSourceDir, cwd) {
+    const targetDir = path.join(cwd, '.claude', 'skills');
+    await fs.mkdir(targetDir, { recursive: true });
+    const skillNames = [];
+    try {
+        const entries = await fs.readdir(skillsSourceDir, { withFileTypes: true });
+        for (const entry of entries) {
+            if (entry.isDirectory()) {
+                const srcSkillDir = path.join(skillsSourceDir, entry.name);
+                const destSkillDir = path.join(targetDir, entry.name);
+                await copyDir(srcSkillDir, destSkillDir);
+                skillNames.push(entry.name);
+            }
+            else if (entry.name === 'SKILL.md') {
+                // Single skill file at root level
+                const skillName = path.basename(skillsSourceDir);
+                await fs.mkdir(path.join(targetDir, skillName), { recursive: true });
+                await fs.copyFile(path.join(skillsSourceDir, entry.name), path.join(targetDir, skillName, 'SKILL.md'));
+                skillNames.push(skillName);
+            }
+        }
+    }
+    catch (err) {
+        throw new Error(`Failed to setup local skills from ${skillsSourceDir}: ${err instanceof Error ? err.message : String(err)}`);
+    }
+    return skillNames;
+}
+/**
+ * Remove .claude/skills/ from the working directory.
+ */
+export async function cleanupLocalSkills(cwd) {
+    const skillsDir = path.join(cwd, '.claude', 'skills');
+    try {
+        await fs.rm(skillsDir, { recursive: true, force: true });
+    }
+    catch {
+        // Ignore cleanup errors
+    }
+}
+/**
+ * Recursively copy a directory.
+ */
+async function copyDir(src, dest) {
+    await fs.mkdir(dest, { recursive: true });
+    const entries = await fs.readdir(src, { withFileTypes: true });
+    for (const entry of entries) {
+        const srcPath = path.join(src, entry.name);
+        const destPath = path.join(dest, entry.name);
+        if (entry.isDirectory()) {
+            await copyDir(srcPath, destPath);
+        }
+        else {
+            await fs.copyFile(srcPath, destPath);
+        }
+    }
+}
+//# sourceMappingURL=skill-setup.js.map