@skilljack/evals 1.0.0

package/dist/src/session/session-logger.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Session logger for capturing evaluation run events.
+ *
+ * Captures tool calls, text output, metrics, and eval results
+ * for debugging and reporting purposes.
+ */
+import type { SessionLogEntry, MetricsData } from '../types.js';
+export declare class SessionLogger {
+    private log;
+    private logDir;
+    constructor(task: string, logDir?: string);
+    addEntry(type: SessionLogEntry['type'], data: unknown): void;
+    addTextMessage(text: string): void;
+    addToolUse(name: string, input: unknown): void;
+    addToolResult(name: string, success: boolean): void;
+    addAssistantMessage(content: unknown[]): void;
+    setMetrics(metrics: MetricsData): void;
+    markAsError(errorMessage: string): void;
+    getEntries(): SessionLogEntry[];
+    getSessionId(): string;
+    /**
+     * Save session log to disk as both JSON and human-readable markdown.
+     */
+    save(): Promise<{
+        jsonPath: string;
+        mdPath: string;
+    }>;
+    private generateReadableLog;
+}
+//# sourceMappingURL=session-logger.d.ts.map

package/dist/src/session/session-logger.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"session-logger.d.ts","sourceRoot":"","sources":["../../../src/session/session-logger.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,OAAO,KAAK,EAAE,eAAe,EAAc,WAAW,EAAE,MAAM,aAAa,CAAC;AAE5E,qBAAa,aAAa;IACxB,OAAO,CAAC,GAAG,CAAa;IACxB,OAAO,CAAC,MAAM,CAAS;gBAEX,IAAI,EAAE,MAAM,EAAE,MAAM,GAAE,MAAyB;IAY3D,QAAQ,CAAC,IAAI,EAAE,eAAe,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,OAAO,GAAG,IAAI;IAQ5D,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAIlC,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,IAAI;IAI9C,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,GAAG,IAAI;IAInD,mBAAmB,CAAC,OAAO,EAAE,OAAO,EAAE,GAAG,IAAI;IAI7C,UAAU,CAAC,OAAO,EAAE,WAAW,GAAG,IAAI;IAItC,WAAW,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI;IAKvC,UAAU,IAAI,eAAe,EAAE;IAI/B,YAAY,IAAI,MAAM;IAItB;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAiB3D,OAAO,CAAC,mBAAmB;CAoF5B"}

package/dist/src/session/session-logger.js

@@ -0,0 +1,157 @@
+/**
+ * Session logger for capturing evaluation run events.
+ *
+ * Captures tool calls, text output, metrics, and eval results
+ * for debugging and reporting purposes.
+ */
+import * as fs from 'fs/promises';
+import * as path from 'path';
+export class SessionLogger {
+    log;
+    logDir;
+    constructor(task, logDir = './results/logs') {
+        const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+        this.logDir = logDir;
+        this.log = {
+            sessionId: `eval-${task}-${timestamp}`,
+            task,
+            startTime: new Date().toISOString(),
+            status: 'success',
+            entries: [],
+        };
+    }
+    addEntry(type, data) {
+        this.log.entries.push({
+            timestamp: new Date().toISOString(),
+            type,
+            data,
+        });
+    }
+    addTextMessage(text) {
+        this.addEntry('text', { text });
+    }
+    addToolUse(name, input) {
+        this.addEntry('tool_use', { name, input });
+    }
+    addToolResult(name, success) {
+        this.addEntry('tool_result', { name, success });
+    }
+    addAssistantMessage(content) {
+        this.addEntry('assistant', { content });
+    }
+    setMetrics(metrics) {
+        this.log.metrics = metrics;
+    }
+    markAsError(errorMessage) {
+        this.log.status = 'error';
+        this.log.errorMessage = errorMessage;
+    }
+    getEntries() {
+        return this.log.entries;
+    }
+    getSessionId() {
+        return this.log.sessionId;
+    }
+    /**
+     * Save session log to disk as both JSON and human-readable markdown.
+     */
+    async save() {
+        this.log.endTime = new Date().toISOString();
+        await fs.mkdir(this.logDir, { recursive: true });
+        const prefix = this.log.status === 'error' ? 'FAILED__' : '';
+        const baseName = `${prefix}${this.log.sessionId}`;
+        const jsonPath = path.join(this.logDir, `${baseName}.json`);
+        const mdPath = path.join(this.logDir, `${baseName}.md`);
+        await fs.writeFile(jsonPath, JSON.stringify(this.log, null, 2));
+        await fs.writeFile(mdPath, this.generateReadableLog());
+        return { jsonPath, mdPath };
+    }
+    generateReadableLog() {
+        const lines = [];
+        lines.push(`# Eval Session: ${this.log.sessionId}`);
+        lines.push(`**Task:** ${this.log.task}`);
+        lines.push(`**Start:** ${this.log.startTime}`);
+        lines.push(`**End:** ${this.log.endTime || 'In progress'}`);
+        lines.push(`**Status:** ${this.log.status === 'success' ? 'PASS' : 'FAIL'} (${this.log.status})`);
+        if (this.log.errorMessage) {
+            lines.push(`**Error:** ${this.log.errorMessage}`);
+        }
+        lines.push('');
+        if (this.log.metrics) {
+            lines.push('## Metrics');
+            lines.push(`- **Duration:** ${formatDuration(this.log.metrics.timing.totalElapsedMs)}`);
+            lines.push(`- **Cost:** $${this.log.metrics.cost.toFixed(6)}`);
+            lines.push(`- **Turns:** ${this.log.metrics.turns}`);
+            lines.push(`- **Tokens:** ${this.log.metrics.tokens.total.toLocaleString()}`);
+            lines.push('');
+        }
+        lines.push('## Events');
+        lines.push('');
+        let toolCount = 0;
+        for (const entry of this.log.entries) {
+            const time = entry.timestamp.split('T')[1]?.split('.')[0] || '';
+            switch (entry.type) {
+                case 'text': {
+                    const data = entry.data;
+                    const preview = data.text.length > 500 ? data.text.substring(0, 500) + '...' : data.text;
+                    lines.push(`### [${time}] Text`);
+                    lines.push('```');
+                    lines.push(preview);
+                    lines.push('```');
+                    lines.push('');
+                    break;
+                }
+                case 'tool_use': {
+                    toolCount++;
+                    const data = entry.data;
+                    lines.push(`### [${time}] Tool #${toolCount}: ${data.name}`);
+                    const inputStr = JSON.stringify(data.input, null, 2);
+                    lines.push('```json');
+                    lines.push(inputStr.length > 1000 ? inputStr.substring(0, 1000) + '\n...' : inputStr);
+                    lines.push('```');
+                    lines.push('');
+                    break;
+                }
+                case 'tool_result': {
+                    const data = entry.data;
+                    lines.push(`- Tool result: ${data.name} - ${data.success ? 'Success' : 'Failed'}`);
+                    lines.push('');
+                    break;
+                }
+                case 'assistant': {
+                    const data = entry.data;
+                    lines.push(`### [${time}] Assistant`);
+                    for (const chunk of data.content) {
+                        if (typeof chunk === 'object' && chunk !== null) {
+                            const c = chunk;
+                            if (c.type === 'text' && c.text) {
+                                const preview = c.text.length > 300 ? c.text.substring(0, 300) + '...' : c.text;
+                                lines.push('```');
+                                lines.push(preview);
+                                lines.push('```');
+                            }
+                            else if (c.type === 'tool_use' && c.name) {
+                                lines.push(`Tool call: ${c.name}`);
+                            }
+                        }
+                    }
+                    lines.push('');
+                    break;
+                }
+            }
+        }
+        lines.push('---');
+        lines.push(`Total tool calls: ${toolCount}`);
+        return lines.join('\n');
+    }
+}
+function formatDuration(ms) {
+    if (ms < 1000)
+        return `${ms}ms`;
+    if (ms < 60000)
+        return `${(ms / 1000).toFixed(2)}s`;
+    const min = Math.floor(ms / 60000);
+    const sec = ((ms % 60000) / 1000).toFixed(1);
+    return `${min}m ${sec}s`;
+}
+//# sourceMappingURL=session-logger.js.map

package/dist/src/session/session-logger.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"session-logger.js","sourceRoot":"","sources":["../../../src/session/session-logger.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,MAAM,aAAa,CAAC;AAClC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAG7B,MAAM,OAAO,aAAa;IAChB,GAAG,CAAa;IAChB,MAAM,CAAS;IAEvB,YAAY,IAAY,EAAE,SAAiB,gBAAgB;QACzD,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QACjE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,GAAG,GAAG;YACT,SAAS,EAAE,QAAQ,IAAI,IAAI,SAAS,EAAE;YACtC,IAAI;YACJ,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,MAAM,EAAE,SAAS;YACjB,OAAO,EAAE,EAAE;SACZ,CAAC;IACJ,CAAC;IAED,QAAQ,CAAC,IAA6B,EAAE,IAAa;QACnD,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC;YACpB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,IAAI;YACJ,IAAI;SACL,CAAC,CAAC;IACL,CAAC;IAED,cAAc,CAAC,IAAY;QACzB,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;IAClC,CAAC;IAED,UAAU,CAAC,IAAY,EAAE,KAAc;QACrC,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAC7C,CAAC;IAED,aAAa,CAAC,IAAY,EAAE,OAAgB;QAC1C,IAAI,CAAC,QAAQ,CAAC,aAAa,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;IAClD,CAAC;IAED,mBAAmB,CAAC,OAAkB;QACpC,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;IAC1C,CAAC;IAED,UAAU,CAAC,OAAoB;QAC7B,IAAI,CAAC,GAAG,CAAC,OAAO,GAAG,OAAO,CAAC;IAC7B,CAAC;IAED,WAAW,CAAC,YAAoB;QAC9B,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,OAAO,CAAC;QAC1B,IAAI,CAAC,GAAG,CAAC,YAAY,GAAG,YAAY,CAAC;IACvC,CAAC;IAED,UAAU;QACR,OAAO,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC;IAC1B,CAAC;IAED,YAAY;QACV,OAAO,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC;IAC5B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI;QACR,IAAI,CAAC,GAAG,CAAC,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAE5C,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAEjD,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7D,MAAM,QAAQ,GAAG,GAAG,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC;QAElD,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,QAAQ,OAAO,CAAC,CAAC;QAC5D,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,QAAQ,KAAK,CAAC,CAAC;QAExD,MAAM,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAChE,MAAM,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,mBAAmB,EAAE,CAAC,CAAC;QAEvD,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;IAC9B,CAAC;IAEO,mBAAmB;QACzB,MAAM,KAAK,GAAa,EAAE,CAAC;QAE3B,KAAK,CAAC,IAAI,CAAC,mBAAmB,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC;QACpD,KAAK,CAAC,IAAI,CAAC,aAAa,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;QACzC,KAAK,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC;QAC/C,KAAK,CAAC,IAAI,CAAC,YAAY,IAAI,CAAC,GAAG,CAAC,OAAO,IAAI,aAAa,EAAE,CAAC,CAAC;QAC5D,KAAK,CAAC,IAAI,CAAC,eAAe,IAAI,CAAC,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC;QAClG,IAAI,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,CAAC;YAC1B,KAAK,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,CAAC,CAAC;QACpD,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAEf,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;YACrB,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YACzB,KAAK,CAAC,IAAI,CAAC,mBAAmB,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;YACxF,KAAK,CAAC,IAAI,CAAC,gBAAgB,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC/D,KAAK,CAAC,IAAI,CAAC,gBAAgB,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;YACrD,KAAK,CAAC,IAAI,CAAC,iBAAiB,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;YAC9E,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACjB,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACxB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAEf,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;YACrC,MAAM,IAAI,GAAG,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAEhE,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;gBACnB,KAAK,MAAM,CAAC,CAAC,CAAC;oBACZ,MAAM,IAAI,GAAG,KAAK,CAAC,IAAwB,CAAC;oBAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;oBACzF,KAAK,CAAC,IAAI,CAAC,QAAQ,IAAI,QAAQ,CAAC,CAAC;oBACjC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBAClB,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;oBACpB,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBAClB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;oBACf,MAAM;gBACR,CAAC;gBACD,KAAK,UAAU,CAAC,CAAC,CAAC;oBAChB,SAAS,EAAE,CAAC;oBACZ,MAAM,IAAI,GAAG,KAAK,CAAC,IAAwC,CAAC;oBAC5D,KAAK,CAAC,IAAI,CAAC,QAAQ,IAAI,WAAW,SAAS,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;oBAC7D,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;oBACrD,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;oBACtB,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;oBACtF,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBAClB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;oBACf,MAAM;gBACR,CAAC;gBACD,KAAK,aAAa,CAAC,CAAC,CAAC;oBACnB,MAAM,IAAI,GAAG,KAAK,CAAC,IAA0C,CAAC;oBAC9D,KAAK,CAAC,IAAI,CAAC,kBAAkB,IAAI,CAAC,IAAI,MAAM,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;oBACnF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;oBACf,MAAM;gBACR,CAAC;gBACD,KAAK,WAAW,CAAC,CAAC,CAAC;oBACjB,MAAM,IAAI,GAAG,KAAK,CAAC,IAA8B,CAAC;oBAClD,KAAK,CAAC,IAAI,CAAC,QAAQ,IAAI,aAAa,CAAC,CAAC;oBACtC,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;wBACjC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;4BAChD,MAAM,CAAC,GAAG,KAAuD,CAAC;4BAClE,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;gCAChC,MAAM,OAAO,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;gCAChF,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gCAClB,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gCACpB,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;4BACpB,CAAC;iCAAM,IAAI,CAAC,CAAC,IAAI,KAAK,UAAU,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;gCAC3C,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;4BACrC,CAAC;wBACH,CAAC;oBACH,CAAC;oBACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;oBACf,MAAM;gBACR,CAAC;YACH,CAAC;QACH,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClB,KAAK,CAAC,IAAI,CAAC,qBAAqB,SAAS,EAAE,CAAC,CAAC;QAE7C,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;CACF;AAED,SAAS,cAAc,CAAC,EAAU;IAChC,IAAI,EAAE,GAAG,IAAI;QAAE,OAAO,GAAG,EAAE,IAAI,CAAC;IAChC,IAAI,EAAE,GAAG,KAAK;QAAE,OAAO,GAAG,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;IACpD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,KAAK,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAC7C,OAAO,GAAG,GAAG,KAAK,GAAG,GAAG,CAAC;AAC3B,CAAC"}

package/dist/src/types.d.ts

@@ -0,0 +1,227 @@
+/**
+ * Type definitions for the skill evaluation framework.
+ */
+export interface EvalCriteria {
+    dimension: 'discovery' | 'adherence' | 'output';
+    weight: number;
+    description: string;
+}
+export interface DeterministicCheck {
+    expectSkillActivation: boolean;
+    expectMarker?: string;
+    expectToolCalls?: string[];
+    expectNoToolCalls?: string[];
+}
+export interface FixtureConfig {
+    state: string;
+    setup?: string;
+    teardown?: string;
+}
+export interface EvalTask {
+    id: string;
+    prompt: string;
+    expectedSkillLoad: string;
+    criteria: EvalCriteria[];
+    goldenChecklist: string[];
+    deterministic?: DeterministicCheck;
+    fixture?: FixtureConfig;
+}
+export interface EvalDefaults {
+    expectedSkillLoad?: string;
+    criteria?: Partial<Record<'discovery' | 'adherence' | 'output', {
+        weight?: number;
+        description?: string;
+    }>>;
+}
+export interface SkillEvaluation {
+    skillName: string;
+    version?: string;
+    defaults?: EvalDefaults;
+    tasks: EvalTask[];
+}
+export interface ToolCallRecord {
+    tool: string;
+    toolUseId: string;
+    timestamp: number;
+    input?: unknown;
+}
+export interface TaskResult {
+    taskId: string;
+    prompt: string;
+    output: string;
+    durationMs: number;
+    numTurns: number;
+    costUsd: number;
+    skillLoads: string[];
+    toolCalls: ToolCallRecord[];
+    isError: boolean;
+    errorMessage: string;
+}
+export interface RunnerOptions {
+    cwd?: string;
+    parallel?: boolean;
+    model?: string;
+    settingSources?: Array<'user' | 'project' | 'local'>;
+    /** Count Read calls to SKILL.md as skill discovery (default: false) */
+    countReadAsFallback?: boolean;
+    /** Directories the agent is allowed to write to */
+    allowedWriteDirs?: string[];
+}
+export interface DeterministicResult {
+    skillActivated: boolean;
+    skillName?: string;
+    markerFound: boolean | null;
+    expectedToolsCalled: boolean | null;
+    unexpectedToolsCalled: boolean | null;
+    passed: boolean;
+    details: string[];
+}
+export type FailureCategory = 'discovery_failure' | 'false_positive' | 'instruction_ambiguity' | 'missing_guidance' | 'agent_error' | 'none';
+export interface JudgeScore {
+    taskId: string;
+    discovery: number;
+    adherence: number;
+    outputQuality: number;
+    weightedScore: number;
+    failureCategory: FailureCategory;
+    reasoning: string;
+}
+export interface JudgeOptions {
+    model?: string;
+    outputTruncation?: number;
+}
+export interface CombinedScore {
+    taskId: string;
+    deterministic: DeterministicResult | null;
+    judge: JudgeScore | null;
+    discovery: number;
+    adherence: number;
+    outputQuality: number;
+    weightedScore: number;
+    failureCategory: FailureCategory;
+    reasoning: string;
+}
+export interface SessionLogEntry {
+    timestamp: string;
+    type: 'text' | 'tool_use' | 'tool_result' | 'assistant';
+    data: unknown;
+}
+export interface MetricsData {
+    timestamp: string;
+    task: string;
+    timing: {
+        totalElapsedMs: number;
+        sdkDurationMs: number;
+        apiDurationMs: number;
+        overheadMs: number;
+    };
+    tokens: {
+        input: number;
+        output: number;
+        cacheRead: number;
+        cacheCreation: number;
+        total: number;
+    };
+    cost: number;
+    turns: number;
+    modelUsage: Record<string, {
+        inputTokens: number;
+        outputTokens: number;
+        costUSD: number;
+    }>;
+}
+export interface SessionLog {
+    sessionId: string;
+    task: string;
+    startTime: string;
+    endTime?: string;
+    status: 'success' | 'error';
+    errorMessage?: string;
+    entries: SessionLogEntry[];
+    metrics?: MetricsData;
+}
+export interface EvaluationSummary {
+    totalTasks: number;
+    discoveryAccuracy: number;
+    avgAdherence: number;
+    avgOutputQuality: number;
+    avgWeightedScore: number;
+    totalDurationMs: number;
+    totalCostUsd: number;
+}
+export interface FailureBreakdown {
+    category: FailureCategory;
+    count: number;
+    percentage: number;
+}
+export interface ReportMetadata {
+    skillPath: string;
+    gitCommit?: string;
+    gitBranch?: string;
+    version?: string;
+    agentModel: string;
+    judgeModel: string;
+}
+export interface EvaluationReport {
+    skillName: string;
+    timestamp: string;
+    passed: boolean;
+    failureReasons: string[];
+    metadata?: ReportMetadata;
+    summary: EvaluationSummary;
+    failureBreakdown: FailureBreakdown[];
+    tasks: Array<{
+        task: EvalTask;
+        result: TaskResult;
+        score: CombinedScore;
+        sessionLogPath?: string;
+    }>;
+}
+export interface SdkTextBlock {
+    type: 'text';
+    text: string;
+}
+export interface SdkToolUseBlock {
+    type: 'tool_use';
+    id: string;
+    name: string;
+    input: Record<string, unknown>;
+}
+export type SdkContentBlock = SdkTextBlock | SdkToolUseBlock | {
+    type: string;
+    [key: string]: unknown;
+};
+export interface SdkAssistantMessage {
+    type: 'assistant';
+    message: {
+        content: SdkContentBlock[];
+    };
+}
+export interface SdkResultMessage {
+    type: 'result';
+    result?: string;
+    duration_ms: number;
+    duration_api_ms: number;
+    num_turns: number;
+    total_cost_usd: number;
+    usage: {
+        input_tokens: number;
+        output_tokens: number;
+        cache_read_input_tokens: number;
+        cache_creation_input_tokens: number;
+    };
+    modelUsage: Record<string, {
+        inputTokens: number;
+        outputTokens: number;
+        costUSD: number;
+    }>;
+}
+export type SdkStreamMessage = SdkAssistantMessage | SdkResultMessage | {
+    type: string;
+    [key: string]: unknown;
+};
+export declare function isAssistantMessage(msg: unknown): msg is SdkAssistantMessage;
+export declare function isResultMessage(msg: unknown): msg is SdkResultMessage;
+export declare function isTextBlock(block: unknown): block is SdkTextBlock;
+export declare function isToolUseBlock(block: unknown): block is SdkToolUseBlock;
+//# sourceMappingURL=types.d.ts.map

package/dist/src/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,WAAW,GAAG,WAAW,GAAG,QAAQ,CAAC;IAChD,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,kBAAkB;IACjC,qBAAqB,EAAE,OAAO,CAAC;IAC/B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;CAC9B;AAED,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,iBAAiB,EAAE,MAAM,CAAC;IAC1B,QAAQ,EAAE,YAAY,EAAE,CAAC;IACzB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,aAAa,CAAC,EAAE,kBAAkB,CAAC;IACnC,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AAED,MAAM,WAAW,YAAY;IAC3B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,WAAW,GAAG,WAAW,GAAG,QAAQ,EAAE;QAAE,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC,CAAC;CAC7G;AAED,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB,KAAK,EAAE,QAAQ,EAAE,CAAC;CACnB;AAMD,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,SAAS,EAAE,cAAc,EAAE,CAAC;IAC5B,OAAO,EAAE,OAAO,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,aAAa;IAC5B,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,CAAC;IACrD,uEAAuE;IACvE,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,mDAAmD;IACnD,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;CAC7B;AAMD,MAAM,WAAW,mBAAmB;IAClC,cAAc,EAAE,OAAO,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,OAAO,GAAG,IAAI,CAAC;IAC5B,mBAAmB,EAAE,OAAO,GAAG,IAAI,CAAC;IACpC,qBAAqB,EAAE,OAAO,GAAG,IAAI,CAAC;IACtC,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB;AAMD,MAAM,MAAM,eAAe,GACvB,mBAAmB,GACnB,gBAAgB,GAChB,uBAAuB,GACvB,kBAAkB,GAClB,aAAa,GACb,MAAM,CAAC;AAEX,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,eAAe,CAAC;IACjC,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,YAAY;IAC3B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAMD,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,mBAAmB,GAAG,IAAI,CAAC;IAC1C,KAAK,EAAE,UAAU,GAAG,IAAI,CAAC;IAGzB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,eAAe,CAAC;IACjC,SAAS,EAAE,MAAM,CAAC;CACnB;AAMD,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,GAAG,UAAU,GAAG,aAAa,GAAG,WAAW,CAAC;IACxD,IAAI,EAAE,OAAO,CAAC;CACf;AAED,MAAM,WAAW,WAAW;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE;QACN,cAAc,EAAE,MAAM,CAAC;QACvB,aAAa,EAAE,MAAM,CAAC;QACtB,aAAa,EAAE,MAAM,CAAC;QACtB,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,MAAM,EAAE;QACN,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,EAAE,MAAM,CAAC;QAClB,aAAa,EAAE,MAAM,CAAC;QACtB,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;IACF,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE;QACzB,WAAW,EAAE,MAAM,CAAC;QACpB,YAAY,EAAE,MAAM,CAAC;QACrB,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC,CAAC;CACJ;AAED,MAAM,WAAW,UAAU;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,SAAS,GAAG,OAAO,CAAC;IAC5B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,OAAO,EAAE,eAAe,EAAE,CAAC;IAC3B,OAAO,CAAC,EAAE,WAAW,CAAC;CACvB;AAMD,MAAM,WAAW,iBAAiB;IAChC,UAAU,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;IACzB,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,eAAe,CAAC;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,cAAc;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,OAAO,CAAC;IAChB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,QAAQ,CAAC,EAAE,cAAc,CAAC;IAC1B,OAAO,EAAE,iBAAiB,CAAC;IAC3B,gBAAgB,EAAE,gBAAgB,EAAE,CAAC;IACrC,KAAK,EAAE,KAAK,CAAC;QACX,IAAI,EAAE,QAAQ,CAAC;QACf,MAAM,EAAE,UAAU,CAAC;QACnB,KAAK,EAAE,aAAa,CAAC;QACrB,cAAc,CAAC,EAAE,MAAM,CAAC;KACzB,CAAC,CAAC;CACJ;AAMD,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,UAAU,CAAC;IACjB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAChC;AAED,MAAM,MAAM,eAAe,GAAG,YAAY,GAAG,eAAe,GAAG;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,CAAC;AAExG,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,WAAW,CAAC;IAClB,OAAO,EAAE;QACP,OAAO,EAAE,eAAe,EAAE,CAAC;KAC5B,CAAC;CACH;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;IACvB,KAAK,EAAE;QACL,YAAY,EAAE,MAAM,CAAC;QACrB,aAAa,EAAE,MAAM,CAAC;QACtB,uBAAuB,EAAE,MAAM,CAAC;QAChC,2BAA2B,EAAE,MAAM,CAAC;KACrC,CAAC;IACF,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE;QACzB,WAAW,EAAE,MAAM,CAAC;QACpB,YAAY,EAAE,MAAM,CAAC;QACrB,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC,CAAC;CACJ;AAED,MAAM,MAAM,gBAAgB,GAAG,mBAAmB,GAAG,gBAAgB,GAAG;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,CAAC;AAEjH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,OAAO,GAAG,GAAG,IAAI,mBAAmB,CAE3E;AAED,wBAAgB,eAAe,CAAC,GAAG,EAAE,OAAO,GAAG,GAAG,IAAI,gBAAgB,CAErE;AAED,wBAAgB,WAAW,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,YAAY,CAEjE;AAED,wBAAgB,cAAc,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,eAAe,CAEvE"}

package/dist/src/types.js

@@ -0,0 +1,16 @@
+/**
+ * Type definitions for the skill evaluation framework.
+ */
+export function isAssistantMessage(msg) {
+    return typeof msg === 'object' && msg !== null && msg.type === 'assistant';
+}
+export function isResultMessage(msg) {
+    return typeof msg === 'object' && msg !== null && msg.type === 'result';
+}
+export function isTextBlock(block) {
+    return typeof block === 'object' && block !== null && block.type === 'text';
+}
+export function isToolUseBlock(block) {
+    return typeof block === 'object' && block !== null && block.type === 'tool_use';
+}
+//# sourceMappingURL=types.js.map

package/dist/src/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAuRH,MAAM,UAAU,kBAAkB,CAAC,GAAY;IAC7C,OAAO,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI,IAAK,GAAyB,CAAC,IAAI,KAAK,WAAW,CAAC;AACpG,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,GAAY;IAC1C,OAAO,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI,IAAK,GAAyB,CAAC,IAAI,KAAK,QAAQ,CAAC;AACjG,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,KAAc;IACxC,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAK,KAA2B,CAAC,IAAI,KAAK,MAAM,CAAC;AACrG,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,KAAc;IAC3C,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAK,KAA2B,CAAC,IAAI,KAAK,UAAU,CAAC;AACzG,CAAC"}

package/package.json

@@ -0,0 +1,44 @@
+{
+  "name": "@skilljack/evals",
+  "version": "1.0.0",
+  "type": "module",
+  "description": "CLI for evaluating AI agent skill discoverability, adherence, and output quality. Runs as standalone CLI or GitHub Action.",
+  "bin": {
+    "skilljack-evals": "./dist/src/cli.js"
+  },
+  "main": "./dist/src/index.js",
+  "types": "./dist/src/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/src/index.js",
+      "types": "./dist/src/index.d.ts"
+    }
+  },
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsx src/cli.ts",
+    "start": "node dist/src/cli.js",
+    "clean": "rm -rf dist",
+    "typecheck": "tsc --noEmit"
+  },
+  "dependencies": {
+    "@anthropic-ai/claude-agent-sdk": "^0.1.42",
+    "commander": "^12.0.0",
+    "dotenv": "^16.4.0",
+    "js-yaml": "^4.1.0"
+  },
+  "devDependencies": {
+    "@actions/core": "^1.10.0",
+    "@types/js-yaml": "^4.0.9",
+    "@types/node": "^20.0.0",
+    "tsx": "^4.0.0",
+    "typescript": "^5.0.0"
+  },
+  "engines": {
+    "node": ">=20.0.0"
+  },
+  "files": [
+    "dist",
+    "action"
+  ]
+}