@skilljack/evals 1.0.0

package/dist/src/parser.js

@@ -0,0 +1,205 @@
+/**
+ * YAML parser for skill evaluation task files.
+ *
+ * Supports enriched YAML schema with:
+ * - defaults block (shared criteria, expected_skill_load)
+ * - deterministic block (marker-based checks, tool call expectations)
+ * - fixture block (setup/teardown scripts per task)
+ */
+import yaml from 'js-yaml';
+import * as fs from 'fs/promises';
+// ============================================
+// Parser
+// ============================================
+/**
+ * Parse a YAML evaluation file into a SkillEvaluation object.
+ */
+export async function parseEvalFile(filePath) {
+    const content = await fs.readFile(filePath, 'utf-8');
+    const raw = yaml.load(content);
+    if (!raw || !raw.skill) {
+        throw new Error(`Invalid evaluation file: missing 'skill' field`);
+    }
+    const defaults = raw.defaults ? parseDefaults(raw.defaults) : undefined;
+    const tasks = (raw.tasks || []).map((t) => parseTask(t, defaults));
+    return {
+        skillName: raw.skill,
+        version: raw.version,
+        defaults,
+        tasks,
+    };
+}
+/**
+ * Parse the defaults block.
+ */
+function parseDefaults(raw) {
+    const defaults = {};
+    if (raw.expected_skill_load) {
+        defaults.expectedSkillLoad = raw.expected_skill_load;
+    }
+    if (raw.criteria) {
+        defaults.criteria = {};
+        for (const dim of ['discovery', 'adherence', 'output']) {
+            const rawCrit = raw.criteria[dim];
+            if (rawCrit) {
+                defaults.criteria[dim] = {
+                    weight: rawCrit.weight,
+                    description: rawCrit.description,
+                };
+            }
+        }
+    }
+    return defaults;
+}
+/**
+ * Parse a single task from raw YAML, merging with defaults.
+ */
+function parseTask(raw, defaults) {
+    // Merge expected skill load: task overrides defaults
+    const expectedSkillLoad = raw.expected_skill_load
+        ?? defaults?.expectedSkillLoad
+        ?? '';
+    // Merge criteria: task-level overrides default-level
+    const criteria = [];
+    const dimensions = ['discovery', 'adherence', 'output'];
+    for (const dim of dimensions) {
+        const taskCrit = raw.criteria?.[dim];
+        const defaultCrit = defaults?.criteria?.[dim];
+        if (taskCrit || defaultCrit) {
+            criteria.push({
+                dimension: dim,
+                weight: taskCrit?.weight ?? defaultCrit?.weight ?? 0.33,
+                description: taskCrit?.description ?? defaultCrit?.description ?? '',
+            });
+        }
+    }
+    // Parse deterministic block
+    let deterministic;
+    if (raw.deterministic) {
+        deterministic = {
+            expectSkillActivation: raw.deterministic.expect_skill_activation ?? true,
+            expectMarker: raw.deterministic.expect_marker,
+            expectToolCalls: raw.deterministic.expect_tool_calls,
+            expectNoToolCalls: raw.deterministic.expect_no_tool_calls,
+        };
+    }
+    // Parse fixture block
+    let fixture;
+    if (raw.fixture) {
+        fixture = {
+            state: raw.fixture.state ?? 'default',
+            setup: raw.fixture.setup,
+            teardown: raw.fixture.teardown,
+        };
+    }
+    return {
+        id: raw.id || '',
+        prompt: raw.prompt || '',
+        expectedSkillLoad,
+        criteria,
+        goldenChecklist: raw.golden_checklist || [],
+        deterministic,
+        fixture,
+    };
+}
+// ============================================
+// Validation
+// ============================================
+/**
+ * Validate a YAML evaluation file and return any errors.
+ */
+export async function validateEvalFile(filePath) {
+    const errors = [];
+    let content;
+    try {
+        content = await fs.readFile(filePath, 'utf-8');
+    }
+    catch {
+        return [`Cannot read file: ${filePath}`];
+    }
+    let raw;
+    try {
+        raw = yaml.load(content);
+    }
+    catch (e) {
+        return [`Invalid YAML: ${e instanceof Error ? e.message : String(e)}`];
+    }
+    if (!raw) {
+        return ['File is empty'];
+    }
+    if (!raw.skill) {
+        errors.push("Missing required field: 'skill'");
+    }
+    if (!raw.tasks || !Array.isArray(raw.tasks)) {
+        errors.push("Missing or invalid 'tasks' array");
+        return errors;
+    }
+    const taskIds = new Set();
+    for (let i = 0; i < raw.tasks.length; i++) {
+        const task = raw.tasks[i];
+        const prefix = `tasks[${i}]`;
+        if (!task.id) {
+            errors.push(`${prefix}: Missing 'id'`);
+        }
+        else if (taskIds.has(task.id)) {
+            errors.push(`${prefix}: Duplicate task id '${task.id}'`);
+        }
+        else {
+            taskIds.add(task.id);
+        }
+        if (!task.prompt) {
+            errors.push(`${prefix}: Missing 'prompt'`);
+        }
+        // Validate criteria weights sum roughly to 1
+        if (task.criteria) {
+            const weights = Object.values(task.criteria)
+                .filter((c) => c !== undefined)
+                .map((c) => c.weight ?? 0.33);
+            const sum = weights.reduce((a, b) => a + b, 0);
+            if (weights.length > 0 && Math.abs(sum - 1) > 0.1) {
+                errors.push(`${prefix}: Criteria weights sum to ${sum.toFixed(2)}, expected ~1.0`);
+            }
+        }
+    }
+    return errors;
+}
+// ============================================
+// Template Generation
+// ============================================
+/**
+ * Generate a YAML template for a new skill evaluation.
+ */
+export function createEvalTemplate(skillName, numTasks = 5) {
+    const prefix = skillName.slice(0, 2).toLowerCase();
+    const tasks = Array.from({ length: numTasks }, (_, i) => {
+        const taskId = `${prefix}-${String(i + 1).padStart(3, '0')}`;
+        return `  - id: ${taskId}
+    prompt: "TODO: Write a realistic prompt that should trigger ${skillName}"
+    expected_skill_load: ${skillName}
+    deterministic:
+      expect_skill_activation: true
+      # expect_marker: "OPTIONAL_MARKER_TEXT"
+    criteria:
+      discovery: { weight: 0.3, description: "Should load ${skillName} based on task context" }
+      adherence: { weight: 0.4, description: "Should follow ${skillName} instructions" }
+      output: { weight: 0.3, description: "Should produce quality output meeting requirements" }
+    golden_checklist:
+      - "TODO: Add expected behavior 1"
+      - "TODO: Add expected behavior 2"
+      - "TODO: Add expected behavior 3"`;
+    });
+    return `skill: ${skillName}
+version: "1.0"
+
+defaults:
+  expected_skill_load: ${skillName}
+  criteria:
+    discovery: { weight: 0.3 }
+    adherence: { weight: 0.4 }
+    output: { weight: 0.3 }
+
+tasks:
+${tasks.join('\n\n')}
+`;
+}
+//# sourceMappingURL=parser.js.map

package/dist/src/parser.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parser.js","sourceRoot":"","sources":["../../src/parser.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,IAAI,MAAM,SAAS,CAAC;AAC3B,OAAO,KAAK,EAAE,MAAM,aAAa,CAAC;AA8DlC,+CAA+C;AAC/C,SAAS;AACT,+CAA+C;AAE/C;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,QAAgB;IAClD,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACrD,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAgB,CAAC;IAE9C,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;IACpE,CAAC;IAED,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACxE,MAAM,KAAK,GAAG,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC;IAEnE,OAAO;QACL,SAAS,EAAE,GAAG,CAAC,KAAK;QACpB,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,QAAQ;QACR,KAAK;KACN,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAS,aAAa,CAAC,GAAgB;IACrC,MAAM,QAAQ,GAAiB,EAAE,CAAC;IAElC,IAAI,GAAG,CAAC,mBAAmB,EAAE,CAAC;QAC5B,QAAQ,CAAC,iBAAiB,GAAG,GAAG,CAAC,mBAAmB,CAAC;IACvD,CAAC;IAED,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;QACjB,QAAQ,CAAC,QAAQ,GAAG,EAAE,CAAC;QACvB,KAAK,MAAM,GAAG,IAAI,CAAC,WAAW,EAAE,WAAW,EAAE,QAAQ,CAAU,EAAE,CAAC;YAChE,MAAM,OAAO,GAAG,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;YAClC,IAAI,OAAO,EAAE,CAAC;gBACZ,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG;oBACvB,MAAM,EAAE,OAAO,CAAC,MAAM;oBACtB,WAAW,EAAE,OAAO,CAAC,WAAW;iBACjC,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,SAAS,SAAS,CAAC,GAAY,EAAE,QAAuB;IACtD,qDAAqD;IACrD,MAAM,iBAAiB,GAAG,GAAG,CAAC,mBAAmB;WAC5C,QAAQ,EAAE,iBAAiB;WAC3B,EAAE,CAAC;IAER,qDAAqD;IACrD,MAAM,QAAQ,GAAmB,EAAE,CAAC;IACpC,MAAM,UAAU,GAAG,CAAC,WAAW,EAAE,WAAW,EAAE,QAAQ,CAAU,CAAC;IAEjE,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC7B,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,EAAE,CAAC,GAAG,CAAC,CAAC;QACrC,MAAM,WAAW,GAAG,QAAQ,EAAE,QAAQ,EAAE,CAAC,GAAG,CAAC,CAAC;QAE9C,IAAI,QAAQ,IAAI,WAAW,EAAE,CAAC;YAC5B,QAAQ,CAAC,IAAI,CAAC;gBACZ,SAAS,EAAE,GAAG;gBACd,MAAM,EAAE,QAAQ,EAAE,MAAM,IAAI,WAAW,EAAE,MAAM,IAAI,IAAI;gBACvD,WAAW,EAAE,QAAQ,EAAE,WAAW,IAAI,WAAW,EAAE,WAAW,IAAI,EAAE;aACrE,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,4BAA4B;IAC5B,IAAI,aAA6C,CAAC;IAClD,IAAI,GAAG,CAAC,aAAa,EAAE,CAAC;QACtB,aAAa,GAAG;YACd,qBAAqB,EAAE,GAAG,CAAC,aAAa,CAAC,uBAAuB,IAAI,IAAI;YACxE,YAAY,EAAE,GAAG,CAAC,aAAa,CAAC,aAAa;YAC7C,eAAe,EAAE,GAAG,CAAC,aAAa,CAAC,iBAAiB;YACpD,iBAAiB,EAAE,GAAG,CAAC,aAAa,CAAC,oBAAoB;SAC1D,CAAC;IACJ,CAAC;IAED,sBAAsB;IACtB,IAAI,OAAkC,CAAC;IACvC,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;QAChB,OAAO,GAAG;YACR,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,KAAK,IAAI,SAAS;YACrC,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,KAAK;YACxB,QAAQ,EAAE,GAAG,CAAC,OAAO,CAAC,QAAQ;SAC/B,CAAC;IACJ,CAAC;IAED,OAAO;QACL,EAAE,EAAE,GAAG,CAAC,EAAE,IAAI,EAAE;QAChB,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,EAAE;QACxB,iBAAiB;QACjB,QAAQ;QACR,eAAe,EAAE,GAAG,CAAC,gBAAgB,IAAI,EAAE;QAC3C,aAAa;QACb,OAAO;KACR,CAAC;AACJ,CAAC;AAED,+CAA+C;AAC/C,aAAa;AACb,+CAA+C;AAE/C;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,QAAgB;IACrD,MAAM,MAAM,GAAa,EAAE,CAAC;IAE5B,IAAI,OAAe,CAAC;IACpB,IAAI,CAAC;QACH,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACjD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,qBAAqB,QAAQ,EAAE,CAAC,CAAC;IAC3C,CAAC;IAED,IAAI,GAAgB,CAAC;IACrB,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAgB,CAAC;IAC1C,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,CAAC,iBAAiB,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACzE,CAAC;IAED,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,OAAO,CAAC,eAAe,CAAC,CAAC;IAC3B,CAAC;IAED,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;QACf,MAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;IACjD,CAAC;IAED,IAAI,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;QAC5C,MAAM,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;QAChD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC1B,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC;QAE7B,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;YACb,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,gBAAgB,CAAC,CAAC;QACzC,CAAC;aAAM,IAAI,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;YAChC,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,wBAAwB,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC;QAC3D,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACvB,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,oBAAoB,CAAC,CAAC;QAC7C,CAAC;QAED,6CAA6C;QAC7C,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;iBACzC,MAAM,CAAC,CAAC,CAAC,EAAoB,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC;iBAChD,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC;YAChC,MAAM,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;YAC/C,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,GAAG,EAAE,CAAC;gBAClD,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,6BAA6B,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC;YACrF,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,+CAA+C;AAC/C,sBAAsB;AACtB,+CAA+C;AAE/C;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,SAAiB,EAAE,QAAQ,GAAG,CAAC;IAChE,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;IAEnD,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACtD,MAAM,MAAM,GAAG,GAAG,MAAM,IAAI,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;QAC7D,OAAO,WAAW,MAAM;kEACsC,SAAS;2BAChD,SAAS;;;;;4DAKwB,SAAS;8DACP,SAAS;;;;;wCAK/B,CAAC;IACvC,CAAC,CAAC,CAAC;IAEH,OAAO,UAAU,SAAS;;;;yBAIH,SAAS;;;;;;;EAOhC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC;CACnB,CAAC;AACF,CAAC"}

package/dist/src/pipeline.d.ts

@@ -0,0 +1,53 @@
+/**
+ * Evaluation pipeline orchestrator.
+ *
+ * Coordinates the full evaluation flow:
+ * parse tasks → setup skills → run agent → score → report → check thresholds
+ */
+import { type EvalConfig } from './config.js';
+import type { SkillEvaluation, TaskResult, CombinedScore, EvaluationReport } from './types.js';
+export interface PipelineOptions {
+    /** Path to tasks YAML file */
+    tasksFile: string;
+    /** Path to eval.config.yaml */
+    configPath?: string;
+    /** Config overrides from CLI flags */
+    configOverrides?: Partial<EvalConfig>;
+    /** Working directory for agent execution */
+    cwd?: string;
+    /** Path to skills directory (for local skill setup) */
+    skillsDir?: string;
+    /** Comma-separated task IDs to run (empty = all) */
+    taskFilter?: string;
+    /** Skip deterministic scoring */
+    noDeterministic?: boolean;
+    /** Skip LLM judge scoring */
+    noJudge?: boolean;
+    /** Enable verbose logging */
+    verbose?: boolean;
+}
+export interface PipelineResult {
+    passed: boolean;
+    failureReasons: string[];
+    evaluation: SkillEvaluation;
+    results: TaskResult[];
+    scores: CombinedScore[];
+    report: EvaluationReport;
+    reportPath?: string;
+    jsonPath?: string;
+    markdownSummary: string;
+}
+/**
+ * Run the full evaluation pipeline.
+ */
+export declare function runPipeline(options: PipelineOptions): Promise<PipelineResult>;
+/**
+ * Score existing results (no runner).
+ */
+export declare function scorePipeline(resultsPath: string, options?: {
+    configPath?: string;
+    configOverrides?: Partial<EvalConfig>;
+    noJudge?: boolean;
+    noDeterministic?: boolean;
+}): Promise<PipelineResult>;
+//# sourceMappingURL=pipeline.d.ts.map

package/dist/src/pipeline.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../src/pipeline.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAWH,OAAO,EAAc,KAAK,UAAU,EAAE,MAAM,aAAa,CAAC;AAC1D,OAAO,KAAK,EACV,eAAe,EACf,UAAU,EACV,aAAa,EACb,gBAAgB,EAGjB,MAAM,YAAY,CAAC;AAEpB,MAAM,WAAW,eAAe;IAC9B,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,+BAA+B;IAC/B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,sCAAsC;IACtC,eAAe,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;IACtC,4CAA4C;IAC5C,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,uDAAuD;IACvD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,oDAAoD;IACpD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iCAAiC;IACjC,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,6BAA6B;IAC7B,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,6BAA6B;IAC7B,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,OAAO,CAAC;IAChB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,UAAU,EAAE,eAAe,CAAC;IAC5B,OAAO,EAAE,UAAU,EAAE,CAAC;IACtB,MAAM,EAAE,aAAa,EAAE,CAAC;IACxB,MAAM,EAAE,gBAAgB,CAAC;IACzB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,eAAe,EAAE,MAAM,CAAC;CACzB;AAED;;GAEG;AACH,wBAAsB,WAAW,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC,CAuHnF;AAED;;GAEG;AACH,wBAAsB,aAAa,CACjC,WAAW,EAAE,MAAM,EACnB,OAAO,GAAE;IACP,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,eAAe,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;IACtC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,eAAe,CAAC,EAAE,OAAO,CAAC;CACtB,GACL,OAAO,CAAC,cAAc,CAAC,CA+CzB"}

package/dist/src/pipeline.js

@@ -0,0 +1,185 @@
+/**
+ * Evaluation pipeline orchestrator.
+ *
+ * Coordinates the full evaluation flow:
+ * parse tasks → setup skills → run agent → score → report → check thresholds
+ */
+import * as path from 'path';
+import * as fs from 'fs/promises';
+import { parseEvalFile } from './parser.js';
+import { SkillEvalRunner } from './runner/runner.js';
+import { setupLocalSkills, cleanupLocalSkills } from './runner/skill-setup.js';
+import { scoreAll } from './scorer/scorer.js';
+import { SessionLogger } from './session/session-logger.js';
+import { generateReport, generateJsonResults } from './report/report.js';
+import { generateGitHubSummary, writeGitHubSummary } from './report/github-summary.js';
+import { loadConfig } from './config.js';
+/**
+ * Run the full evaluation pipeline.
+ */
+export async function runPipeline(options) {
+    const config = await loadConfig(options.configPath, options.configOverrides);
+    const cwd = options.cwd || process.cwd();
+    // 1. Parse tasks
+    console.log(`Parsing tasks from: ${options.tasksFile}`);
+    let evaluation = await parseEvalFile(options.tasksFile);
+    // Filter tasks if specified
+    if (options.taskFilter) {
+        const filterIds = new Set(options.taskFilter.split(',').map((s) => s.trim()));
+        evaluation = {
+            ...evaluation,
+            tasks: evaluation.tasks.filter((t) => filterIds.has(t.id)),
+        };
+        console.log(`Filtered to ${evaluation.tasks.length} task(s): ${options.taskFilter}`);
+    }
+    if (evaluation.tasks.length === 0) {
+        throw new Error('No tasks to run');
+    }
+    console.log(`Running ${evaluation.tasks.length} task(s) for skill: ${evaluation.skillName}`);
+    // 2. Setup local skills
+    // Auto-detect skills/ directory relative to tasks file if not explicitly provided
+    let skillsDir = options.skillsDir;
+    if (!skillsDir) {
+        const tasksDir = path.dirname(path.resolve(options.tasksFile));
+        const autoSkillsDir = path.join(tasksDir, 'skills');
+        try {
+            const stat = await fs.stat(autoSkillsDir);
+            if (stat.isDirectory()) {
+                skillsDir = autoSkillsDir;
+            }
+        }
+        catch {
+            // No skills/ directory found, that's fine
+        }
+    }
+    let skillsSetup = false;
+    if (skillsDir) {
+        console.log(`Setting up local skills from: ${skillsDir}`);
+        const skillNames = await setupLocalSkills(skillsDir, cwd);
+        skillsSetup = true;
+        console.log(`Skills configured: ${skillNames.join(', ')}`);
+    }
+    try {
+        // 3. Run agent against tasks
+        console.log('\n--- Running Tasks ---\n');
+        const runner = new SkillEvalRunner({
+            cwd,
+            model: config.defaultAgentModel,
+            parallel: false,
+            allowedWriteDirs: config.allowedWriteDirs,
+        });
+        const logDir = path.join(config.outputDir, 'logs');
+        const results = await runner.runAll(evaluation, (task) => new SessionLogger(task.id, logDir));
+        // 4. Score results
+        console.log('\n--- Scoring ---\n');
+        const scorerOptions = {
+            noDeterministic: options.noDeterministic,
+            noJudge: options.noJudge,
+            judgeOptions: { model: config.defaultJudgeModel },
+        };
+        const scores = await scoreAll(evaluation.tasks, results, scorerOptions);
+        // 5. Generate reports
+        console.log('\n--- Generating Reports ---\n');
+        const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+        const reportBaseName = `${evaluation.skillName}-${timestamp}`;
+        const reportPath = path.join(config.outputDir, `${reportBaseName}.md`);
+        const jsonPath = path.join(config.outputDir, `${reportBaseName}.json`);
+        const metadata = {
+            skillPath: options.tasksFile,
+            agentModel: config.defaultAgentModel,
+            judgeModel: config.defaultJudgeModel,
+        };
+        await generateReport(evaluation, results, scores, reportPath, metadata);
+        const report = await generateJsonResults(evaluation, results, scores, jsonPath, metadata);
+        // 6. GitHub summary
+        if (config.githubSummary) {
+            const wrote = await writeGitHubSummary(report);
+            if (wrote) {
+                console.log('GitHub step summary written');
+            }
+        }
+        const markdownSummary = generateGitHubSummary(report);
+        // 7. Print summary
+        printSummary(report);
+        return {
+            passed: report.passed,
+            failureReasons: report.failureReasons,
+            evaluation,
+            results,
+            scores,
+            report,
+            reportPath,
+            jsonPath,
+            markdownSummary,
+        };
+    }
+    finally {
+        // Cleanup local skills
+        if (skillsSetup) {
+            await cleanupLocalSkills(cwd);
+        }
+    }
+}
+/**
+ * Score existing results (no runner).
+ */
+export async function scorePipeline(resultsPath, options = {}) {
+    const config = await loadConfig(options.configPath, options.configOverrides);
+    const data = JSON.parse(await fs.readFile(resultsPath, 'utf-8'));
+    const evaluation = {
+        skillName: data.skillName,
+        tasks: data.tasks,
+    };
+    const results = data.results;
+    const scorerOptions = {
+        noDeterministic: options.noDeterministic,
+        noJudge: options.noJudge,
+        judgeOptions: { model: config.defaultJudgeModel },
+    };
+    console.log(`Scoring ${results.length} result(s)...`);
+    const scores = await scoreAll(evaluation.tasks, results, scorerOptions);
+    const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+    const reportBaseName = `${evaluation.skillName}-scored-${timestamp}`;
+    const reportPath = path.join(config.outputDir, `${reportBaseName}.md`);
+    const jsonPath = path.join(config.outputDir, `${reportBaseName}.json`);
+    const metadata = {
+        skillPath: resultsPath,
+        agentModel: data.metadata?.agentModel || config.defaultAgentModel,
+        judgeModel: config.defaultJudgeModel,
+    };
+    await generateReport(evaluation, results, scores, reportPath, metadata);
+    const report = await generateJsonResults(evaluation, results, scores, jsonPath, metadata);
+    const markdownSummary = generateGitHubSummary(report);
+    printSummary(report);
+    return {
+        passed: report.passed,
+        failureReasons: report.failureReasons,
+        evaluation,
+        results,
+        scores,
+        report,
+        reportPath,
+        jsonPath,
+        markdownSummary,
+    };
+}
+function printSummary(report) {
+    const s = report.summary;
+    console.log('\n' + '='.repeat(50));
+    console.log(`  Skill Evaluation: ${report.skillName}`);
+    console.log('='.repeat(50));
+    console.log(`  Result: ${report.passed ? 'PASS' : 'FAIL'}`);
+    console.log(`  Discovery: ${(s.discoveryAccuracy * 100).toFixed(0)}%`);
+    console.log(`  Avg Adherence: ${s.avgAdherence.toFixed(2)}/5`);
+    console.log(`  Avg Output Quality: ${s.avgOutputQuality.toFixed(2)}/5`);
+    console.log(`  Weighted Score: ${s.avgWeightedScore.toFixed(2)}`);
+    console.log(`  Duration: ${(s.totalDurationMs / 1000).toFixed(1)}s | Cost: $${s.totalCostUsd.toFixed(4)}`);
+    if (!report.passed && report.failureReasons.length > 0) {
+        console.log(`\n  Failures:`);
+        for (const reason of report.failureReasons) {
+            console.log(`    - ${reason}`);
+        }
+    }
+    console.log('='.repeat(50));
+}
+//# sourceMappingURL=pipeline.js.map

package/dist/src/pipeline.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pipeline.js","sourceRoot":"","sources":["../../src/pipeline.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,EAAE,MAAM,aAAa,CAAC;AAClC,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAC/E,OAAO,EAAE,QAAQ,EAAsB,MAAM,oBAAoB,CAAC;AAClE,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,mBAAmB,EAAkB,MAAM,oBAAoB,CAAC;AACzF,OAAO,EAAE,qBAAqB,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AACvF,OAAO,EAAE,UAAU,EAAmB,MAAM,aAAa,CAAC;AA2C1D;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,OAAwB;IACxD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,eAAe,CAAC,CAAC;IAC7E,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IAEzC,iBAAiB;IACjB,OAAO,CAAC,GAAG,CAAC,uBAAuB,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;IACxD,IAAI,UAAU,GAAG,MAAM,aAAa,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAExD,4BAA4B;IAC5B,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;QACvB,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAC9E,UAAU,GAAG;YACX,GAAG,UAAU;YACb,KAAK,EAAE,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;SAC3D,CAAC;QACF,OAAO,CAAC,GAAG,CAAC,eAAe,UAAU,CAAC,KAAK,CAAC,MAAM,aAAa,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC;IACvF,CAAC;IAED,IAAI,UAAU,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;IACrC,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,WAAW,UAAU,CAAC,KAAK,CAAC,MAAM,uBAAuB,UAAU,CAAC,SAAS,EAAE,CAAC,CAAC;IAE7F,wBAAwB;IACxB,kFAAkF;IAClF,IAAI,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;IAClC,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC;QAC/D,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QACpD,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YAC1C,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;gBACvB,SAAS,GAAG,aAAa,CAAC;YAC5B,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,0CAA0C;QAC5C,CAAC;IACH,CAAC;IAED,IAAI,WAAW,GAAG,KAAK,CAAC;IACxB,IAAI,SAAS,EAAE,CAAC;QACd,OAAO,CAAC,GAAG,CAAC,iCAAiC,SAAS,EAAE,CAAC,CAAC;QAC1D,MAAM,UAAU,GAAG,MAAM,gBAAgB,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;QAC1D,WAAW,GAAG,IAAI,CAAC;QACnB,OAAO,CAAC,GAAG,CAAC,sBAAsB,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED,IAAI,CAAC;QACH,6BAA6B;QAC7B,OAAO,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC;QACzC,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC;YACjC,GAAG;YACH,KAAK,EAAE,MAAM,CAAC,iBAAiB;YAC/B,QAAQ,EAAE,KAAK;YACf,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;SAC1C,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QACnD,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,MAAM,CACjC,UAAU,EACV,CAAC,IAAc,EAAE,EAAE,CAAC,IAAI,aAAa,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,CACvD,CAAC;QAEF,mBAAmB;QACnB,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;QACnC,MAAM,aAAa,GAAkB;YACnC,eAAe,EAAE,OAAO,CAAC,eAAe;YACxC,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,YAAY,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,iBAAiB,EAAE;SAClD,CAAC;QACF,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,UAAU,CAAC,KAAK,EAAE,OAAO,EAAE,aAAa,CAAC,CAAC;QAExE,sBAAsB;QACtB,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;QAC9C,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QACjE,MAAM,cAAc,GAAG,GAAG,UAAU,CAAC,SAAS,IAAI,SAAS,EAAE,CAAC;QAC9D,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,GAAG,cAAc,KAAK,CAAC,CAAC;QACvE,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,GAAG,cAAc,OAAO,CAAC,CAAC;QAEvE,MAAM,QAAQ,GAAmB;YAC/B,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,UAAU,EAAE,MAAM,CAAC,iBAAiB;YACpC,UAAU,EAAE,MAAM,CAAC,iBAAiB;SACrC,CAAC;QAEF,MAAM,cAAc,CAAC,UAAU,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;QACxE,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC,UAAU,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;QAE1F,oBAAoB;QACpB,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;YACzB,MAAM,KAAK,GAAG,MAAM,kBAAkB,CAAC,MAAM,CAAC,CAAC;YAC/C,IAAI,KAAK,EAAE,CAAC;gBACV,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC;YAC7C,CAAC;QACH,CAAC;QAED,MAAM,eAAe,GAAG,qBAAqB,CAAC,MAAM,CAAC,CAAC;QAEtD,mBAAmB;QACnB,YAAY,CAAC,MAAM,CAAC,CAAC;QAErB,OAAO;YACL,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,cAAc,EAAE,MAAM,CAAC,cAAc;YACrC,UAAU;YACV,OAAO;YACP,MAAM;YACN,MAAM;YACN,UAAU;YACV,QAAQ;YACR,eAAe;SAChB,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,uBAAuB;QACvB,IAAI,WAAW,EAAE,CAAC;YAChB,MAAM,kBAAkB,CAAC,GAAG,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,WAAmB,EACnB,UAKI,EAAE;IAEN,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,eAAe,CAAC,CAAC;IAE7E,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC;IACjE,MAAM,UAAU,GAAoB;QAClC,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,KAAK,EAAE,IAAI,CAAC,KAAK;KAClB,CAAC;IACF,MAAM,OAAO,GAAiB,IAAI,CAAC,OAAO,CAAC;IAE3C,MAAM,aAAa,GAAkB;QACnC,eAAe,EAAE,OAAO,CAAC,eAAe;QACxC,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,YAAY,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,iBAAiB,EAAE;KAClD,CAAC;IAEF,OAAO,CAAC,GAAG,CAAC,WAAW,OAAO,CAAC,MAAM,eAAe,CAAC,CAAC;IACtD,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,UAAU,CAAC,KAAK,EAAE,OAAO,EAAE,aAAa,CAAC,CAAC;IAExE,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IACjE,MAAM,cAAc,GAAG,GAAG,UAAU,CAAC,SAAS,WAAW,SAAS,EAAE,CAAC;IACrE,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,GAAG,cAAc,KAAK,CAAC,CAAC;IACvE,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,GAAG,cAAc,OAAO,CAAC,CAAC;IAEvE,MAAM,QAAQ,GAAmB;QAC/B,SAAS,EAAE,WAAW;QACtB,UAAU,EAAE,IAAI,CAAC,QAAQ,EAAE,UAAU,IAAI,MAAM,CAAC,iBAAiB;QACjE,UAAU,EAAE,MAAM,CAAC,iBAAiB;KACrC,CAAC;IAEF,MAAM,cAAc,CAAC,UAAU,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;IACxE,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC,UAAU,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAE1F,MAAM,eAAe,GAAG,qBAAqB,CAAC,MAAM,CAAC,CAAC;IACtD,YAAY,CAAC,MAAM,CAAC,CAAC;IAErB,OAAO;QACL,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,cAAc,EAAE,MAAM,CAAC,cAAc;QACrC,UAAU;QACV,OAAO;QACP,MAAM;QACN,MAAM;QACN,UAAU;QACV,QAAQ;QACR,eAAe;KAChB,CAAC;AACJ,CAAC;AAED,SAAS,YAAY,CAAC,MAAwB;IAC5C,MAAM,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC;IACzB,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IACnC,OAAO,CAAC,GAAG,CAAC,uBAAuB,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;IACvD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5B,OAAO,CAAC,GAAG,CAAC,aAAa,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;IAC5D,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC,iBAAiB,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IACvE,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC/D,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACxE,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAClE,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC,eAAe,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC3G,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvD,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;QAC7B,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,cAAc,EAAE,CAAC;YAC3C,OAAO,CAAC,GAAG,CAAC,SAAS,MAAM,EAAE,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;AAC9B,CAAC"}

package/dist/src/report/github-summary.d.ts

@@ -0,0 +1,15 @@
+/**
+ * GitHub Actions job summary generation.
+ *
+ * Produces condensed markdown suitable for $GITHUB_STEP_SUMMARY.
+ */
+import type { EvaluationReport } from '../types.js';
+/**
+ * Generate a condensed summary for GitHub Actions.
+ */
+export declare function generateGitHubSummary(report: EvaluationReport): string;
+/**
+ * Write summary to $GITHUB_STEP_SUMMARY if available.
+ */
+export declare function writeGitHubSummary(report: EvaluationReport): Promise<boolean>;
+//# sourceMappingURL=github-summary.d.ts.map

package/dist/src/report/github-summary.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"github-summary.d.ts","sourceRoot":"","sources":["../../../src/report/github-summary.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,KAAK,EACV,gBAAgB,EAIjB,MAAM,aAAa,CAAC;AAErB;;GAEG;AACH,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,gBAAgB,GAAG,MAAM,CAqDtE;AAED;;GAEG;AACH,wBAAsB,kBAAkB,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,OAAO,CAAC,CAOnF"}

package/dist/src/report/github-summary.js

@@ -0,0 +1,77 @@
+/**
+ * GitHub Actions job summary generation.
+ *
+ * Produces condensed markdown suitable for $GITHUB_STEP_SUMMARY.
+ */
+import * as fs from 'fs/promises';
+/**
+ * Generate a condensed summary for GitHub Actions.
+ */
+export function generateGitHubSummary(report) {
+    const { summary, failureBreakdown, tasks } = report;
+    const lines = [];
+    const icon = report.passed ? ':white_check_mark:' : ':x:';
+    lines.push(`## ${icon} Skill Evaluation: ${report.skillName}`);
+    lines.push('');
+    // Summary table
+    lines.push('| Metric | Value | Status |');
+    lines.push('|--------|-------|--------|');
+    lines.push(`| Discovery Rate | ${(summary.discoveryAccuracy * 100).toFixed(0)}% (${Math.round(summary.discoveryAccuracy * summary.totalTasks)}/${summary.totalTasks}) | ${summary.discoveryAccuracy >= 0.8 ? 'PASS' : 'FAIL'} |`);
+    lines.push(`| Avg Adherence | ${summary.avgAdherence.toFixed(1)}/5 | ${summary.avgAdherence >= 4.0 ? 'PASS' : 'FAIL'} |`);
+    lines.push(`| Avg Output Quality | ${summary.avgOutputQuality.toFixed(1)}/5 | ${summary.avgOutputQuality >= 4.0 ? 'PASS' : 'FAIL'} |`);
+    lines.push(`| Weighted Score | ${summary.avgWeightedScore.toFixed(2)} | |`);
+    lines.push(`| Duration | ${(summary.totalDurationMs / 1000).toFixed(1)}s | |`);
+    lines.push(`| Cost | $${summary.totalCostUsd.toFixed(4)} | |`);
+    lines.push('');
+    // Failures
+    const failures = tasks.filter((t) => t.score.failureCategory !== 'none');
+    if (failures.length > 0) {
+        lines.push(`### Failures (${failures.length})`);
+        lines.push('');
+        lines.push('| Task | Category | Details |');
+        lines.push('|------|----------|---------|');
+        for (const f of failures) {
+            const cat = formatCategory(f.score.failureCategory);
+            const reason = f.score.reasoning.slice(0, 80) + (f.score.reasoning.length > 80 ? '...' : '');
+            lines.push(`| ${f.task.id} | ${cat} | ${reason} |`);
+        }
+        lines.push('');
+    }
+    // Per-task details in collapsible
+    lines.push('<details><summary>All task results</summary>');
+    lines.push('');
+    lines.push('| Task | Discovery | Adherence | Output | Weighted | Status |');
+    lines.push('|------|-----------|-----------|--------|----------|--------|');
+    for (const t of tasks) {
+        const s = t.score;
+        const status = s.failureCategory === 'none' ? 'PASS' : 'FAIL';
+        lines.push(`| ${t.task.id} | ${s.discovery} | ${s.adherence}/5 | ${s.outputQuality}/5 | ${s.weightedScore.toFixed(2)} | ${status} |`);
+    }
+    lines.push('');
+    lines.push('</details>');
+    lines.push('');
+    if (!report.passed && report.failureReasons.length > 0) {
+        lines.push(`**Failure reasons:** ${report.failureReasons.join('; ')}`);
+    }
+    return lines.join('\n');
+}
+/**
+ * Write summary to $GITHUB_STEP_SUMMARY if available.
+ */
+export async function writeGitHubSummary(report) {
+    const summaryPath = process.env.GITHUB_STEP_SUMMARY;
+    if (!summaryPath)
+        return false;
+    const summary = generateGitHubSummary(report);
+    await fs.appendFile(summaryPath, summary + '\n');
+    return true;
+}
+function formatCategory(cat) {
+    if (cat === 'none')
+        return 'No Failure';
+    return cat
+        .split('_')
+        .map((w) => w.charAt(0).toUpperCase() + w.slice(1))
+        .join(' ');
+}
+//# sourceMappingURL=github-summary.js.map

package/dist/src/report/github-summary.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"github-summary.js","sourceRoot":"","sources":["../../../src/report/github-summary.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,MAAM,aAAa,CAAC;AAQlC;;GAEG;AACH,MAAM,UAAU,qBAAqB,CAAC,MAAwB;IAC5D,MAAM,EAAE,OAAO,EAAE,gBAAgB,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;IACpD,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,KAAK,CAAC;IAC1D,KAAK,CAAC,IAAI,CAAC,MAAM,IAAI,sBAAsB,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;IAC/D,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,gBAAgB;IAChB,KAAK,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;IAC1C,KAAK,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;IAC1C,KAAK,CAAC,IAAI,CAAC,sBAAsB,CAAC,OAAO,CAAC,iBAAiB,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,iBAAiB,GAAG,OAAO,CAAC,UAAU,CAAC,IAAI,OAAO,CAAC,UAAU,OAAO,OAAO,CAAC,iBAAiB,IAAI,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC;IAClO,KAAK,CAAC,IAAI,CAAC,qBAAqB,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,OAAO,CAAC,YAAY,IAAI,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC;IAC1H,KAAK,CAAC,IAAI,CAAC,0BAA0B,OAAO,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,OAAO,CAAC,gBAAgB,IAAI,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC;IACvI,KAAK,CAAC,IAAI,CAAC,sBAAsB,OAAO,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAC5E,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IAC/E,KAAK,CAAC,IAAI,CAAC,aAAa,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAC/D,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,WAAW;IACX,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,eAAe,KAAK,MAAM,CAAC,CAAC;IACzE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,KAAK,CAAC,IAAI,CAAC,iBAAiB,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;QAChD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;QAC5C,KAAK,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;QAC5C,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;YACzB,MAAM,GAAG,GAAG,cAAc,CAAC,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;YACpD,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC7F,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,MAAM,IAAI,CAAC,CAAC;QACtD,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,kCAAkC;IAClC,KAAK,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAC;IAC3D,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,+DAA+D,CAAC,CAAC;IAC5E,KAAK,CAAC,IAAI,CAAC,+DAA+D,CAAC,CAAC;IAC5E,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC;QAClB,MAAM,MAAM,GAAG,CAAC,CAAC,eAAe,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;QAC9D,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC,SAAS,MAAM,CAAC,CAAC,SAAS,QAAQ,CAAC,CAAC,aAAa,QAAQ,CAAC,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,MAAM,IAAI,CAAC,CAAC;IACxI,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IACzB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvD,KAAK,CAAC,IAAI,CAAC,wBAAwB,MAAM,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzE,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,MAAwB;IAC/D,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC;IACpD,IAAI,CAAC,WAAW;QAAE,OAAO,KAAK,CAAC;IAE/B,MAAM,OAAO,GAAG,qBAAqB,CAAC,MAAM,CAAC,CAAC;IAC9C,MAAM,EAAE,CAAC,UAAU,CAAC,WAAW,EAAE,OAAO,GAAG,IAAI,CAAC,CAAC;IACjD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,cAAc,CAAC,GAAW;IACjC,IAAI,GAAG,KAAK,MAAM;QAAE,OAAO,YAAY,CAAC;IACxC,OAAO,GAAG;SACP,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;SAClD,IAAI,CAAC,GAAG,CAAC,CAAC;AACf,CAAC"}

package/dist/src/report/report.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Report generation for skill evaluation results.
+ *
+ * Generates markdown and JSON reports from combined evaluation scores.
+ */
+import type { SkillEvaluation, TaskResult, CombinedScore, EvaluationReport, EvaluationSummary, FailureBreakdown, ReportMetadata } from '../types.js';
+/**
+ * Generate a markdown report from evaluation results.
+ */
+export declare function generateReport(evaluation: SkillEvaluation, results: TaskResult[], scores: CombinedScore[], outputPath?: string, metadata?: ReportMetadata): Promise<string>;
+/**
+ * Generate JSON report for programmatic analysis.
+ */
+export declare function generateJsonResults(evaluation: SkillEvaluation, results: TaskResult[], scores: CombinedScore[], outputPath?: string, metadata?: ReportMetadata): Promise<EvaluationReport>;
+/**
+ * Compute summary statistics from combined scores.
+ */
+export declare function computeSummary(results: TaskResult[], scores: CombinedScore[]): EvaluationSummary;
+/**
+ * Compute failure category breakdown.
+ */
+export declare function computeFailureBreakdown(scores: CombinedScore[]): FailureBreakdown[];
+//# sourceMappingURL=report.d.ts.map

package/dist/src/report/report.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../../../src/report/report.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAIH,OAAO,KAAK,EACV,eAAe,EACf,UAAU,EACV,aAAa,EACb,gBAAgB,EAChB,iBAAiB,EACjB,gBAAgB,EAEhB,cAAc,EACf,MAAM,aAAa,CAAC;AAGrB;;GAEG;AACH,wBAAsB,cAAc,CAClC,UAAU,EAAE,eAAe,EAC3B,OAAO,EAAE,UAAU,EAAE,EACrB,MAAM,EAAE,aAAa,EAAE,EACvB,UAAU,CAAC,EAAE,MAAM,EACnB,QAAQ,CAAC,EAAE,cAAc,GACxB,OAAO,CAAC,MAAM,CAAC,CAsHjB;AAED;;GAEG;AACH,wBAAsB,mBAAmB,CACvC,UAAU,EAAE,eAAe,EAC3B,OAAO,EAAE,UAAU,EAAE,EACrB,MAAM,EAAE,aAAa,EAAE,EACvB,UAAU,CAAC,EAAE,MAAM,EACnB,QAAQ,CAAC,EAAE,cAAc,GACxB,OAAO,CAAC,gBAAgB,CAAC,CAuD3B;AAED;;GAEG;AACH,wBAAgB,cAAc,CAC5B,OAAO,EAAE,UAAU,EAAE,EACrB,MAAM,EAAE,aAAa,EAAE,GACtB,iBAAiB,CAmBnB;AAED;;GAEG;AACH,wBAAgB,uBAAuB,CAAC,MAAM,EAAE,aAAa,EAAE,GAAG,gBAAgB,EAAE,CAenF"}