@skilljack/evals 1.0.0

package/dist/src/cli.js

@@ -0,0 +1,264 @@
+#!/usr/bin/env node
+/**
+ * CLI for skill evaluation framework.
+ *
+ * Primary command: `skilljack-evals run` — runs the full evaluation pipeline.
+ * Also supports: score, report, create-eval, validate.
+ */
+import 'dotenv/config';
+import { Command } from 'commander';
+import * as fs from 'fs/promises';
+import * as path from 'path';
+import { parseEvalFile, createEvalTemplate, validateEvalFile } from './parser.js';
+import { runPipeline, scorePipeline } from './pipeline.js';
+import { generateReport, generateJsonResults } from './report/report.js';
+import { SkillJudge } from './scorer/judge.js';
+const program = new Command();
+program
+    .name('skilljack-evals')
+    .description('Skill evaluation CLI — run evaluations, score results, generate reports')
+    .version('1.0.0');
+// ============================================
+// Primary command: run
+// ============================================
+program
+    .command('run')
+    .description('Run the full evaluation pipeline: execute tasks → score → report')
+    .argument('<tasks>', 'Path to tasks YAML file')
+    .option('--model <model>', 'Agent model (default: sonnet)')
+    .option('--judge-model <model>', 'Judge model (default: haiku)')
+    .option('--config <path>', 'Path to eval.config.yaml')
+    .option('--output-dir <dir>', 'Output directory for results')
+    .option('--timeout <ms>', 'Per-task timeout in milliseconds')
+    .option('--tasks <ids>', 'Comma-separated task IDs to run')
+    .option('--skills-dir <path>', 'Path to skills directory for local setup')
+    .option('--cwd <path>', 'Working directory for agent execution')
+    .option('--threshold-discovery <rate>', 'Min discovery rate (0-1)')
+    .option('--threshold-score <score>', 'Min avg score (1-5)')
+    .option('--no-judge', 'Skip LLM judge scoring (deterministic only)')
+    .option('--no-deterministic', 'Skip deterministic scoring (LLM judge only)')
+    .option('--github-summary', 'Write GitHub Actions step summary')
+    .option('--verbose', 'Enable verbose output')
+    .action(async (tasksFile, options) => {
+    try {
+        const configOverrides = {};
+        if (options.model)
+            configOverrides.defaultAgentModel = options.model;
+        if (options.judgeModel)
+            configOverrides.defaultJudgeModel = options.judgeModel;
+        if (options.outputDir)
+            configOverrides.outputDir = options.outputDir;
+        if (options.timeout)
+            configOverrides.taskTimeoutMs = parseInt(options.timeout, 10);
+        if (options.thresholdDiscovery)
+            configOverrides.discoveryThreshold = parseFloat(options.thresholdDiscovery);
+        if (options.thresholdScore)
+            configOverrides.scoreThreshold = parseFloat(options.thresholdScore);
+        if (options.githubSummary)
+            configOverrides.githubSummary = true;
+        const result = await runPipeline({
+            tasksFile,
+            configPath: options.config,
+            configOverrides,
+            cwd: options.cwd,
+            skillsDir: options.skillsDir,
+            taskFilter: options.tasks,
+            noJudge: options.judge === false,
+            noDeterministic: options.deterministic === false,
+            verbose: options.verbose,
+        });
+        if (!result.passed) {
+            process.exit(1);
+        }
+    }
+    catch (error) {
+        console.error(`Error: ${error instanceof Error ? error.message : String(error)}`);
+        process.exit(1);
+    }
+});
+// ============================================
+// Score existing results
+// ============================================
+program
+    .command('score')
+    .description('Score existing results JSON (no runner)')
+    .argument('<results>', 'Path to results JSON file')
+    .option('--judge-model <model>', 'Judge model')
+    .option('--config <path>', 'Path to eval.config.yaml')
+    .option('--no-judge', 'Skip LLM judge')
+    .option('--no-deterministic', 'Skip deterministic checks')
+    .action(async (resultsFile, options) => {
+    try {
+        const configOverrides = {};
+        if (options.judgeModel)
+            configOverrides.defaultJudgeModel = options.judgeModel;
+        const result = await scorePipeline(resultsFile, {
+            configPath: options.config,
+            configOverrides,
+            noJudge: options.judge === false,
+            noDeterministic: options.deterministic === false,
+        });
+        if (!result.passed) {
+            process.exit(1);
+        }
+    }
+    catch (error) {
+        console.error(`Error: ${error instanceof Error ? error.message : String(error)}`);
+        process.exit(1);
+    }
+});
+// ============================================
+// Report generation
+// ============================================
+program
+    .command('report')
+    .description('Generate report from results JSON')
+    .requiredOption('-r, --results <path>', 'Path to results JSON file')
+    .option('-o, --output <path>', 'Output markdown file')
+    .option('--json <path>', 'Also output JSON report')
+    .action(async (options) => {
+    try {
+        const resultsData = await fs.readFile(options.results, 'utf-8');
+        const data = JSON.parse(resultsData);
+        const evaluation = {
+            skillName: data.skillName,
+            tasks: data.tasks.map((t) => t.task),
+        };
+        const results = data.tasks.map((t) => t.result);
+        const scores = data.tasks.map((t) => t.score);
+        const report = await generateReport(evaluation, results, scores, options.output, data.metadata);
+        if (!options.output) {
+            console.log(report);
+        }
+        if (options.json) {
+            await generateJsonResults(evaluation, results, scores, options.json, data.metadata);
+        }
+    }
+    catch (error) {
+        console.error(`Error: ${error instanceof Error ? error.message : String(error)}`);
+        process.exit(1);
+    }
+});
+// ============================================
+// Create eval template
+// ============================================
+program
+    .command('create-eval')
+    .description('Create an evaluation template for a skill')
+    .argument('<skill_name>', 'Name of the skill')
+    .option('-o, --output <path>', 'Output path for template')
+    .option('-n, --num-tasks <number>', 'Number of placeholder tasks', '10')
+    .action(async (skillName, options) => {
+    const outputPath = options.output || path.join(process.cwd(), 'evals', skillName, 'tasks.yaml');
+    const numTasks = parseInt(options.numTasks, 10);
+    await fs.mkdir(path.dirname(outputPath), { recursive: true });
+    const template = createEvalTemplate(skillName, numTasks);
+    await fs.writeFile(outputPath, template);
+    console.log(`Created evaluation template: ${outputPath}`);
+    console.log();
+    console.log('Next steps:');
+    console.log(`1. Edit ${outputPath} to add real evaluation tasks`);
+    console.log(`2. Run: skilljack-evals run ${outputPath}`);
+});
+// ============================================
+// Parse (legacy, for interop)
+// ============================================
+program
+    .command('parse')
+    .description('Parse tasks YAML and output JSON')
+    .argument('<skill_name>', 'Name of the skill')
+    .option('-f, --eval-file <path>', 'Path to evaluation YAML file')
+    .option('-o, --output <path>', 'Output JSON file (default: stdout)')
+    .action(async (skillName, options) => {
+    const baseDir = path.join(process.cwd(), 'evals', skillName);
+    const evalFile = options.evalFile || path.join(baseDir, 'tasks.yaml');
+    try {
+        await fs.access(evalFile);
+    }
+    catch {
+        console.error(`Error: Evaluation file not found: ${evalFile}`);
+        process.exit(1);
+    }
+    const evaluation = await parseEvalFile(evalFile);
+    const json = JSON.stringify(evaluation, null, 2);
+    if (options.output) {
+        await fs.writeFile(options.output, json);
+        console.error(`Parsed ${evaluation.tasks.length} tasks to: ${options.output}`);
+    }
+    else {
+        console.log(json);
+    }
+});
+// ============================================
+// Validate
+// ============================================
+program
+    .command('validate')
+    .description('Validate a tasks YAML file')
+    .argument('<file>', 'Path to tasks YAML file')
+    .action(async (file) => {
+    const errors = await validateEvalFile(file);
+    if (errors.length === 0) {
+        const evaluation = await parseEvalFile(file);
+        console.log(`Valid: ${evaluation.tasks.length} task(s) for skill '${evaluation.skillName}'`);
+    }
+    else {
+        console.error(`Validation errors in ${file}:`);
+        for (const error of errors) {
+            console.error(`  - ${error}`);
+        }
+        process.exit(1);
+    }
+});
+// ============================================
+// Judge (legacy single-task scoring)
+// ============================================
+program
+    .command('judge')
+    .description('Score a single task result using LLM judge')
+    .requiredOption('--task-id <id>', 'Task ID')
+    .requiredOption('--prompt <text>', 'Original task prompt')
+    .requiredOption('--expected-skill <name>', 'Expected skill to be loaded')
+    .requiredOption('--output <text>', 'Agent output to judge')
+    .option('--skill-loads <skills>', 'Comma-separated list of skills loaded', '')
+    .option('--checklist <items>', 'Comma-separated golden checklist items', '')
+    .option('--model <model>', 'Judge model (default: haiku)')
+    .option('-o, --output-file <path>', 'Output JSON file (default: stdout)')
+    .action(async (options) => {
+    const task = {
+        id: options.taskId,
+        prompt: options.prompt,
+        expectedSkillLoad: options.expectedSkill,
+        criteria: [
+            { dimension: 'discovery', weight: 0.3, description: 'Skill discovery' },
+            { dimension: 'adherence', weight: 0.4, description: 'Instruction adherence' },
+            { dimension: 'output', weight: 0.3, description: 'Output quality' },
+        ],
+        goldenChecklist: options.checklist ? options.checklist.split(',').map(s => s.trim()) : [],
+    };
+    const result = {
+        taskId: options.taskId,
+        prompt: options.prompt,
+        output: options.output,
+        durationMs: 0,
+        numTurns: 0,
+        costUsd: 0,
+        skillLoads: options.skillLoads ? options.skillLoads.split(',').map(s => s.trim()) : [],
+        toolCalls: [],
+        isError: false,
+        errorMessage: '',
+    };
+    const judge = new SkillJudge({ model: options.model });
+    console.error(`Judging task ${options.taskId}...`);
+    const score = await judge.judgeResult(task, result);
+    const json = JSON.stringify(score, null, 2);
+    if (options.outputFile) {
+        await fs.writeFile(options.outputFile, json);
+        console.error(`Score saved to: ${options.outputFile}`);
+    }
+    else {
+        console.log(json);
+    }
+});
+program.parse();
+//# sourceMappingURL=cli.js.map

package/dist/src/cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../src/cli.ts"],"names":[],"mappings":";AAEA;;;;;GAKG;AAEH,OAAO,eAAe,CAAC;AACvB,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,KAAK,EAAE,MAAM,aAAa,CAAC;AAClC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAClF,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC3D,OAAO,EAAE,cAAc,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzE,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAI/C,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;AAE9B,OAAO;KACJ,IAAI,CAAC,iBAAiB,CAAC;KACvB,WAAW,CAAC,yEAAyE,CAAC;KACtF,OAAO,CAAC,OAAO,CAAC,CAAC;AAEpB,+CAA+C;AAC/C,uBAAuB;AACvB,+CAA+C;AAE/C,OAAO;KACJ,OAAO,CAAC,KAAK,CAAC;KACd,WAAW,CAAC,kEAAkE,CAAC;KAC/E,QAAQ,CAAC,SAAS,EAAE,yBAAyB,CAAC;KAC9C,MAAM,CAAC,iBAAiB,EAAE,+BAA+B,CAAC;KAC1D,MAAM,CAAC,uBAAuB,EAAE,8BAA8B,CAAC;KAC/D,MAAM,CAAC,iBAAiB,EAAE,0BAA0B,CAAC;KACrD,MAAM,CAAC,oBAAoB,EAAE,8BAA8B,CAAC;KAC5D,MAAM,CAAC,gBAAgB,EAAE,kCAAkC,CAAC;KAC5D,MAAM,CAAC,eAAe,EAAE,iCAAiC,CAAC;KAC1D,MAAM,CAAC,qBAAqB,EAAE,0CAA0C,CAAC;KACzE,MAAM,CAAC,cAAc,EAAE,uCAAuC,CAAC;KAC/D,MAAM,CAAC,8BAA8B,EAAE,0BAA0B,CAAC;KAClE,MAAM,CAAC,2BAA2B,EAAE,qBAAqB,CAAC;KAC1D,MAAM,CAAC,YAAY,EAAE,6CAA6C,CAAC;KACnE,MAAM,CAAC,oBAAoB,EAAE,6CAA6C,CAAC;KAC3E,MAAM,CAAC,kBAAkB,EAAE,mCAAmC,CAAC;KAC/D,MAAM,CAAC,WAAW,EAAE,uBAAuB,CAAC;KAC5C,MAAM,CAAC,KAAK,EAAE,SAAiB,EAAE,OAejC,EAAE,EAAE;IACH,IAAI,CAAC;QACH,MAAM,eAAe,GAAwB,EAAE,CAAC;QAChD,IAAI,OAAO,CAAC,KAAK;YAAE,eAAe,CAAC,iBAAiB,GAAG,OAAO,CAAC,KAAK,CAAC;QACrE,IAAI,OAAO,CAAC,UAAU;YAAE,eAAe,CAAC,iBAAiB,GAAG,OAAO,CAAC,UAAU,CAAC;QAC/E,IAAI,OAAO,CAAC,SAAS;YAAE,eAAe,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QACrE,IAAI,OAAO,CAAC,OAAO;YAAE,eAAe,CAAC,aAAa,GAAG,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QACnF,IAAI,OAAO,CAAC,kBAAkB;YAAE,eAAe,CAAC,kBAAkB,GAAG,UAAU,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;QAC5G,IAAI,OAAO,CAAC,cAAc;YAAE,eAAe,CAAC,cAAc,GAAG,UAAU,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;QAChG,IAAI,OAAO,CAAC,aAAa;YAAE,eAAe,CAAC,aAAa,GAAG,IAAI,CAAC;QAEhE,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC;YAC/B,SAAS;YACT,UAAU,EAAE,OAAO,CAAC,MAAM;YAC1B,eAAe;YACf,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,UAAU,EAAE,OAAO,CAAC,KAAK;YACzB,OAAO,EAAE,OAAO,CAAC,KAAK,KAAK,KAAK;YAChC,eAAe,EAAE,OAAO,CAAC,aAAa,KAAK,KAAK;YAChD,OAAO,EAAE,OAAO,CAAC,OAAO;SACzB,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACnB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,UAAU,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAClF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,+CAA+C;AAC/C,yBAAyB;AACzB,+CAA+C;AAE/C,OAAO;KACJ,OAAO,CAAC,OAAO,CAAC;KAChB,WAAW,CAAC,yCAAyC,CAAC;KACtD,QAAQ,CAAC,WAAW,EAAE,2BAA2B,CAAC;KAClD,MAAM,CAAC,uBAAuB,EAAE,aAAa,CAAC;KAC9C,MAAM,CAAC,iBAAiB,EAAE,0BAA0B,CAAC;KACrD,MAAM,CAAC,YAAY,EAAE,gBAAgB,CAAC;KACtC,MAAM,CAAC,oBAAoB,EAAE,2BAA2B,CAAC;KACzD,MAAM,CAAC,KAAK,EAAE,WAAmB,EAAE,OAKnC,EAAE,EAAE;IACH,IAAI,CAAC;QACH,MAAM,eAAe,GAAwB,EAAE,CAAC;QAChD,IAAI,OAAO,CAAC,UAAU;YAAE,eAAe,CAAC,iBAAiB,GAAG,OAAO,CAAC,UAAU,CAAC;QAE/E,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,WAAW,EAAE;YAC9C,UAAU,EAAE,OAAO,CAAC,MAAM;YAC1B,eAAe;YACf,OAAO,EAAE,OAAO,CAAC,KAAK,KAAK,KAAK;YAChC,eAAe,EAAE,OAAO,CAAC,aAAa,KAAK,KAAK;SACjD,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACnB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,UAAU,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAClF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,+CAA+C;AAC/C,oBAAoB;AACpB,+CAA+C;AAE/C,OAAO;KACJ,OAAO,CAAC,QAAQ,CAAC;KACjB,WAAW,CAAC,mCAAmC,CAAC;KAChD,cAAc,CAAC,sBAAsB,EAAE,2BAA2B,CAAC;KACnE,MAAM,CAAC,qBAAqB,EAAE,sBAAsB,CAAC;KACrD,MAAM,CAAC,eAAe,EAAE,yBAAyB,CAAC;KAClD,MAAM,CAAC,KAAK,EAAE,OAId,EAAE,EAAE;IACH,IAAI,CAAC;QACH,MAAM,WAAW,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAChE,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAIlC,CAAC;QAEF,MAAM,UAAU,GAAoB;YAClC,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;SACrC,CAAC;QACF,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QAChD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QAE9C,MAAM,MAAM,GAAG,MAAM,cAAc,CACjC,UAAU,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,CAC3D,CAAC;QAEF,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;YACpB,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACtB,CAAC;QAED,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACjB,MAAM,mBAAmB,CACvB,UAAU,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,QAAQ,CACzD,CAAC;QACJ,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,UAAU,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAClF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,+CAA+C;AAC/C,uBAAuB;AACvB,+CAA+C;AAE/C,OAAO;KACJ,OAAO,CAAC,aAAa,CAAC;KACtB,WAAW,CAAC,2CAA2C,CAAC;KACxD,QAAQ,CAAC,cAAc,EAAE,mBAAmB,CAAC;KAC7C,MAAM,CAAC,qBAAqB,EAAE,0BAA0B,CAAC;KACzD,MAAM,CAAC,0BAA0B,EAAE,6BAA6B,EAAE,IAAI,CAAC;KACvE,MAAM,CAAC,KAAK,EAAE,SAAiB,EAAE,OAGjC,EAAE,EAAE;IACH,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC;IAChG,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IAEhD,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE9D,MAAM,QAAQ,GAAG,kBAAkB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IACzD,MAAM,EAAE,CAAC,SAAS,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;IAEzC,OAAO,CAAC,GAAG,CAAC,gCAAgC,UAAU,EAAE,CAAC,CAAC;IAC1D,OAAO,CAAC,GAAG,EAAE,CAAC;IACd,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;IAC3B,OAAO,CAAC,GAAG,CAAC,WAAW,UAAU,+BAA+B,CAAC,CAAC;IAClE,OAAO,CAAC,GAAG,CAAC,+BAA+B,UAAU,EAAE,CAAC,CAAC;AAC3D,CAAC,CAAC,CAAC;AAEL,+CAA+C;AAC/C,8BAA8B;AAC9B,+CAA+C;AAE/C,OAAO;KACJ,OAAO,CAAC,OAAO,CAAC;KAChB,WAAW,CAAC,kCAAkC,CAAC;KAC/C,QAAQ,CAAC,cAAc,EAAE,mBAAmB,CAAC;KAC7C,MAAM,CAAC,wBAAwB,EAAE,8BAA8B,CAAC;KAChE,MAAM,CAAC,qBAAqB,EAAE,oCAAoC,CAAC;KACnE,MAAM,CAAC,KAAK,EAAE,SAAiB,EAAE,OAGjC,EAAE,EAAE;IACH,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IAC7D,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;IAEtE,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC5B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,KAAK,CAAC,qCAAqC,QAAQ,EAAE,CAAC,CAAC;QAC/D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,UAAU,GAAG,MAAM,aAAa,CAAC,QAAQ,CAAC,CAAC;IACjD,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IAEjD,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QACzC,OAAO,CAAC,KAAK,CAAC,UAAU,UAAU,CAAC,KAAK,CAAC,MAAM,cAAc,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IACjF,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,+CAA+C;AAC/C,WAAW;AACX,+CAA+C;AAE/C,OAAO;KACJ,OAAO,CAAC,UAAU,CAAC;KACnB,WAAW,CAAC,4BAA4B,CAAC;KACzC,QAAQ,CAAC,QAAQ,EAAE,yBAAyB,CAAC;KAC7C,MAAM,CAAC,KAAK,EAAE,IAAY,EAAE,EAAE;IAC7B,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,IAAI,CAAC,CAAC;IAE5C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,MAAM,UAAU,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,CAAC;QAC7C,OAAO,CAAC,GAAG,CAAC,UAAU,UAAU,CAAC,KAAK,CAAC,MAAM,uBAAuB,UAAU,CAAC,SAAS,GAAG,CAAC,CAAC;IAC/F,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,KAAK,CAAC,wBAAwB,IAAI,GAAG,CAAC,CAAC;QAC/C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,OAAO,CAAC,KAAK,CAAC,OAAO,KAAK,EAAE,CAAC,CAAC;QAChC,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,+CAA+C;AAC/C,qCAAqC;AACrC,+CAA+C;AAE/C,OAAO;KACJ,OAAO,CAAC,OAAO,CAAC;KAChB,WAAW,CAAC,4CAA4C,CAAC;KACzD,cAAc,CAAC,gBAAgB,EAAE,SAAS,CAAC;KAC3C,cAAc,CAAC,iBAAiB,EAAE,sBAAsB,CAAC;KACzD,cAAc,CAAC,yBAAyB,EAAE,6BAA6B,CAAC;KACxE,cAAc,CAAC,iBAAiB,EAAE,uBAAuB,CAAC;KAC1D,MAAM,CAAC,wBAAwB,EAAE,uCAAuC,EAAE,EAAE,CAAC;KAC7E,MAAM,CAAC,qBAAqB,EAAE,wCAAwC,EAAE,EAAE,CAAC;KAC3E,MAAM,CAAC,iBAAiB,EAAE,8BAA8B,CAAC;KACzD,MAAM,CAAC,0BAA0B,EAAE,oCAAoC,CAAC;KACxE,MAAM,CAAC,KAAK,EAAE,OASd,EAAE,EAAE;IACH,MAAM,IAAI,GAAa;QACrB,EAAE,EAAE,OAAO,CAAC,MAAM;QAClB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,iBAAiB,EAAE,OAAO,CAAC,aAAa;QACxC,QAAQ,EAAE;YACR,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,EAAE,GAAG,EAAE,WAAW,EAAE,iBAAiB,EAAE;YACvE,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,EAAE,GAAG,EAAE,WAAW,EAAE,uBAAuB,EAAE;YAC7E,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,WAAW,EAAE,gBAAgB,EAAE;SACpE;QACD,eAAe,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE;KAC1F,CAAC;IAEF,MAAM,MAAM,GAAe;QACzB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,UAAU,EAAE,CAAC;QACb,QAAQ,EAAE,CAAC;QACX,OAAO,EAAE,CAAC;QACV,UAAU,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE;QACtF,SAAS,EAAE,EAAE;QACb,OAAO,EAAE,KAAK;QACd,YAAY,EAAE,EAAE;KACjB,CAAC;IAEF,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;IACvD,OAAO,CAAC,KAAK,CAAC,gBAAgB,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC;IACnD,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,WAAW,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAEpD,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IAC5C,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;QACvB,MAAM,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;QAC7C,OAAO,CAAC,KAAK,CAAC,mBAAmB,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC;IACzD,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO,CAAC,KAAK,EAAE,CAAC"}

package/dist/src/config.d.ts

@@ -0,0 +1,52 @@
+/**
+ * Centralized configuration for the skill evaluation framework.
+ *
+ * Configuration is loaded with the following precedence (lowest to highest):
+ * 1. Built-in defaults
+ * 2. Config file (eval.config.yaml or custom path)
+ * 3. Environment variables (EVAL_* prefix)
+ * 4. Programmatic overrides (CLI flags or API)
+ *
+ * Supports both Anthropic API and Bedrock via Agent SDK env vars:
+ * - Anthropic: Set ANTHROPIC_API_KEY
+ * - Bedrock: Set CLAUDE_CODE_USE_BEDROCK=1, AWS_REGION, AWS_PROFILE
+ */
+export interface EvalConfig {
+    defaultAgentModel: string;
+    defaultJudgeModel: string;
+    defaultWeights: {
+        discovery: number;
+        adherence: number;
+        output: number;
+    };
+    judgeOutputTruncation: number;
+    reportOutputTruncation: number;
+    taskTimeoutMs: number;
+    exitOnFailure: boolean;
+    outputDir: string;
+    githubSummary: boolean;
+    discoveryThreshold: number;
+    scoreThreshold: number;
+    allowedWriteDirs: string[];
+}
+/**
+ * Default configuration values.
+ */
+export declare const DEFAULT_CONFIG: EvalConfig;
+/**
+ * Load full configuration with all sources merged.
+ *
+ * @param configPath - Optional path to eval.config.yaml
+ * @param overrides - Optional programmatic overrides (CLI flags)
+ */
+export declare function loadConfig(configPath?: string, overrides?: Partial<EvalConfig>): Promise<EvalConfig>;
+/**
+ * Load configuration synchronously (env vars + defaults only, no file).
+ * Useful when you can't await — e.g., in constructors.
+ */
+export declare function loadConfigSync(overrides?: Partial<EvalConfig>): EvalConfig;
+/**
+ * Get default weights for scoring dimensions.
+ */
+export declare function getDefaultWeights(config?: EvalConfig): Map<string, number>;
+//# sourceMappingURL=config.d.ts.map

package/dist/src/config.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/config.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAMH,MAAM,WAAW,UAAU;IAEzB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,iBAAiB,EAAE,MAAM,CAAC;IAG1B,cAAc,EAAE;QACd,SAAS,EAAE,MAAM,CAAC;QAClB,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC;IAGF,qBAAqB,EAAE,MAAM,CAAC;IAC9B,sBAAsB,EAAE,MAAM,CAAC;IAG/B,aAAa,EAAE,MAAM,CAAC;IAGtB,aAAa,EAAE,OAAO,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,OAAO,CAAC;IAGvB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,cAAc,EAAE,MAAM,CAAC;IAGvB,gBAAgB,EAAE,MAAM,EAAE,CAAC;CAC5B;AAED;;GAEG;AACH,eAAO,MAAM,cAAc,EAAE,UAiB5B,CAAC;AAyJF;;;;;GAKG;AACH,wBAAsB,UAAU,CAC9B,UAAU,CAAC,EAAE,MAAM,EACnB,SAAS,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,GAC9B,OAAO,CAAC,UAAU,CAAC,CAKrB;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAAC,SAAS,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,GAAG,UAAU,CAG1E;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,CAAC,EAAE,UAAU,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAO1E"}

package/dist/src/config.js

@@ -0,0 +1,194 @@
+/**
+ * Centralized configuration for the skill evaluation framework.
+ *
+ * Configuration is loaded with the following precedence (lowest to highest):
+ * 1. Built-in defaults
+ * 2. Config file (eval.config.yaml or custom path)
+ * 3. Environment variables (EVAL_* prefix)
+ * 4. Programmatic overrides (CLI flags or API)
+ *
+ * Supports both Anthropic API and Bedrock via Agent SDK env vars:
+ * - Anthropic: Set ANTHROPIC_API_KEY
+ * - Bedrock: Set CLAUDE_CODE_USE_BEDROCK=1, AWS_REGION, AWS_PROFILE
+ */
+import yaml from 'js-yaml';
+import * as fs from 'fs/promises';
+import * as path from 'path';
+/**
+ * Default configuration values.
+ */
+export const DEFAULT_CONFIG = {
+    defaultAgentModel: 'sonnet',
+    defaultJudgeModel: 'haiku',
+    defaultWeights: {
+        discovery: 0.3,
+        adherence: 0.4,
+        output: 0.3,
+    },
+    judgeOutputTruncation: 5000,
+    reportOutputTruncation: 2000,
+    taskTimeoutMs: 300000, // 5 minutes
+    exitOnFailure: true,
+    outputDir: './results',
+    githubSummary: false,
+    discoveryThreshold: 0.8,
+    scoreThreshold: 4.0,
+    allowedWriteDirs: ['./results/', './fixtures/'],
+};
+/**
+ * Load a YAML config file if it exists.
+ */
+async function loadConfigFile(configPath) {
+    const filePath = configPath || path.join(process.cwd(), 'eval.config.yaml');
+    try {
+        const content = await fs.readFile(filePath, 'utf-8');
+        const raw = yaml.load(content);
+        if (!raw)
+            return {};
+        const config = {};
+        if (raw.models?.agent)
+            config.defaultAgentModel = raw.models.agent;
+        if (raw.models?.judge)
+            config.defaultJudgeModel = raw.models.judge;
+        if (raw.scoring?.weights) {
+            config.defaultWeights = {
+                discovery: raw.scoring.weights.discovery ?? DEFAULT_CONFIG.defaultWeights.discovery,
+                adherence: raw.scoring.weights.adherence ?? DEFAULT_CONFIG.defaultWeights.adherence,
+                output: raw.scoring.weights.output ?? DEFAULT_CONFIG.defaultWeights.output,
+            };
+        }
+        if (raw.thresholds?.discovery_rate !== undefined)
+            config.discoveryThreshold = raw.thresholds.discovery_rate;
+        if (raw.thresholds?.avg_score !== undefined)
+            config.scoreThreshold = raw.thresholds.avg_score;
+        if (raw.runner?.timeout_ms !== undefined)
+            config.taskTimeoutMs = raw.runner.timeout_ms;
+        if (raw.runner?.allowed_write_dirs)
+            config.allowedWriteDirs = raw.runner.allowed_write_dirs;
+        if (raw.output?.dir)
+            config.outputDir = raw.output.dir;
+        if (raw.output?.judge_truncation !== undefined)
+            config.judgeOutputTruncation = raw.output.judge_truncation;
+        if (raw.output?.report_truncation !== undefined)
+            config.reportOutputTruncation = raw.output.report_truncation;
+        if (raw.ci?.exit_on_failure !== undefined)
+            config.exitOnFailure = raw.ci.exit_on_failure;
+        if (raw.ci?.github_summary !== undefined)
+            config.githubSummary = raw.ci.github_summary;
+        return config;
+    }
+    catch {
+        // Config file not found or invalid — that's fine
+        return {};
+    }
+}
+/**
+ * Load configuration from environment variables.
+ *
+ * Supported variables:
+ * - EVAL_AGENT_MODEL: Model for task execution (default: 'sonnet')
+ * - EVAL_JUDGE_MODEL: Model for scoring (default: 'haiku')
+ * - EVAL_OUTPUT_TRUNCATION: Max chars to show judge (default: 5000)
+ * - EVAL_REPORT_TRUNCATION: Max chars in reports (default: 2000)
+ * - EVAL_TASK_TIMEOUT_MS: Per-task timeout in ms (default: 300000)
+ * - EVAL_EXIT_ON_FAILURE: Exit with code 1 on failures (default: true)
+ * - EVAL_OUTPUT_DIR: Directory for results (default: './results')
+ * - EVAL_DISCOVERY_THRESHOLD: Min discovery rate 0-1 (default: 0.8)
+ * - EVAL_SCORE_THRESHOLD: Min avg score 1-5 (default: 4.0)
+ * - EVAL_GITHUB_SUMMARY: Write GitHub Actions summary (default: false)
+ */
+function loadEnvConfig() {
+    const config = {};
+    if (process.env.EVAL_AGENT_MODEL)
+        config.defaultAgentModel = process.env.EVAL_AGENT_MODEL;
+    if (process.env.EVAL_JUDGE_MODEL)
+        config.defaultJudgeModel = process.env.EVAL_JUDGE_MODEL;
+    const truncation = parseInt(process.env.EVAL_OUTPUT_TRUNCATION || '', 10);
+    if (!isNaN(truncation))
+        config.judgeOutputTruncation = truncation;
+    const reportTruncation = parseInt(process.env.EVAL_REPORT_TRUNCATION || '', 10);
+    if (!isNaN(reportTruncation))
+        config.reportOutputTruncation = reportTruncation;
+    const timeout = parseInt(process.env.EVAL_TASK_TIMEOUT_MS || '', 10);
+    if (!isNaN(timeout))
+        config.taskTimeoutMs = timeout;
+    if (process.env.EVAL_EXIT_ON_FAILURE !== undefined) {
+        config.exitOnFailure = process.env.EVAL_EXIT_ON_FAILURE !== 'false';
+    }
+    if (process.env.EVAL_OUTPUT_DIR)
+        config.outputDir = process.env.EVAL_OUTPUT_DIR;
+    const discoveryThreshold = parseFloat(process.env.EVAL_DISCOVERY_THRESHOLD || '');
+    if (!isNaN(discoveryThreshold))
+        config.discoveryThreshold = discoveryThreshold;
+    const scoreThreshold = parseFloat(process.env.EVAL_SCORE_THRESHOLD || '');
+    if (!isNaN(scoreThreshold))
+        config.scoreThreshold = scoreThreshold;
+    if (process.env.EVAL_GITHUB_SUMMARY !== undefined) {
+        config.githubSummary = process.env.EVAL_GITHUB_SUMMARY === 'true';
+    }
+    return config;
+}
+/**
+ * Deep merge multiple partial configs into a full config.
+ */
+function mergeConfigs(...configs) {
+    const result = { ...DEFAULT_CONFIG };
+    for (const config of configs) {
+        if (config.defaultAgentModel !== undefined)
+            result.defaultAgentModel = config.defaultAgentModel;
+        if (config.defaultJudgeModel !== undefined)
+            result.defaultJudgeModel = config.defaultJudgeModel;
+        if (config.defaultWeights !== undefined)
+            result.defaultWeights = { ...result.defaultWeights, ...config.defaultWeights };
+        if (config.judgeOutputTruncation !== undefined)
+            result.judgeOutputTruncation = config.judgeOutputTruncation;
+        if (config.reportOutputTruncation !== undefined)
+            result.reportOutputTruncation = config.reportOutputTruncation;
+        if (config.taskTimeoutMs !== undefined)
+            result.taskTimeoutMs = config.taskTimeoutMs;
+        if (config.exitOnFailure !== undefined)
+            result.exitOnFailure = config.exitOnFailure;
+        if (config.outputDir !== undefined)
+            result.outputDir = config.outputDir;
+        if (config.githubSummary !== undefined)
+            result.githubSummary = config.githubSummary;
+        if (config.discoveryThreshold !== undefined)
+            result.discoveryThreshold = config.discoveryThreshold;
+        if (config.scoreThreshold !== undefined)
+            result.scoreThreshold = config.scoreThreshold;
+        if (config.allowedWriteDirs !== undefined)
+            result.allowedWriteDirs = config.allowedWriteDirs;
+    }
+    return result;
+}
+/**
+ * Load full configuration with all sources merged.
+ *
+ * @param configPath - Optional path to eval.config.yaml
+ * @param overrides - Optional programmatic overrides (CLI flags)
+ */
+export async function loadConfig(configPath, overrides) {
+    const fileConfig = await loadConfigFile(configPath);
+    const envConfig = loadEnvConfig();
+    return mergeConfigs(fileConfig, envConfig, overrides ?? {});
+}
+/**
+ * Load configuration synchronously (env vars + defaults only, no file).
+ * Useful when you can't await — e.g., in constructors.
+ */
+export function loadConfigSync(overrides) {
+    const envConfig = loadEnvConfig();
+    return mergeConfigs(envConfig, overrides ?? {});
+}
+/**
+ * Get default weights for scoring dimensions.
+ */
+export function getDefaultWeights(config) {
+    const c = config ?? DEFAULT_CONFIG;
+    return new Map([
+        ['discovery', c.defaultWeights.discovery],
+        ['adherence', c.defaultWeights.adherence],
+        ['output', c.defaultWeights.output],
+    ]);
+}
+//# sourceMappingURL=config.js.map

package/dist/src/config.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/config.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,IAAI,MAAM,SAAS,CAAC;AAC3B,OAAO,KAAK,EAAE,MAAM,aAAa,CAAC;AAClC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAkC7B;;GAEG;AACH,MAAM,CAAC,MAAM,cAAc,GAAe;IACxC,iBAAiB,EAAE,QAAQ;IAC3B,iBAAiB,EAAE,OAAO;IAC1B,cAAc,EAAE;QACd,SAAS,EAAE,GAAG;QACd,SAAS,EAAE,GAAG;QACd,MAAM,EAAE,GAAG;KACZ;IACD,qBAAqB,EAAE,IAAI;IAC3B,sBAAsB,EAAE,IAAI;IAC5B,aAAa,EAAE,MAAM,EAAE,YAAY;IACnC,aAAa,EAAE,IAAI;IACnB,SAAS,EAAE,WAAW;IACtB,aAAa,EAAE,KAAK;IACpB,kBAAkB,EAAE,GAAG;IACvB,cAAc,EAAE,GAAG;IACnB,gBAAgB,EAAE,CAAC,YAAY,EAAE,aAAa,CAAC;CAChD,CAAC;AAoCF;;GAEG;AACH,KAAK,UAAU,cAAc,CAAC,UAAmB;IAC/C,MAAM,QAAQ,GAAG,UAAU,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,kBAAkB,CAAC,CAAC;IAE5E,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QACrD,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAkB,CAAC;QAChD,IAAI,CAAC,GAAG;YAAE,OAAO,EAAE,CAAC;QAEpB,MAAM,MAAM,GAAwB,EAAE,CAAC;QAEvC,IAAI,GAAG,CAAC,MAAM,EAAE,KAAK;YAAE,MAAM,CAAC,iBAAiB,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC;QACnE,IAAI,GAAG,CAAC,MAAM,EAAE,KAAK;YAAE,MAAM,CAAC,iBAAiB,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC;QAEnE,IAAI,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE,CAAC;YACzB,MAAM,CAAC,cAAc,GAAG;gBACtB,SAAS,EAAE,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,IAAI,cAAc,CAAC,cAAc,CAAC,SAAS;gBACnF,SAAS,EAAE,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,IAAI,cAAc,CAAC,cAAc,CAAC,SAAS;gBACnF,MAAM,EAAE,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,IAAI,cAAc,CAAC,cAAc,CAAC,MAAM;aAC3E,CAAC;QACJ,CAAC;QAED,IAAI,GAAG,CAAC,UAAU,EAAE,cAAc,KAAK,SAAS;YAAE,MAAM,CAAC,kBAAkB,GAAG,GAAG,CAAC,UAAU,CAAC,cAAc,CAAC;QAC5G,IAAI,GAAG,CAAC,UAAU,EAAE,SAAS,KAAK,SAAS;YAAE,MAAM,CAAC,cAAc,GAAG,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC;QAE9F,IAAI,GAAG,CAAC,MAAM,EAAE,UAAU,KAAK,SAAS;YAAE,MAAM,CAAC,aAAa,GAAG,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC;QACvF,IAAI,GAAG,CAAC,MAAM,EAAE,kBAAkB;YAAE,MAAM,CAAC,gBAAgB,GAAG,GAAG,CAAC,MAAM,CAAC,kBAAkB,CAAC;QAE5F,IAAI,GAAG,CAAC,MAAM,EAAE,GAAG;YAAE,MAAM,CAAC,SAAS,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC;QACvD,IAAI,GAAG,CAAC,MAAM,EAAE,gBAAgB,KAAK,SAAS;YAAE,MAAM,CAAC,qBAAqB,GAAG,GAAG,CAAC,MAAM,CAAC,gBAAgB,CAAC;QAC3G,IAAI,GAAG,CAAC,MAAM,EAAE,iBAAiB,KAAK,SAAS;YAAE,MAAM,CAAC,sBAAsB,GAAG,GAAG,CAAC,MAAM,CAAC,iBAAiB,CAAC;QAE9G,IAAI,GAAG,CAAC,EAAE,EAAE,eAAe,KAAK,SAAS;YAAE,MAAM,CAAC,aAAa,GAAG,GAAG,CAAC,EAAE,CAAC,eAAe,CAAC;QACzF,IAAI,GAAG,CAAC,EAAE,EAAE,cAAc,KAAK,SAAS;YAAE,MAAM,CAAC,aAAa,GAAG,GAAG,CAAC,EAAE,CAAC,cAAc,CAAC;QAEvF,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,MAAM,CAAC;QACP,iDAAiD;QACjD,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,SAAS,aAAa;IACpB,MAAM,MAAM,GAAwB,EAAE,CAAC;IAEvC,IAAI,OAAO,CAAC,GAAG,CAAC,gBAAgB;QAAE,MAAM,CAAC,iBAAiB,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC;IAC1F,IAAI,OAAO,CAAC,GAAG,CAAC,gBAAgB;QAAE,MAAM,CAAC,iBAAiB,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC;IAE1F,MAAM,UAAU,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,sBAAsB,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;IAC1E,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC;QAAE,MAAM,CAAC,qBAAqB,GAAG,UAAU,CAAC;IAElE,MAAM,gBAAgB,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,sBAAsB,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;IAChF,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC;QAAE,MAAM,CAAC,sBAAsB,GAAG,gBAAgB,CAAC;IAE/E,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;IACrE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC;QAAE,MAAM,CAAC,aAAa,GAAG,OAAO,CAAC;IAEpD,IAAI,OAAO,CAAC,GAAG,CAAC,oBAAoB,KAAK,SAAS,EAAE,CAAC;QACnD,MAAM,CAAC,aAAa,GAAG,OAAO,CAAC,GAAG,CAAC,oBAAoB,KAAK,OAAO,CAAC;IACtE,CAAC;IAED,IAAI,OAAO,CAAC,GAAG,CAAC,eAAe;QAAE,MAAM,CAAC,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC;IAEhF,MAAM,kBAAkB,GAAG,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,IAAI,EAAE,CAAC,CAAC;IAClF,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC;QAAE,MAAM,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;IAE/E,MAAM,cAAc,GAAG,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,EAAE,CAAC,CAAC;IAC1E,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC;QAAE,MAAM,CAAC,cAAc,GAAG,cAAc,CAAC;IAEnE,IAAI,OAAO,CAAC,GAAG,CAAC,mBAAmB,KAAK,SAAS,EAAE,CAAC;QAClD,MAAM,CAAC,aAAa,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,KAAK,MAAM,CAAC;IACpE,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,SAAS,YAAY,CAAC,GAAG,OAA8B;IACrD,MAAM,MAAM,GAAG,EAAE,GAAG,cAAc,EAAE,CAAC;IAErC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,MAAM,CAAC,iBAAiB,KAAK,SAAS;YAAE,MAAM,CAAC,iBAAiB,GAAG,MAAM,CAAC,iBAAiB,CAAC;QAChG,IAAI,MAAM,CAAC,iBAAiB,KAAK,SAAS;YAAE,MAAM,CAAC,iBAAiB,GAAG,MAAM,CAAC,iBAAiB,CAAC;QAChG,IAAI,MAAM,CAAC,cAAc,KAAK,SAAS;YAAE,MAAM,CAAC,cAAc,GAAG,EAAE,GAAG,MAAM,CAAC,cAAc,EAAE,GAAG,MAAM,CAAC,cAAc,EAAE,CAAC;QACxH,IAAI,MAAM,CAAC,qBAAqB,KAAK,SAAS;YAAE,MAAM,CAAC,qBAAqB,GAAG,MAAM,CAAC,qBAAqB,CAAC;QAC5G,IAAI,MAAM,CAAC,sBAAsB,KAAK,SAAS;YAAE,MAAM,CAAC,sBAAsB,GAAG,MAAM,CAAC,sBAAsB,CAAC;QAC/G,IAAI,MAAM,CAAC,aAAa,KAAK,SAAS;YAAE,MAAM,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,CAAC;QACpF,IAAI,MAAM,CAAC,aAAa,KAAK,SAAS;YAAE,MAAM,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,CAAC;QACpF,IAAI,MAAM,CAAC,SAAS,KAAK,SAAS;YAAE,MAAM,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;QACxE,IAAI,MAAM,CAAC,aAAa,KAAK,SAAS;YAAE,MAAM,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,CAAC;QACpF,IAAI,MAAM,CAAC,kBAAkB,KAAK,SAAS;YAAE,MAAM,CAAC,kBAAkB,GAAG,MAAM,CAAC,kBAAkB,CAAC;QACnG,IAAI,MAAM,CAAC,cAAc,KAAK,SAAS;YAAE,MAAM,CAAC,cAAc,GAAG,MAAM,CAAC,cAAc,CAAC;QACvF,IAAI,MAAM,CAAC,gBAAgB,KAAK,SAAS;YAAE,MAAM,CAAC,gBAAgB,GAAG,MAAM,CAAC,gBAAgB,CAAC;IAC/F,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,UAAmB,EACnB,SAA+B;IAE/B,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,UAAU,CAAC,CAAC;IACpD,MAAM,SAAS,GAAG,aAAa,EAAE,CAAC;IAElC,OAAO,YAAY,CAAC,UAAU,EAAE,SAAS,EAAE,SAAS,IAAI,EAAE,CAAC,CAAC;AAC9D,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,SAA+B;IAC5D,MAAM,SAAS,GAAG,aAAa,EAAE,CAAC;IAClC,OAAO,YAAY,CAAC,SAAS,EAAE,SAAS,IAAI,EAAE,CAAC,CAAC;AAClD,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,MAAmB;IACnD,MAAM,CAAC,GAAG,MAAM,IAAI,cAAc,CAAC;IACnC,OAAO,IAAI,GAAG,CAAC;QACb,CAAC,WAAW,EAAE,CAAC,CAAC,cAAc,CAAC,SAAS,CAAC;QACzC,CAAC,WAAW,EAAE,CAAC,CAAC,cAAc,CAAC,SAAS,CAAC;QACzC,CAAC,QAAQ,EAAE,CAAC,CAAC,cAAc,CAAC,MAAM,CAAC;KACpC,CAAC,CAAC;AACL,CAAC"}

package/dist/src/index.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Skill evaluation framework.
+ *
+ * Provides tools for evaluating AI agent skill discoverability,
+ * adherence, and output quality.
+ *
+ * @packageDocumentation
+ */
+export type { EvalCriteria, EvalTask, EvalDefaults, SkillEvaluation, DeterministicCheck, DeterministicResult, FixtureConfig, ToolCallRecord, TaskResult, RunnerOptions, FailureCategory, JudgeScore, JudgeOptions, CombinedScore, SessionLogEntry, MetricsData, SessionLog, EvaluationSummary, FailureBreakdown, ReportMetadata, EvaluationReport, } from './types.js';
+export { loadConfig, loadConfigSync, getDefaultWeights, DEFAULT_CONFIG } from './config.js';
+export type { EvalConfig } from './config.js';
+export { parseEvalFile, createEvalTemplate, validateEvalFile } from './parser.js';
+export { SkillEvalRunner } from './runner/runner.js';
+export { setupLocalSkills, cleanupLocalSkills } from './runner/skill-setup.js';
+export { createToolPolicy } from './runner/security.js';
+export { scoreTask, scoreAll } from './scorer/scorer.js';
+export { scoreDeterministic } from './scorer/deterministic.js';
+export { SkillJudge } from './scorer/judge.js';
+export { SessionLogger } from './session/session-logger.js';
+export { generateReport, generateJsonResults, computeSummary, computeFailureBreakdown } from './report/report.js';
+export { generateGitHubSummary, writeGitHubSummary } from './report/github-summary.js';
+export { runPipeline, scorePipeline } from './pipeline.js';
+export type { PipelineOptions, PipelineResult } from './pipeline.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/src/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,YAAY,EACV,YAAY,EACZ,QAAQ,EACR,YAAY,EACZ,eAAe,EACf,kBAAkB,EAClB,mBAAmB,EACnB,aAAa,EACb,cAAc,EACd,UAAU,EACV,aAAa,EACb,eAAe,EACf,UAAU,EACV,YAAY,EACZ,aAAa,EACb,eAAe,EACf,WAAW,EACX,UAAU,EACV,iBAAiB,EACjB,gBAAgB,EAChB,cAAc,EACd,gBAAgB,GACjB,MAAM,YAAY,CAAC;AAGpB,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC5F,YAAY,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAG9C,OAAO,EAAE,aAAa,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAGlF,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAC/E,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAGxD,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAC/D,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAG/C,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAG5D,OAAO,EAAE,cAAc,EAAE,mBAAmB,EAAE,cAAc,EAAE,uBAAuB,EAAE,MAAM,oBAAoB,CAAC;AAClH,OAAO,EAAE,qBAAqB,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAGvF,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC3D,YAAY,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC"}

package/dist/src/index.js

@@ -0,0 +1,28 @@
+/**
+ * Skill evaluation framework.
+ *
+ * Provides tools for evaluating AI agent skill discoverability,
+ * adherence, and output quality.
+ *
+ * @packageDocumentation
+ */
+// Config
+export { loadConfig, loadConfigSync, getDefaultWeights, DEFAULT_CONFIG } from './config.js';
+// Parser
+export { parseEvalFile, createEvalTemplate, validateEvalFile } from './parser.js';
+// Runner
+export { SkillEvalRunner } from './runner/runner.js';
+export { setupLocalSkills, cleanupLocalSkills } from './runner/skill-setup.js';
+export { createToolPolicy } from './runner/security.js';
+// Scorer
+export { scoreTask, scoreAll } from './scorer/scorer.js';
+export { scoreDeterministic } from './scorer/deterministic.js';
+export { SkillJudge } from './scorer/judge.js';
+// Session
+export { SessionLogger } from './session/session-logger.js';
+// Report
+export { generateReport, generateJsonResults, computeSummary, computeFailureBreakdown } from './report/report.js';
+export { generateGitHubSummary, writeGitHubSummary } from './report/github-summary.js';
+// Pipeline
+export { runPipeline, scorePipeline } from './pipeline.js';
+//# sourceMappingURL=index.js.map

package/dist/src/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AA2BH,SAAS;AACT,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAG5F,SAAS;AACT,OAAO,EAAE,aAAa,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAElF,SAAS;AACT,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAC/E,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAExD,SAAS;AACT,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAC/D,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAE/C,UAAU;AACV,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAE5D,SAAS;AACT,OAAO,EAAE,cAAc,EAAE,mBAAmB,EAAE,cAAc,EAAE,uBAAuB,EAAE,MAAM,oBAAoB,CAAC;AAClH,OAAO,EAAE,qBAAqB,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAEvF,WAAW;AACX,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC"}

package/dist/src/parser.d.ts

@@ -0,0 +1,22 @@
+/**
+ * YAML parser for skill evaluation task files.
+ *
+ * Supports enriched YAML schema with:
+ * - defaults block (shared criteria, expected_skill_load)
+ * - deterministic block (marker-based checks, tool call expectations)
+ * - fixture block (setup/teardown scripts per task)
+ */
+import type { SkillEvaluation } from './types.js';
+/**
+ * Parse a YAML evaluation file into a SkillEvaluation object.
+ */
+export declare function parseEvalFile(filePath: string): Promise<SkillEvaluation>;
+/**
+ * Validate a YAML evaluation file and return any errors.
+ */
+export declare function validateEvalFile(filePath: string): Promise<string[]>;
+/**
+ * Generate a YAML template for a new skill evaluation.
+ */
+export declare function createEvalTemplate(skillName: string, numTasks?: number): string;
+//# sourceMappingURL=parser.d.ts.map

package/dist/src/parser.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parser.d.ts","sourceRoot":"","sources":["../../src/parser.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAIH,OAAO,KAAK,EAGV,eAAe,EAIhB,MAAM,YAAY,CAAC;AA0DpB;;GAEG;AACH,wBAAsB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC,CAiB9E;AA0FD;;GAEG;AACH,wBAAsB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CA4D1E;AAMD;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,SAAI,GAAG,MAAM,CAkC1E"}