agent-skill-manager 1.20.0 → 1.22.0

package/node_modules/skillgrade/dist/reporters/browser.js

@@ -0,0 +1,90 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runBrowserPreview = runBrowserPreview;
+const http = __importStar(require("http"));
+const fs = __importStar(require("fs-extra"));
+const path = __importStar(require("path"));
+async function runBrowserPreview(resultsDir, port = 3847) {
+    const resolved = path.resolve(resultsDir);
+    const htmlPath = path.join(__dirname, '..', 'viewer.html');
+    const server = http.createServer(async (req, res) => {
+        const url = new URL(req.url || '/', `http://localhost:${port}`);
+        if (url.pathname === '/api/reports') {
+            const files = (await fs.readdir(resolved)).filter(f => f.endsWith('.json')).reverse();
+            const reports = [];
+            for (const file of files) {
+                try {
+                    const report = await fs.readJSON(path.join(resolved, file));
+                    reports.push({ file, ...report });
+                }
+                catch { /* skip malformed */ }
+            }
+            res.writeHead(200, { 'Content-Type': 'application/json' });
+            res.end(JSON.stringify(reports));
+        }
+        else if (url.pathname === '/api/report') {
+            const file = url.searchParams.get('file');
+            if (!file) {
+                res.writeHead(400);
+                res.end('Missing file param');
+                return;
+            }
+            const filePath = path.join(resolved, file);
+            if (await fs.pathExists(filePath)) {
+                const report = await fs.readJSON(filePath);
+                res.writeHead(200, { 'Content-Type': 'application/json' });
+                res.end(JSON.stringify(report));
+            }
+            else {
+                res.writeHead(404);
+                res.end('Not found');
+            }
+        }
+        else {
+            const html = await fs.readFile(htmlPath, 'utf-8');
+            res.writeHead(200, { 'Content-Type': 'text/html' });
+            res.end(html);
+        }
+    });
+    server.listen(port, () => {
+        const addr = server.address();
+        const actualPort = typeof addr === 'object' ? addr.port : port;
+        console.log(`\nskillgrade preview`);
+        console.log(`\n  url       http://localhost:${actualPort}`);
+        console.log(`  results   ${resolved}\n`);
+    });
+}
+//# sourceMappingURL=browser.js.map

package/node_modules/skillgrade/dist/reporters/cli.d.ts

@@ -0,0 +1,2 @@
+export declare function runCliPreview(resultsDir: string): Promise<void>;
+//# sourceMappingURL=cli.d.ts.map

package/node_modules/skillgrade/dist/reporters/cli.js

@@ -0,0 +1,116 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runCliPreview = runCliPreview;
+const fs = __importStar(require("fs-extra"));
+const path = __importStar(require("path"));
+const cli_1 = require("../utils/cli");
+// ─── Main ──────────────────────────────────────────────────
+async function runCliPreview(resultsDir) {
+    const resolved = path.resolve(resultsDir);
+    const files = (await fs.readdir(resolved))
+        .filter(f => f.endsWith('.json'))
+        .reverse();
+    if (!files.length) {
+        console.log(`\n  ${cli_1.fmt.dim('No reports found in')} ${resolved}\n`);
+        return;
+    }
+    console.log(`\n${cli_1.fmt.bold('skillgrade preview')}  ${cli_1.fmt.dim(`${files.length} reports from ${resolved}`)}\n`);
+    for (const file of files) {
+        let report;
+        try {
+            report = await fs.readJSON(path.join(resolved, file));
+        }
+        catch {
+            continue;
+        }
+        const passRate = report.pass_rate ?? 0;
+        const isPass = passRate >= 0.5;
+        const trials = report.trials || [];
+        const avgDur = trials.reduce((s, t) => s + (t.duration_ms || 0), 0) / (trials.length || 1);
+        const totalTokens = trials.reduce((s, t) => s + (t.input_tokens || 0) + (t.output_tokens || 0), 0);
+        // ── Report header
+        const status = isPass ? cli_1.fmt.pass('PASS') : cli_1.fmt.fail('FAIL');
+        (0, cli_1.header)(`${status}  ${report.task}`);
+        // Timestamp from filename
+        const ts = file.match(/\d{4}-\d{2}-\d{2}T[\d-]+/)?.[0]?.replace(/-(\d{2})-(\d{2})-/g, ':$1:$2:') || '';
+        if (ts)
+            console.log(`    ${cli_1.fmt.dim(ts)}`);
+        console.log();
+        // ── Summary metrics
+        const metrics = [
+            ['Pass Rate', `${(passRate * 100).toFixed(1)}%`],
+            ['pass@k', report.pass_at_k != null ? `${(report.pass_at_k * 100).toFixed(1)}%` : '—'],
+            ['pass^k', report.pass_pow_k != null ? `${(report.pass_pow_k * 100).toFixed(1)}%` : '—'],
+            ['Avg Duration', `${(avgDur / 1000).toFixed(1)}s`],
+            ['Total Tokens', `~${totalTokens}`],
+            ['Skills', report.skills_used?.join(', ') || 'none'],
+        ];
+        for (const [label, value] of metrics) {
+            console.log(`    ${cli_1.fmt.dim(label.padEnd(14))} ${cli_1.fmt.bold(value)}`);
+        }
+        console.log();
+        // ── Trials
+        for (const trial of trials) {
+            const tp = trial.reward >= 0.5;
+            const trialStatus = tp ? cli_1.fmt.pass('PASS') : cli_1.fmt.fail('FAIL');
+            const reward = cli_1.fmt.bold(trial.reward.toFixed(2));
+            const dur = `${((trial.duration_ms || 0) / 1000).toFixed(1)}s`;
+            const cmds = `${trial.n_commands || 0} cmds`;
+            const graders = (trial.grader_results || []).map((g) => {
+                const scoreStr = g.score.toFixed(1);
+                const colored = g.score >= 0.5 ? cli_1.fmt.green(scoreStr) : cli_1.fmt.red(scoreStr);
+                return `${cli_1.fmt.dim(g.grader_type)} ${colored}`;
+            }).join('  ');
+            console.log(`    ${cli_1.fmt.dim(`${trial.trial_id}`.padEnd(4))} ${trialStatus}  ${reward}  ${cli_1.fmt.dim(dur.padEnd(7))} ${cli_1.fmt.dim(cmds.padEnd(7))} ${graders}`);
+        }
+        console.log();
+        // ── LLM grader details
+        const hasLlm = trials.some((t) => t.grader_results?.some((g) => g.grader_type === 'llm_rubric'));
+        if (hasLlm) {
+            for (const trial of trials) {
+                const llmGraders = (trial.grader_results || []).filter((g) => g.grader_type === 'llm_rubric');
+                for (const g of llmGraders) {
+                    const scoreStr = g.score >= 0.5 ? cli_1.fmt.green(g.score.toFixed(2)) : cli_1.fmt.red(g.score.toFixed(2));
+                    console.log(`    ${cli_1.fmt.dim(`trial ${trial.trial_id}`)} ${scoreStr} ${cli_1.fmt.dim(g.details.substring(0, 100))}`);
+                }
+            }
+            console.log();
+        }
+        console.log(`    ${cli_1.fmt.dim(file)}`);
+        console.log();
+    }
+}
+//# sourceMappingURL=cli.js.map

package/node_modules/skillgrade/dist/skillgrade.d.ts

@@ -0,0 +1,20 @@
+#!/usr/bin/env node
+/**
+ * skillgrade CLI
+ *
+ * Usage:
+ *   skillgrade                     Run all eval tasks from eval.yaml
+ *   skillgrade init                Generate eval.yaml from detected skills
+ *   skillgrade preview [browser]   View results (CLI default, or browser)
+ *   skillgrade <task-name>         Run a specific eval
+ *
+ * Options:
+ *   --trials=N         Override trial count
+ *   --parallel=N       Run trials concurrently
+ *   --validate         Run reference solutions to verify graders
+ *   --ci               CI mode: exit non-zero if below threshold
+ *   --threshold=0.8    Pass rate threshold for --ci
+ *   --preview          Open results after running
+ */
+export {};
+//# sourceMappingURL=skillgrade.d.ts.map

package/node_modules/skillgrade/dist/skillgrade.js

@@ -0,0 +1,179 @@
+#!/usr/bin/env node
+"use strict";
+/**
+ * skillgrade CLI
+ *
+ * Usage:
+ *   skillgrade                     Run all eval tasks from eval.yaml
+ *   skillgrade init                Generate eval.yaml from detected skills
+ *   skillgrade preview [browser]   View results (CLI default, or browser)
+ *   skillgrade <task-name>         Run a specific eval
+ *
+ * Options:
+ *   --trials=N         Override trial count
+ *   --parallel=N       Run trials concurrently
+ *   --validate         Run reference solutions to verify graders
+ *   --ci               CI mode: exit non-zero if below threshold
+ *   --threshold=0.8    Pass rate threshold for --ci
+ *   --preview          Open results after running
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+const init_1 = require("./commands/init");
+const run_1 = require("./commands/run");
+const preview_1 = require("./commands/preview");
+const cli_1 = require("./utils/cli");
+const os = __importStar(require("os"));
+const path = __importStar(require("path"));
+async function main() {
+    const args = process.argv.slice(2);
+    const command = args[0];
+    const cwd = process.cwd();
+    // Parse global flags
+    const getFlag = (name) => args.find(a => a.startsWith(`--${name}=`))?.split('=')[1];
+    const hasFlag = (name) => args.includes(`--${name}`);
+    if (command === '--help' || command === '-h') {
+        printHelp();
+        return;
+    }
+    if (command === '--version' || command === '-v') {
+        const pkg = require('../package.json');
+        console.log(pkg.version);
+        return;
+    }
+    if (command === 'init') {
+        await (0, init_1.runInit)(cwd, { force: hasFlag('force') });
+        return;
+    }
+    if (command === 'preview') {
+        const mode = args[1] === 'browser' ? 'browser' : 'cli';
+        const outputDir = getFlag('output') || path.join(os.tmpdir(), 'skillgrade');
+        await (0, preview_1.runPreview)(cwd, mode, outputDir);
+        return;
+    }
+    // Default: run evals
+    const taskName = command && !command.startsWith('-') ? command : undefined;
+    const openPreview = hasFlag('preview');
+    // Preset modes (can be overridden by --trials)
+    let preset;
+    let presetTrials;
+    if (hasFlag('smoke')) {
+        preset = 'smoke';
+        presetTrials = 5;
+    }
+    else if (hasFlag('reliable')) {
+        preset = 'reliable';
+        presetTrials = 15;
+    }
+    else if (hasFlag('regression')) {
+        preset = 'regression';
+        presetTrials = 30;
+    }
+    const explicitTrials = getFlag('trials') ? parseInt(getFlag('trials')) : undefined;
+    // Resolve eval filter: --eval flag, deprecated --task flag, or positional arg
+    let evalFilter;
+    if (getFlag('eval')) {
+        evalFilter = getFlag('eval');
+    }
+    else if (getFlag('task')) {
+        console.log(`  ${cli_1.fmt.dim('note:')} --task is deprecated, use --eval instead\n`);
+        evalFilter = getFlag('task');
+    }
+    else if (taskName) {
+        evalFilter = taskName;
+    }
+    const outputDir = getFlag('output') || path.join(os.tmpdir(), 'skillgrade');
+    await (0, run_1.runEvals)(cwd, {
+        eval: evalFilter,
+        trials: explicitTrials ?? presetTrials,
+        parallel: getFlag('parallel') ? parseInt(getFlag('parallel')) : undefined,
+        validate: hasFlag('validate'),
+        ci: hasFlag('ci'),
+        threshold: getFlag('threshold') ? parseFloat(getFlag('threshold')) : undefined,
+        preset,
+        agent: getFlag('agent'),
+        provider: getFlag('provider'),
+        grader: getFlag('grader'),
+        output: outputDir,
+    });
+    if (openPreview) {
+        await (0, preview_1.runPreview)(cwd, 'cli', outputDir);
+    }
+}
+function printHelp() {
+    console.log(`
+  skillgrade - The easiest way to evaluate your Agent Skills
+
+  Usage:
+    skillgrade                     Run all evals from eval.yaml
+    skillgrade init [--force]      Generate eval.yaml (--force to overwrite)
+    skillgrade preview [browser]   View results (CLI default, or browser)
+    skillgrade <eval-name>         Run a specific eval
+
+  Presets:
+    --smoke            Quick smoke test (5 trials, reports pass@k)
+    --reliable         Reliable pass rate (15 trials, reports mean reward)
+    --regression       High-confidence regression (30 trials, reports pass^k)
+
+  Options:
+    --eval=NAME[,NAME] Run specific evals by name (comma-separated)
+    --grader=TYPE      Run only graders of this type (deterministic|llm_rubric)
+    --trials=N         Override trial count (overrides preset)
+    --parallel=N       Run trials concurrently
+    --agent=gemini|claude|codex   Override agent (default: auto-detect from API key)
+    --provider=docker|local Override provider (default: docker)
+    --output=DIR       Output directory for reports and temp files
+                       Default: $TMPDIR/skillgrade
+    --validate         Verify graders using reference solutions
+    --ci               CI mode: exit non-zero if below threshold
+    --threshold=0.8    Pass rate threshold for CI mode
+    --preview          Open CLI results after running
+
+  Examples:
+    skillgrade init                # scaffold eval.yaml
+    skillgrade init --force        # overwrite existing eval.yaml
+    skillgrade                     # run all evals
+    skillgrade --smoke             # quick 5-trial smoke test
+    skillgrade --eval=fix-linting  # run a specific eval
+    skillgrade --eval=foo,bar      # run multiple evals
+    skillgrade --regression --ci   # CI regression with 30 trials
+    skillgrade preview browser     # open web UI
+`);
+}
+main().catch(err => {
+    console.error(err);
+    process.exit(1);
+});
+//# sourceMappingURL=skillgrade.js.map

package/node_modules/skillgrade/dist/types.d.ts

@@ -0,0 +1,72 @@
+export interface CommandResult {
+    stdout: string;
+    stderr: string;
+    exitCode: number;
+}
+export interface GraderConfig {
+    type: 'deterministic' | 'llm_rubric';
+    command?: string;
+    rubric?: string;
+    model?: string;
+    weight: number;
+}
+export interface GraderResult {
+    grader_type: string;
+    score: number;
+    weight: number;
+    details: string;
+}
+export interface LogEntry {
+    type: 'agent_start' | 'command' | 'agent_result' | 'grader' | 'reward';
+    timestamp: string;
+    instruction?: string;
+    command?: string;
+    stdout?: string;
+    stderr?: string;
+    exitCode?: number;
+    output?: string;
+    value?: number;
+    grader_result?: GraderResult;
+}
+export interface TrialResult {
+    trial_id: number;
+    reward: number;
+    grader_results: GraderResult[];
+    duration_ms: number;
+    n_commands: number;
+    input_tokens: number;
+    output_tokens: number;
+    session_log: LogEntry[];
+}
+export interface EvalReport {
+    task: string;
+    pass_rate: number;
+    pass_at_k: number;
+    pass_pow_k: number;
+    trials: TrialResult[];
+    skills_used: string[];
+}
+export declare abstract class BaseAgent {
+    abstract run(instruction: string, workspacePath: string, runCommand: (cmd: string) => Promise<CommandResult>): Promise<string>;
+}
+/** Options passed to environment providers for setup */
+export interface EnvironmentSetupOpts {
+    timeoutSec: number;
+    environment: {
+        cpus: number;
+        memory_mb: number;
+    };
+}
+export interface EnvironmentProvider {
+    /** One-time setup: build image, inject skills. Returns reusable handle. */
+    prepare?(taskPath: string, skillsPaths: string[], opts: EnvironmentSetupOpts, env?: Record<string, string>): Promise<string>;
+    /** Per-trial setup: create isolated workspace. */
+    setup(taskPath: string, skillsPaths: string[], opts: EnvironmentSetupOpts, env?: Record<string, string>): Promise<string>;
+    /** Per-trial cleanup. */
+    cleanup(workspacePath: string): Promise<void>;
+    /** One-time teardown. */
+    teardown?(): Promise<void>;
+    runCommand(workspacePath: string, command: string, env?: Record<string, string>): Promise<CommandResult>;
+    diagnose?(workspacePath: string): Promise<string>;
+}
+//# sourceMappingURL=types.d.ts.map

package/node_modules/skillgrade/dist/types.js

@@ -0,0 +1,7 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BaseAgent = void 0;
+class BaseAgent {
+}
+exports.BaseAgent = BaseAgent;
+//# sourceMappingURL=types.js.map

package/node_modules/skillgrade/dist/utils/cli.d.ts

@@ -0,0 +1,56 @@
+/**
+ * CLI formatting utilities.
+ *
+ * Uses ANSI codes that work on both light and dark terminals.
+ * Respects NO_COLOR env var (https://no-color.org/).
+ */
+export declare const fmt: {
+    bold: (s: string) => string;
+    dim: (s: string) => string;
+    green: (s: string) => string;
+    red: (s: string) => string;
+    cyan: (s: string) => string;
+    pass: (s: string) => string;
+    fail: (s: string) => string;
+};
+/** Print a section header with a rule line */
+export declare function header(title: string, width?: number): void;
+/** Print a labeled key-value pair */
+export declare function kv(label: string, value: string, indent?: number): void;
+/** Print a trial result row */
+export declare function trialRow(trialId: number, total: number, reward: number, duration: string, commands: number, graders: {
+    type: string;
+    score: number;
+}[]): void;
+/** Print the results summary block */
+export declare function resultsSummary(passRate: number, passAtK: number, passPowK: number, trials: number, preset?: string): void;
+/** Print a validation result */
+export declare function validationResult(passed: boolean, reward: number, graders: {
+    type: string;
+    score: number;
+    details: string;
+}[]): void;
+/**
+ * In-place spinner that shows current phase + elapsed time on one line.
+ *
+ * Usage:
+ *   const s = new Spinner('1/3', 'building image');
+ *   s.update('running agent');
+ *   s.update('grading');
+ *   s.stop('PASS  0.85  120.3s  2 cmds');
+ */
+export declare class Spinner {
+    private frames;
+    private frameIdx;
+    private interval;
+    private prefix;
+    private phase;
+    private startTime;
+    constructor(label: string, initialPhase: string);
+    private render;
+    /** Update the spinner's phase text */
+    update(phase: string): void;
+    /** Stop the spinner and print the final line (replaces the spinner) */
+    stop(finalLine: string): void;
+}
+//# sourceMappingURL=cli.d.ts.map