agent-skill-manager 1.22.0 → 2.1.0

package/node_modules/skillgrade/dist/reporters/browser.js

@@ -1,90 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
-    };
-})();
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.runBrowserPreview = runBrowserPreview;
-const http = __importStar(require("http"));
-const fs = __importStar(require("fs-extra"));
-const path = __importStar(require("path"));
-async function runBrowserPreview(resultsDir, port = 3847) {
-    const resolved = path.resolve(resultsDir);
-    const htmlPath = path.join(__dirname, '..', 'viewer.html');
-    const server = http.createServer(async (req, res) => {
-        const url = new URL(req.url || '/', `http://localhost:${port}`);
-        if (url.pathname === '/api/reports') {
-            const files = (await fs.readdir(resolved)).filter(f => f.endsWith('.json')).reverse();
-            const reports = [];
-            for (const file of files) {
-                try {
-                    const report = await fs.readJSON(path.join(resolved, file));
-                    reports.push({ file, ...report });
-                }
-                catch { /* skip malformed */ }
-            }
-            res.writeHead(200, { 'Content-Type': 'application/json' });
-            res.end(JSON.stringify(reports));
-        }
-        else if (url.pathname === '/api/report') {
-            const file = url.searchParams.get('file');
-            if (!file) {
-                res.writeHead(400);
-                res.end('Missing file param');
-                return;
-            }
-            const filePath = path.join(resolved, file);
-            if (await fs.pathExists(filePath)) {
-                const report = await fs.readJSON(filePath);
-                res.writeHead(200, { 'Content-Type': 'application/json' });
-                res.end(JSON.stringify(report));
-            }
-            else {
-                res.writeHead(404);
-                res.end('Not found');
-            }
-        }
-        else {
-            const html = await fs.readFile(htmlPath, 'utf-8');
-            res.writeHead(200, { 'Content-Type': 'text/html' });
-            res.end(html);
-        }
-    });
-    server.listen(port, () => {
-        const addr = server.address();
-        const actualPort = typeof addr === 'object' ? addr.port : port;
-        console.log(`\nskillgrade preview`);
-        console.log(`\n  url       http://localhost:${actualPort}`);
-        console.log(`  results   ${resolved}\n`);
-    });
-}
-//# sourceMappingURL=browser.js.map

package/node_modules/skillgrade/dist/reporters/cli.d.ts

@@ -1,2 +0,0 @@
-export declare function runCliPreview(resultsDir: string): Promise<void>;
-//# sourceMappingURL=cli.d.ts.map

package/node_modules/skillgrade/dist/reporters/cli.js

@@ -1,116 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
-    };
-})();
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.runCliPreview = runCliPreview;
-const fs = __importStar(require("fs-extra"));
-const path = __importStar(require("path"));
-const cli_1 = require("../utils/cli");
-// ─── Main ──────────────────────────────────────────────────
-async function runCliPreview(resultsDir) {
-    const resolved = path.resolve(resultsDir);
-    const files = (await fs.readdir(resolved))
-        .filter(f => f.endsWith('.json'))
-        .reverse();
-    if (!files.length) {
-        console.log(`\n  ${cli_1.fmt.dim('No reports found in')} ${resolved}\n`);
-        return;
-    }
-    console.log(`\n${cli_1.fmt.bold('skillgrade preview')}  ${cli_1.fmt.dim(`${files.length} reports from ${resolved}`)}\n`);
-    for (const file of files) {
-        let report;
-        try {
-            report = await fs.readJSON(path.join(resolved, file));
-        }
-        catch {
-            continue;
-        }
-        const passRate = report.pass_rate ?? 0;
-        const isPass = passRate >= 0.5;
-        const trials = report.trials || [];
-        const avgDur = trials.reduce((s, t) => s + (t.duration_ms || 0), 0) / (trials.length || 1);
-        const totalTokens = trials.reduce((s, t) => s + (t.input_tokens || 0) + (t.output_tokens || 0), 0);
-        // ── Report header
-        const status = isPass ? cli_1.fmt.pass('PASS') : cli_1.fmt.fail('FAIL');
-        (0, cli_1.header)(`${status}  ${report.task}`);
-        // Timestamp from filename
-        const ts = file.match(/\d{4}-\d{2}-\d{2}T[\d-]+/)?.[0]?.replace(/-(\d{2})-(\d{2})-/g, ':$1:$2:') || '';
-        if (ts)
-            console.log(`    ${cli_1.fmt.dim(ts)}`);
-        console.log();
-        // ── Summary metrics
-        const metrics = [
-            ['Pass Rate', `${(passRate * 100).toFixed(1)}%`],
-            ['pass@k', report.pass_at_k != null ? `${(report.pass_at_k * 100).toFixed(1)}%` : '—'],
-            ['pass^k', report.pass_pow_k != null ? `${(report.pass_pow_k * 100).toFixed(1)}%` : '—'],
-            ['Avg Duration', `${(avgDur / 1000).toFixed(1)}s`],
-            ['Total Tokens', `~${totalTokens}`],
-            ['Skills', report.skills_used?.join(', ') || 'none'],
-        ];
-        for (const [label, value] of metrics) {
-            console.log(`    ${cli_1.fmt.dim(label.padEnd(14))} ${cli_1.fmt.bold(value)}`);
-        }
-        console.log();
-        // ── Trials
-        for (const trial of trials) {
-            const tp = trial.reward >= 0.5;
-            const trialStatus = tp ? cli_1.fmt.pass('PASS') : cli_1.fmt.fail('FAIL');
-            const reward = cli_1.fmt.bold(trial.reward.toFixed(2));
-            const dur = `${((trial.duration_ms || 0) / 1000).toFixed(1)}s`;
-            const cmds = `${trial.n_commands || 0} cmds`;
-            const graders = (trial.grader_results || []).map((g) => {
-                const scoreStr = g.score.toFixed(1);
-                const colored = g.score >= 0.5 ? cli_1.fmt.green(scoreStr) : cli_1.fmt.red(scoreStr);
-                return `${cli_1.fmt.dim(g.grader_type)} ${colored}`;
-            }).join('  ');
-            console.log(`    ${cli_1.fmt.dim(`${trial.trial_id}`.padEnd(4))} ${trialStatus}  ${reward}  ${cli_1.fmt.dim(dur.padEnd(7))} ${cli_1.fmt.dim(cmds.padEnd(7))} ${graders}`);
-        }
-        console.log();
-        // ── LLM grader details
-        const hasLlm = trials.some((t) => t.grader_results?.some((g) => g.grader_type === 'llm_rubric'));
-        if (hasLlm) {
-            for (const trial of trials) {
-                const llmGraders = (trial.grader_results || []).filter((g) => g.grader_type === 'llm_rubric');
-                for (const g of llmGraders) {
-                    const scoreStr = g.score >= 0.5 ? cli_1.fmt.green(g.score.toFixed(2)) : cli_1.fmt.red(g.score.toFixed(2));
-                    console.log(`    ${cli_1.fmt.dim(`trial ${trial.trial_id}`)} ${scoreStr} ${cli_1.fmt.dim(g.details.substring(0, 100))}`);
-                }
-            }
-            console.log();
-        }
-        console.log(`    ${cli_1.fmt.dim(file)}`);
-        console.log();
-    }
-}
-//# sourceMappingURL=cli.js.map

package/node_modules/skillgrade/dist/skillgrade.d.ts

@@ -1,20 +0,0 @@
-#!/usr/bin/env node
-/**
- * skillgrade CLI
- *
- * Usage:
- *   skillgrade                     Run all eval tasks from eval.yaml
- *   skillgrade init                Generate eval.yaml from detected skills
- *   skillgrade preview [browser]   View results (CLI default, or browser)
- *   skillgrade <task-name>         Run a specific eval
- *
- * Options:
- *   --trials=N         Override trial count
- *   --parallel=N       Run trials concurrently
- *   --validate         Run reference solutions to verify graders
- *   --ci               CI mode: exit non-zero if below threshold
- *   --threshold=0.8    Pass rate threshold for --ci
- *   --preview          Open results after running
- */
-export {};
-//# sourceMappingURL=skillgrade.d.ts.map

package/node_modules/skillgrade/dist/skillgrade.js

@@ -1,179 +0,0 @@
-#!/usr/bin/env node
-"use strict";
-/**
- * skillgrade CLI
- *
- * Usage:
- *   skillgrade                     Run all eval tasks from eval.yaml
- *   skillgrade init                Generate eval.yaml from detected skills
- *   skillgrade preview [browser]   View results (CLI default, or browser)
- *   skillgrade <task-name>         Run a specific eval
- *
- * Options:
- *   --trials=N         Override trial count
- *   --parallel=N       Run trials concurrently
- *   --validate         Run reference solutions to verify graders
- *   --ci               CI mode: exit non-zero if below threshold
- *   --threshold=0.8    Pass rate threshold for --ci
- *   --preview          Open results after running
- */
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
-    };
-})();
-Object.defineProperty(exports, "__esModule", { value: true });
-const init_1 = require("./commands/init");
-const run_1 = require("./commands/run");
-const preview_1 = require("./commands/preview");
-const cli_1 = require("./utils/cli");
-const os = __importStar(require("os"));
-const path = __importStar(require("path"));
-async function main() {
-    const args = process.argv.slice(2);
-    const command = args[0];
-    const cwd = process.cwd();
-    // Parse global flags
-    const getFlag = (name) => args.find(a => a.startsWith(`--${name}=`))?.split('=')[1];
-    const hasFlag = (name) => args.includes(`--${name}`);
-    if (command === '--help' || command === '-h') {
-        printHelp();
-        return;
-    }
-    if (command === '--version' || command === '-v') {
-        const pkg = require('../package.json');
-        console.log(pkg.version);
-        return;
-    }
-    if (command === 'init') {
-        await (0, init_1.runInit)(cwd, { force: hasFlag('force') });
-        return;
-    }
-    if (command === 'preview') {
-        const mode = args[1] === 'browser' ? 'browser' : 'cli';
-        const outputDir = getFlag('output') || path.join(os.tmpdir(), 'skillgrade');
-        await (0, preview_1.runPreview)(cwd, mode, outputDir);
-        return;
-    }
-    // Default: run evals
-    const taskName = command && !command.startsWith('-') ? command : undefined;
-    const openPreview = hasFlag('preview');
-    // Preset modes (can be overridden by --trials)
-    let preset;
-    let presetTrials;
-    if (hasFlag('smoke')) {
-        preset = 'smoke';
-        presetTrials = 5;
-    }
-    else if (hasFlag('reliable')) {
-        preset = 'reliable';
-        presetTrials = 15;
-    }
-    else if (hasFlag('regression')) {
-        preset = 'regression';
-        presetTrials = 30;
-    }
-    const explicitTrials = getFlag('trials') ? parseInt(getFlag('trials')) : undefined;
-    // Resolve eval filter: --eval flag, deprecated --task flag, or positional arg
-    let evalFilter;
-    if (getFlag('eval')) {
-        evalFilter = getFlag('eval');
-    }
-    else if (getFlag('task')) {
-        console.log(`  ${cli_1.fmt.dim('note:')} --task is deprecated, use --eval instead\n`);
-        evalFilter = getFlag('task');
-    }
-    else if (taskName) {
-        evalFilter = taskName;
-    }
-    const outputDir = getFlag('output') || path.join(os.tmpdir(), 'skillgrade');
-    await (0, run_1.runEvals)(cwd, {
-        eval: evalFilter,
-        trials: explicitTrials ?? presetTrials,
-        parallel: getFlag('parallel') ? parseInt(getFlag('parallel')) : undefined,
-        validate: hasFlag('validate'),
-        ci: hasFlag('ci'),
-        threshold: getFlag('threshold') ? parseFloat(getFlag('threshold')) : undefined,
-        preset,
-        agent: getFlag('agent'),
-        provider: getFlag('provider'),
-        grader: getFlag('grader'),
-        output: outputDir,
-    });
-    if (openPreview) {
-        await (0, preview_1.runPreview)(cwd, 'cli', outputDir);
-    }
-}
-function printHelp() {
-    console.log(`
-  skillgrade - The easiest way to evaluate your Agent Skills
-
-  Usage:
-    skillgrade                     Run all evals from eval.yaml
-    skillgrade init [--force]      Generate eval.yaml (--force to overwrite)
-    skillgrade preview [browser]   View results (CLI default, or browser)
-    skillgrade <eval-name>         Run a specific eval
-
-  Presets:
-    --smoke            Quick smoke test (5 trials, reports pass@k)
-    --reliable         Reliable pass rate (15 trials, reports mean reward)
-    --regression       High-confidence regression (30 trials, reports pass^k)
-
-  Options:
-    --eval=NAME[,NAME] Run specific evals by name (comma-separated)
-    --grader=TYPE      Run only graders of this type (deterministic|llm_rubric)
-    --trials=N         Override trial count (overrides preset)
-    --parallel=N       Run trials concurrently
-    --agent=gemini|claude|codex   Override agent (default: auto-detect from API key)
-    --provider=docker|local Override provider (default: docker)
-    --output=DIR       Output directory for reports and temp files
-                       Default: $TMPDIR/skillgrade
-    --validate         Verify graders using reference solutions
-    --ci               CI mode: exit non-zero if below threshold
-    --threshold=0.8    Pass rate threshold for CI mode
-    --preview          Open CLI results after running
-
-  Examples:
-    skillgrade init                # scaffold eval.yaml
-    skillgrade init --force        # overwrite existing eval.yaml
-    skillgrade                     # run all evals
-    skillgrade --smoke             # quick 5-trial smoke test
-    skillgrade --eval=fix-linting  # run a specific eval
-    skillgrade --eval=foo,bar      # run multiple evals
-    skillgrade --regression --ci   # CI regression with 30 trials
-    skillgrade preview browser     # open web UI
-`);
-}
-main().catch(err => {
-    console.error(err);
-    process.exit(1);
-});
-//# sourceMappingURL=skillgrade.js.map

package/node_modules/skillgrade/dist/types.d.ts

@@ -1,72 +0,0 @@
-export interface CommandResult {
-    stdout: string;
-    stderr: string;
-    exitCode: number;
-}
-export interface GraderConfig {
-    type: 'deterministic' | 'llm_rubric';
-    command?: string;
-    rubric?: string;
-    model?: string;
-    weight: number;
-}
-export interface GraderResult {
-    grader_type: string;
-    score: number;
-    weight: number;
-    details: string;
-}
-export interface LogEntry {
-    type: 'agent_start' | 'command' | 'agent_result' | 'grader' | 'reward';
-    timestamp: string;
-    instruction?: string;
-    command?: string;
-    stdout?: string;
-    stderr?: string;
-    exitCode?: number;
-    output?: string;
-    value?: number;
-    grader_result?: GraderResult;
-}
-export interface TrialResult {
-    trial_id: number;
-    reward: number;
-    grader_results: GraderResult[];
-    duration_ms: number;
-    n_commands: number;
-    input_tokens: number;
-    output_tokens: number;
-    session_log: LogEntry[];
-}
-export interface EvalReport {
-    task: string;
-    pass_rate: number;
-    pass_at_k: number;
-    pass_pow_k: number;
-    trials: TrialResult[];
-    skills_used: string[];
-}
-export declare abstract class BaseAgent {
-    abstract run(instruction: string, workspacePath: string, runCommand: (cmd: string) => Promise<CommandResult>): Promise<string>;
-}
-/** Options passed to environment providers for setup */
-export interface EnvironmentSetupOpts {
-    timeoutSec: number;
-    environment: {
-        cpus: number;
-        memory_mb: number;
-    };
-}
-export interface EnvironmentProvider {
-    /** One-time setup: build image, inject skills. Returns reusable handle. */
-    prepare?(taskPath: string, skillsPaths: string[], opts: EnvironmentSetupOpts, env?: Record<string, string>): Promise<string>;
-    /** Per-trial setup: create isolated workspace. */
-    setup(taskPath: string, skillsPaths: string[], opts: EnvironmentSetupOpts, env?: Record<string, string>): Promise<string>;
-    /** Per-trial cleanup. */
-    cleanup(workspacePath: string): Promise<void>;
-    /** One-time teardown. */
-    teardown?(): Promise<void>;
-    runCommand(workspacePath: string, command: string, env?: Record<string, string>): Promise<CommandResult>;
-    diagnose?(workspacePath: string): Promise<string>;
-}
-//# sourceMappingURL=types.d.ts.map

package/node_modules/skillgrade/dist/types.js

@@ -1,7 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.BaseAgent = void 0;
-class BaseAgent {
-}
-exports.BaseAgent = BaseAgent;
-//# sourceMappingURL=types.js.map

package/node_modules/skillgrade/dist/utils/cli.d.ts

@@ -1,56 +0,0 @@
-/**
- * CLI formatting utilities.
- *
- * Uses ANSI codes that work on both light and dark terminals.
- * Respects NO_COLOR env var (https://no-color.org/).
- */
-export declare const fmt: {
-    bold: (s: string) => string;
-    dim: (s: string) => string;
-    green: (s: string) => string;
-    red: (s: string) => string;
-    cyan: (s: string) => string;
-    pass: (s: string) => string;
-    fail: (s: string) => string;
-};
-/** Print a section header with a rule line */
-export declare function header(title: string, width?: number): void;
-/** Print a labeled key-value pair */
-export declare function kv(label: string, value: string, indent?: number): void;
-/** Print a trial result row */
-export declare function trialRow(trialId: number, total: number, reward: number, duration: string, commands: number, graders: {
-    type: string;
-    score: number;
-}[]): void;
-/** Print the results summary block */
-export declare function resultsSummary(passRate: number, passAtK: number, passPowK: number, trials: number, preset?: string): void;
-/** Print a validation result */
-export declare function validationResult(passed: boolean, reward: number, graders: {
-    type: string;
-    score: number;
-    details: string;
-}[]): void;
-/**
- * In-place spinner that shows current phase + elapsed time on one line.
- *
- * Usage:
- *   const s = new Spinner('1/3', 'building image');
- *   s.update('running agent');
- *   s.update('grading');
- *   s.stop('PASS  0.85  120.3s  2 cmds');
- */
-export declare class Spinner {
-    private frames;
-    private frameIdx;
-    private interval;
-    private prefix;
-    private phase;
-    private startTime;
-    constructor(label: string, initialPhase: string);
-    private render;
-    /** Update the spinner's phase text */
-    update(phase: string): void;
-    /** Stop the spinner and print the final line (replaces the spinner) */
-    stop(finalLine: string): void;
-}
-//# sourceMappingURL=cli.d.ts.map