@icntswm/skillcheck 0.1.0

package/dist/budget.js

@@ -0,0 +1,29 @@
+/**
+ * Running spend estimate for --budget. Runs killed by early-stop never report
+ * a cost, so unknown costs are estimated at the average known cost (0 until
+ * the first known cost arrives); without this the budget could never trigger.
+ */
+export class Budget {
+    limitUsd;
+    knownSum = 0;
+    knownCount = 0;
+    unknownCount = 0;
+    constructor(limitUsd) {
+        this.limitUsd = limitUsd;
+    }
+    add(cost) {
+        if (cost === null)
+            this.unknownCount++;
+        else {
+            this.knownSum += cost;
+            this.knownCount++;
+        }
+    }
+    get spent() {
+        const avg = this.knownCount > 0 ? this.knownSum / this.knownCount : 0;
+        return this.knownSum + this.unknownCount * avg;
+    }
+    get exceeded() {
+        return this.spent >= this.limitUsd;
+    }
+}

package/dist/cases.js

@@ -0,0 +1,249 @@
+import { readFile } from "node:fs/promises";
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { parse as parseYaml } from "yaml";
+import { commandNames, skillDirNames, skillRoots } from "./describe.js";
+export class ConfigError extends Error {
+    errors;
+    constructor(errors) {
+        super(errors.join("; "));
+        this.errors = errors;
+        this.name = "ConfigError";
+    }
+}
+const CASE_KEYS = [
+    "query", "expect", "expect_any", "forbid", "first",
+    "none", "note", "repeat", "threshold", "agents", "id",
+];
+const SUITE_KEYS = ["agent", "model", "repeat", "threshold", "cases"];
+const DEFAULTS = { agent: "claude", repeat: 1, threshold: 1.0 };
+export const BUILTIN_SKILLS = [
+    "code-review", "simplify", "security-review", "init", "loop", "schedule",
+    "update-config", "keybindings-help", "fewer-permission-prompts",
+    "claude-api", "run",
+    // seen in the init event of a clean config dir, verified by hand
+    "deep-research", "design", "design-sync", "dataviz", "verify", "debug",
+    "batch", "doctor", "workflow-authoring", "run-skill-generator",
+];
+function isObject(v) {
+    return typeof v === "object" && v !== null && !Array.isArray(v);
+}
+function isNonEmptyString(v) {
+    return typeof v === "string" && v.length > 0;
+}
+function isStringArray(v) {
+    return Array.isArray(v) && v.every(isNonEmptyString);
+}
+/** Collect all problems, index-prefixed per case; throws one ConfigError at the end. */
+export function parseSuite(data) {
+    const errors = [];
+    const legacy = Array.isArray(data);
+    let rawCases = [];
+    const suite = { agent: DEFAULTS.agent, repeat: DEFAULTS.repeat, threshold: DEFAULTS.threshold, cases: [] };
+    if (legacy) {
+        rawCases = data;
+        if (rawCases.length === 0)
+            errors.push("no cases");
+    }
+    else if (isObject(data)) {
+        for (const key of Object.keys(data)) {
+            if (!SUITE_KEYS.includes(key))
+                errors.push(`unknown key "${key}"`);
+        }
+        if (typeof data.agent !== "undefined") {
+            if (typeof data.agent === "string")
+                suite.agent = data.agent;
+            else
+                errors.push("agent must be a string");
+        }
+        if (typeof data.model !== "undefined") {
+            if (typeof data.model === "string")
+                suite.model = data.model;
+            else
+                errors.push("model must be a string");
+        }
+        if (typeof data.repeat !== "undefined") {
+            if (isPositiveInt(data.repeat))
+                suite.repeat = data.repeat;
+            else
+                errors.push("repeat must be an integer >= 1");
+        }
+        if (typeof data.threshold !== "undefined") {
+            if (isThreshold(data.threshold))
+                suite.threshold = data.threshold;
+            else
+                errors.push("threshold must be a number with 0 < x <= 1");
+        }
+        if (typeof data.cases === "undefined")
+            errors.push("cases is required");
+        else if (!Array.isArray(data.cases) || data.cases.length === 0)
+            errors.push("cases must be a non-empty array");
+        else
+            rawCases = data.cases;
+    }
+    else {
+        throw new ConfigError(["cases file must be an array of cases or an object with a cases key"]);
+    }
+    const seenIds = new Set();
+    rawCases.forEach((raw, i) => {
+        const idx = i + 1;
+        const c = validateCase(raw, idx, errors);
+        if (!c)
+            return;
+        if (typeof c.id === "string") {
+            if (seenIds.has(c.id))
+                errors.push(`#${idx}: duplicate id "${c.id}"`);
+            seenIds.add(c.id);
+        }
+        suite.cases.push(c);
+    });
+    if (errors.length > 0)
+        throw new ConfigError(errors);
+    return suite;
+}
+function isPositiveInt(v) {
+    return typeof v === "number" && Number.isInteger(v) && v >= 1;
+}
+function isThreshold(v) {
+    return typeof v === "number" && v > 0 && v <= 1;
+}
+function validateCase(raw, idx, errors) {
+    if (!isObject(raw)) {
+        errors.push(`#${idx}: case must be an object`);
+        return null;
+    }
+    for (const key of Object.keys(raw)) {
+        if (!CASE_KEYS.includes(key))
+            errors.push(`#${idx}: unknown key "${key}"`);
+    }
+    const c = {
+        index: idx, query: "", expect: [], expect_any: [], forbid: [], none: false,
+    };
+    let bad = false;
+    const fail = (msg) => { errors.push(`#${idx}: ${msg}`); bad = true; };
+    if (typeof raw.query === "undefined")
+        fail("query is required");
+    else if (!isNonEmptyString(raw.query))
+        fail("query must be a non-empty string");
+    else
+        c.query = raw.query;
+    for (const key of ["expect", "expect_any", "forbid", "agents"]) {
+        const v = raw[key];
+        if (typeof v === "undefined")
+            continue;
+        if (!isStringArray(v)) {
+            fail(`${key} must be an array of non-empty strings`);
+            continue;
+        }
+        c[key] = [...v];
+    }
+    for (const key of ["first", "note", "id"]) {
+        const v = raw[key];
+        if (typeof v === "undefined")
+            continue;
+        if (typeof v !== "string" || (key === "first" && v.length === 0)) {
+            fail(`${key} must be a string`);
+            continue;
+        }
+        c[key] = v;
+    }
+    if (typeof raw.none !== "undefined") {
+        if (typeof raw.none !== "boolean")
+            fail("none must be a boolean");
+        else
+            c.none = raw.none;
+    }
+    if (typeof raw.repeat !== "undefined") {
+        if (isPositiveInt(raw.repeat))
+            c.repeat = raw.repeat;
+        else
+            fail("repeat must be an integer >= 1");
+    }
+    if (typeof raw.threshold !== "undefined") {
+        if (isThreshold(raw.threshold))
+            c.threshold = raw.threshold;
+        else
+            fail("threshold must be a number with 0 < x <= 1");
+    }
+    const hasExpect = c.expect.length > 0;
+    const hasAny = c.expect_any.length > 0;
+    const hasForbid = c.forbid.length > 0;
+    if (!hasExpect && !hasAny && !hasForbid && !c.first && !c.none) {
+        fail("at least one of expect, expect_any, forbid, first, none is required");
+    }
+    if (c.none && (hasExpect || hasAny || c.first)) {
+        fail('none: true contradicts expect/expect_any/first');
+    }
+    const wanted = new Set([...c.expect, ...c.expect_any, ...(c.first ? [c.first] : [])]);
+    for (const name of c.forbid) {
+        if (wanted.has(name))
+            fail(`"${name}" is both expected and forbidden`);
+    }
+    return bad ? null : c;
+}
+/** Read a cases file (.json / .yaml / .yml) and parse it. */
+export async function loadSuite(filePath) {
+    let text;
+    try {
+        text = await readFile(filePath, "utf8");
+    }
+    catch (e) {
+        throw new ConfigError([`cannot read ${filePath}: ${e.message}`]);
+    }
+    const ext = path.extname(filePath).toLowerCase();
+    let data;
+    try {
+        if (ext === ".json")
+            data = JSON.parse(text);
+        else if (ext === ".yaml" || ext === ".yml")
+            data = parseYaml(text);
+        else
+            throw new ConfigError([`unsupported file format "${ext || filePath}": use .json, .yaml or .yml`]);
+    }
+    catch (e) {
+        if (e instanceof ConfigError)
+            throw e;
+        throw new ConfigError([`cannot parse ${filePath}: ${e.message}`]);
+    }
+    return parseSuite(data);
+}
+export function findDefaultCasesFile(cwd) {
+    for (const name of ["skillcheck.yaml", "skillcheck.yml", "skillcheck.json"]) {
+        const p = path.join(cwd, name);
+        try {
+            if (fs.statSync(p).isFile())
+                return p;
+        }
+        catch {
+            // missing, keep looking
+        }
+    }
+    return undefined;
+}
+/** Names of skills/commands discoverable on disk, plus built-in slash skills. */
+export function knownSkillNames(opts) {
+    const out = new Set(BUILTIN_SKILLS);
+    for (const root of skillRoots(opts)) {
+        for (const name of skillDirNames(path.join(root, "skills")))
+            out.add(name);
+        for (const name of commandNames(path.join(root, "commands")))
+            out.add(name);
+    }
+    return out;
+}
+const NAME_FIELDS = ["expect", "expect_any", "forbid", "first"];
+export function unknownNames(suite, known) {
+    const errors = [];
+    for (const c of suite.cases) {
+        for (const field of NAME_FIELDS) {
+            const names = field === "first" ? (c.first ? [c.first] : []) : c[field];
+            for (const name of names) {
+                if (name.includes(":"))
+                    continue; // plugin skills are not checked
+                if (!known.has(name))
+                    errors.push(`#${c.index}: unknown skill "${name}" in ${field}`);
+            }
+        }
+    }
+    return errors;
+}