evalify-cli 0.1.0

package/dist/commands/publish.d.ts

@@ -0,0 +1 @@
+export declare function publish(targetPath?: string): Promise<void>;

package/dist/commands/publish.js

@@ -0,0 +1,86 @@
+import path from "node:path";
+import fs from "node:fs/promises";
+import chalk from "chalk";
+import { getFramework } from "@evalify/frameworks";
+import { header, success, info, dim, error, warn } from "../format.js";
+import { validateEvalsJson } from "../validator.js";
+async function findEvalsFile(targetPath) {
+    const stat = await fs.stat(targetPath);
+    if (stat.isFile())
+        return targetPath;
+    const candidates = [
+        path.join(targetPath, "evals", "evals.json"),
+        path.join(targetPath, "evals.json"),
+    ];
+    for (const candidate of candidates) {
+        try {
+            await fs.access(candidate);
+            return candidate;
+        }
+        catch {
+            // not found, continue
+        }
+    }
+    return null;
+}
+export async function publish(targetPath) {
+    header();
+    const resolvedPath = path.resolve(process.cwd(), targetPath || ".");
+    try {
+        await fs.access(resolvedPath);
+    }
+    catch {
+        error(`Path not found: ${targetPath || "."}`);
+        console.log();
+        return;
+    }
+    info("Publishing eval criteria to registry...");
+    console.log();
+    const filePath = await findEvalsFile(resolvedPath);
+    if (!filePath) {
+        error("No evals.json found");
+        dim("Looked in:");
+        dim(`  ${resolvedPath}/evals.json`);
+        dim(`  ${resolvedPath}/evals/evals.json`);
+        console.log();
+        return;
+    }
+    success(`Found ${path.relative(process.cwd(), filePath)}`);
+    const content = await fs.readFile(filePath, "utf-8");
+    const result = validateEvalsJson(content);
+    if (!result.valid) {
+        console.log();
+        error("Validation failed — cannot publish");
+        for (const e of result.errors) {
+            error(e);
+        }
+        console.log();
+        return;
+    }
+    for (const w of result.warnings) {
+        warn(w);
+    }
+    console.log();
+    console.log(chalk.bold("  Publish summary:"));
+    console.log();
+    if (result.summary["skill_name"]) {
+        dim(`Skill:       ${result.summary["skill_name"]}`);
+    }
+    if (result.summary["name"]) {
+        dim(`Name:        ${result.summary["name"]}`);
+    }
+    if (result.summary["version"]) {
+        dim(`Version:     ${result.summary["version"]}`);
+    }
+    if (result.summary["description"]) {
+        dim(`Description: ${result.summary["description"]}`);
+    }
+    dim(`Format:      ${getFramework(result.format)?.meta.name ?? result.format}`);
+    dim(`Eval count:  ${result.evalCount}`);
+    dim(`File:        ${path.relative(process.cwd(), filePath)}`);
+    console.log();
+    warn("Dry run — publishing is not yet connected to the registry");
+    success("File is valid and ready to publish");
+    console.log();
+}
+//# sourceMappingURL=publish.js.map

package/dist/commands/publish.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"publish.js","sourceRoot":"","sources":["../../src/commands/publish.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACvE,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAEpD,KAAK,UAAU,aAAa,CAAC,UAAkB;IAC7C,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAEvC,IAAI,IAAI,CAAC,MAAM,EAAE;QAAE,OAAO,UAAU,CAAC;IAErC,MAAM,UAAU,GAAG;QACjB,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,OAAO,EAAE,YAAY,CAAC;QAC5C,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,YAAY,CAAC;KACpC,CAAC;IAEF,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,IAAI,CAAC;YACH,MAAM,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YAC3B,OAAO,SAAS,CAAC;QACnB,CAAC;QAAC,MAAM,CAAC;YACP,sBAAsB;QACxB,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,UAAmB;IAC/C,MAAM,EAAE,CAAC;IAET,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,UAAU,IAAI,GAAG,CAAC,CAAC;IAEpE,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;IAChC,CAAC;IAAC,MAAM,CAAC;QACP,KAAK,CAAC,mBAAmB,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;QAC9C,OAAO,CAAC,GAAG,EAAE,CAAC;QACd,OAAO;IACT,CAAC;IAED,IAAI,CAAC,yCAAyC,CAAC,CAAC;IAChD,OAAO,CAAC,GAAG,EAAE,CAAC;IAEd,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,YAAY,CAAC,CAAC;IAEnD,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,KAAK,CAAC,qBAAqB,CAAC,CAAC;QAC7B,GAAG,CAAC,YAAY,CAAC,CAAC;QAClB,GAAG,CAAC,KAAK,YAAY,aAAa,CAAC,CAAC;QACpC,GAAG,CAAC,KAAK,YAAY,mBAAmB,CAAC,CAAC;QAC1C,OAAO,CAAC,GAAG,EAAE,CAAC;QACd,OAAO;IACT,CAAC;IAED,OAAO,CAAC,SAAS,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC;IAE3D,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACrD,MAAM,MAAM,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;IAE1C,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QAClB,OAAO,CAAC,GAAG,EAAE,CAAC;QACd,KAAK,CAAC,oCAAoC,CAAC,CAAC;QAC5C,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YAC9B,KAAK,CAAC,CAAC,CAAC,CAAC;QACX,CAAC;QACD,OAAO,CAAC,GAAG,EAAE,CAAC;QACd,OAAO;IACT,CAAC;IAED,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QAChC,IAAI,CAAC,CAAC,CAAC,CAAC;IACV,CAAC;IAED,OAAO,CAAC,GAAG,EAAE,CAAC;IACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC;IAC9C,OAAO,CAAC,GAAG,EAAE,CAAC;IAEd,IAAI,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,CAAC;QACjC,GAAG,CAAC,gBAAgB,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;IACtD,CAAC;IACD,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAC3B,GAAG,CAAC,gBAAgB,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAChD,CAAC;IACD,IAAI,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;QAC9B,GAAG,CAAC,gBAAgB,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;IACnD,CAAC;IACD,IAAI,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,CAAC;QAClC,GAAG,CAAC,gBAAgB,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;IACvD,CAAC;IACD,GAAG,CAAC,gBAAgB,YAAY,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,IAAI,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IAC/E,GAAG,CAAC,gBAAgB,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;IACxC,GAAG,CAAC,gBAAgB,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC;IAE9D,OAAO,CAAC,GAAG,EAAE,CAAC;IACd,IAAI,CAAC,2DAA2D,CAAC,CAAC;IAClE,OAAO,CAAC,oCAAoC,CAAC,CAAC;IAC9C,OAAO,CAAC,GAAG,EAAE,CAAC;AAChB,CAAC"}

package/dist/commands/pull.d.ts

@@ -0,0 +1 @@
+export declare function pull(slug: string): Promise<void>;

package/dist/commands/pull.js

@@ -0,0 +1,63 @@
+import path from "node:path";
+import fs from "node:fs/promises";
+import { header, success, info, dim, error } from "../format.js";
+const REGISTRY_URL = "https://evalify.sh/api/registry";
+export async function pull(slug) {
+    header();
+    if (!slug) {
+        error("Missing slug argument");
+        console.log();
+        return;
+    }
+    info(`Pulling eval criteria: ${slug}`);
+    console.log();
+    let pack;
+    try {
+        const res = await fetch(`${REGISTRY_URL}/${slug}`);
+        if (res.status === 404) {
+            error(`Criteria not found: ${slug}`);
+            dim(`Check the registry at https://evalify.sh`);
+            console.log();
+            return;
+        }
+        if (!res.ok) {
+            error(`Registry returned ${res.status}`);
+            console.log();
+            return;
+        }
+        pack = await res.json();
+    }
+    catch (err) {
+        error(`Failed to reach registry: ${err.message}`);
+        console.log();
+        return;
+    }
+    const targetDir = path.resolve(process.cwd(), "evals", slug);
+    const targetFile = path.join(targetDir, "evals.json");
+    try {
+        await fs.mkdir(targetDir, { recursive: true });
+        const output = {
+            slug: pack.slug,
+            displayName: pack.displayName,
+            version: pack.version,
+            description: pack.description,
+            domain: pack.domain,
+            author: pack.author,
+            tags: pack.tags,
+            evals: pack.evals,
+        };
+        await fs.writeFile(targetFile, JSON.stringify(output, null, 2) + "\n");
+        success(`Pulled ${pack.displayName} v${pack.version}`);
+        success(`Wrote ${pack.evals.length} eval${pack.evals.length !== 1 ? "s" : ""} to evals/${slug}/evals.json`);
+        console.log();
+        dim(`Author:   ${pack.author}`);
+        dim(`Domain:   ${pack.domain}`);
+        dim(`Location: ${targetFile}`);
+        dim(`To validate: evalify validate evals/${slug}`);
+    }
+    catch (err) {
+        error(`Failed to write file: ${err.message}`);
+    }
+    console.log();
+}
+//# sourceMappingURL=pull.js.map

package/dist/commands/pull.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pull.js","sourceRoot":"","sources":["../../src/commands/pull.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,cAAc,CAAC;AAEjE,MAAM,YAAY,GAAG,iCAAiC,CAAC;AAEvD,MAAM,CAAC,KAAK,UAAU,IAAI,CAAC,IAAY;IACrC,MAAM,EAAE,CAAC;IAET,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,KAAK,CAAC,uBAAuB,CAAC,CAAC;QAC/B,OAAO,CAAC,GAAG,EAAE,CAAC;QACd,OAAO;IACT,CAAC;IAED,IAAI,CAAC,0BAA0B,IAAI,EAAE,CAAC,CAAC;IACvC,OAAO,CAAC,GAAG,EAAE,CAAC;IAEd,IAAI,IASH,CAAC;IAEF,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,YAAY,IAAI,IAAI,EAAE,CAAC,CAAC;QACnD,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YACvB,KAAK,CAAC,uBAAuB,IAAI,EAAE,CAAC,CAAC;YACrC,GAAG,CAAC,0CAA0C,CAAC,CAAC;YAChD,OAAO,CAAC,GAAG,EAAE,CAAC;YACd,OAAO;QACT,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,KAAK,CAAC,qBAAqB,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;YACzC,OAAO,CAAC,GAAG,EAAE,CAAC;YACd,OAAO;QACT,CAAC;QACD,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;IAC1B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,KAAK,CAAC,6BAA8B,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;QAC7D,OAAO,CAAC,GAAG,EAAE,CAAC;QACd,OAAO;IACT,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;IAC7D,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;IAEtD,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAE/C,MAAM,MAAM,GAAG;YACb,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB,CAAC;QAEF,MAAM,EAAE,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;QAEvE,OAAO,CAAC,UAAU,IAAI,CAAC,WAAW,KAAK,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QACvD,OAAO,CAAC,SAAS,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,aAAa,IAAI,aAAa,CAAC,CAAC;QAC5G,OAAO,CAAC,GAAG,EAAE,CAAC;QACd,GAAG,CAAC,aAAa,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAChC,GAAG,CAAC,aAAa,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAChC,GAAG,CAAC,aAAa,UAAU,EAAE,CAAC,CAAC;QAC/B,GAAG,CAAC,uCAAuC,IAAI,EAAE,CAAC,CAAC;IACrD,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,KAAK,CAAC,yBAA0B,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;IAC3D,CAAC;IAED,OAAO,CAAC,GAAG,EAAE,CAAC;AAChB,CAAC"}

package/dist/commands/search.d.ts

@@ -0,0 +1 @@
+export declare function search(query: string): Promise<void>;

package/dist/commands/search.js

@@ -0,0 +1,21 @@
+import chalk from "chalk";
+import { header, info, dim, table } from "../format.js";
+export async function search(query) {
+    header();
+    info(`Searching registry for: ${chalk.bold(query)}`);
+    console.log();
+    // Placeholder results for now
+    const results = [
+        [chalk.bold("Slug"), chalk.bold("Description"), chalk.bold("Evals")],
+        [chalk.dim("─".repeat(20)), chalk.dim("─".repeat(35)), chalk.dim("─".repeat(5))],
+        ["code-review", "Code review quality criteria", "12"],
+        ["summarization", "Text summarization accuracy", "8"],
+        ["safety-checks", "Safety and content policy evals", "24"],
+    ];
+    table(results);
+    console.log();
+    dim(`Showing placeholder results — registry search not yet connected`);
+    dim(`Use: evalify pull <slug> to download criteria`);
+    console.log();
+}
+//# sourceMappingURL=search.js.map

package/dist/commands/search.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"search.js","sourceRoot":"","sources":["../../src/commands/search.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,cAAc,CAAC;AAExD,MAAM,CAAC,KAAK,UAAU,MAAM,CAAC,KAAa;IACxC,MAAM,EAAE,CAAC;IACT,IAAI,CAAC,2BAA2B,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACrD,OAAO,CAAC,GAAG,EAAE,CAAC;IAEd,8BAA8B;IAC9B,MAAM,OAAO,GAAG;QACd,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,EAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACpE,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAChF,CAAC,aAAa,EAAE,8BAA8B,EAAE,IAAI,CAAC;QACrD,CAAC,eAAe,EAAE,6BAA6B,EAAE,GAAG,CAAC;QACrD,CAAC,eAAe,EAAE,iCAAiC,EAAE,IAAI,CAAC;KAC3D,CAAC;IAEF,KAAK,CAAC,OAAO,CAAC,CAAC;IACf,OAAO,CAAC,GAAG,EAAE,CAAC;IACd,GAAG,CAAC,iEAAiE,CAAC,CAAC;IACvE,GAAG,CAAC,+CAA+C,CAAC,CAAC;IACrD,OAAO,CAAC,GAAG,EAAE,CAAC;AAChB,CAAC"}

package/dist/commands/validate.d.ts

@@ -0,0 +1 @@
+export declare function validate(targetPath?: string): Promise<void>;

package/dist/commands/validate.js

@@ -0,0 +1,179 @@
+import path from "node:path";
+import fs from "node:fs/promises";
+import chalk from "chalk";
+import { getFramework } from "@evalify/frameworks";
+import { header, success, info, dim, error, warn } from "../format.js";
+import { validateEvalsJson } from "../validator.js";
+const ALLOWED_EXTENSIONS = new Set([
+    ".json", ".md", ".txt", ".csv", ".yaml", ".yml",
+    ".ts", ".tsx", ".js", ".jsx", ".py", ".sql",
+    ".html", ".css", ".xml", ".toml",
+]);
+async function findEvalsJson(targetPath) {
+    const stat = await fs.stat(targetPath);
+    if (stat.isFile()) {
+        return { evalsPath: targetPath, rootDir: path.dirname(targetPath) };
+    }
+    // It's a directory — scan for evals.json
+    const candidates = [
+        path.join(targetPath, "evals", "evals.json"),
+        path.join(targetPath, "evals.json"),
+    ];
+    for (const candidate of candidates) {
+        try {
+            await fs.access(candidate);
+            return { evalsPath: candidate, rootDir: targetPath };
+        }
+        catch {
+            // not found
+        }
+    }
+    return null;
+}
+async function scanCompanionFiles(rootDir, evalsPath, maxDepth = 2) {
+    const companions = [];
+    const evalsRelative = path.relative(rootDir, evalsPath);
+    async function walk(dir, depth) {
+        if (depth > maxDepth)
+            return;
+        const entries = await fs.readdir(dir, { withFileTypes: true });
+        for (const entry of entries) {
+            if (entry.name.startsWith("."))
+                continue;
+            const fullPath = path.join(dir, entry.name);
+            const relativePath = path.relative(rootDir, fullPath);
+            if (entry.isDirectory()) {
+                await walk(fullPath, depth + 1);
+            }
+            else if (entry.isFile()) {
+                if (relativePath === evalsRelative)
+                    continue;
+                const ext = path.extname(entry.name).toLowerCase();
+                if (ALLOWED_EXTENSIONS.has(ext)) {
+                    companions.push(relativePath);
+                }
+            }
+        }
+    }
+    await walk(rootDir, 0);
+    return companions;
+}
+function extractFileRefs(content) {
+    const parsed = JSON.parse(content);
+    const evals = Array.isArray(parsed) ? parsed : parsed.evals || [];
+    const refs = new Set();
+    for (const item of evals) {
+        if (Array.isArray(item.files)) {
+            for (const f of item.files) {
+                if (typeof f === "string" && f.length > 0)
+                    refs.add(f);
+            }
+        }
+    }
+    return [...refs];
+}
+export async function validate(targetPath) {
+    header();
+    const resolvedPath = path.resolve(process.cwd(), targetPath || ".");
+    // Check if path exists
+    try {
+        await fs.access(resolvedPath);
+    }
+    catch {
+        error(`Path not found: ${targetPath || "."}`);
+        console.log();
+        return;
+    }
+    const stat = await fs.stat(resolvedPath);
+    const isFolder = stat.isDirectory();
+    if (isFolder) {
+        info(`Scanning folder: ${path.basename(resolvedPath)}/`);
+    }
+    else {
+        info(`Validating: ${path.basename(resolvedPath)}`);
+    }
+    const found = await findEvalsJson(resolvedPath);
+    if (!found) {
+        console.log();
+        error("No evals.json found");
+        dim("Looked in:");
+        dim(`  ${resolvedPath}/evals.json`);
+        dim(`  ${resolvedPath}/evals/evals.json`);
+        console.log();
+        return;
+    }
+    const { evalsPath, rootDir } = found;
+    const relEvalsPath = path.relative(process.cwd(), evalsPath);
+    success(`Found ${relEvalsPath}`);
+    // Scan companion files if it's a folder
+    const companions = isFolder ? await scanCompanionFiles(rootDir, evalsPath) : [];
+    if (companions.length > 0) {
+        success(`${companions.length} companion file${companions.length !== 1 ? "s" : ""} found`);
+        for (const c of companions) {
+            dim(c);
+        }
+    }
+    // Read and validate
+    const content = await fs.readFile(evalsPath, "utf-8");
+    const result = validateEvalsJson(content);
+    // Check file references
+    const fileRefs = extractFileRefs(content);
+    if (fileRefs.length > 0) {
+        console.log();
+        const missing = [];
+        const found = [];
+        for (const ref of fileRefs) {
+            if (companions.includes(ref)) {
+                found.push(ref);
+            }
+            else {
+                missing.push(ref);
+            }
+        }
+        if (found.length > 0) {
+            success(`${found.length} referenced file${found.length !== 1 ? "s" : ""} present`);
+        }
+        if (missing.length > 0) {
+            warn(`${missing.length} referenced file${missing.length !== 1 ? "s" : ""} missing`);
+            for (const m of missing) {
+                dim(`${chalk.yellow("missing:")} ${m}`);
+            }
+        }
+    }
+    console.log();
+    if (result.errors.length > 0) {
+        console.log(chalk.bold.red("  Errors:"));
+        for (const e of result.errors) {
+            error(e);
+        }
+        console.log();
+    }
+    if (result.warnings.length > 0) {
+        console.log(chalk.bold.yellow("  Warnings:"));
+        for (const w of result.warnings) {
+            warn(w);
+        }
+        console.log();
+    }
+    console.log(chalk.bold("  Summary:"));
+    console.log();
+    dim(`File:        ${relEvalsPath}`);
+    dim(`Format:      ${getFramework(result.format)?.meta.name ?? result.format}`);
+    dim(`Eval count:  ${result.evalCount}`);
+    if (result.summary["skill_name"])
+        dim(`Skill:       ${result.summary["skill_name"]}`);
+    if (result.summary["name"])
+        dim(`Name:        ${result.summary["name"]}`);
+    if (result.summary["version"])
+        dim(`Version:     ${result.summary["version"]}`);
+    dim(`Valid:       ${result.valid ? chalk.green("yes") : chalk.red("no")}`);
+    console.log();
+    if (result.valid) {
+        success("Validation passed");
+    }
+    else {
+        error("Validation failed");
+    }
+    console.log();
+}
+//# sourceMappingURL=validate.js.map

package/dist/commands/validate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"validate.js","sourceRoot":"","sources":["../../src/commands/validate.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACvE,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAEpD,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC;IACjC,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;IAC/C,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM;IAC3C,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO;CACjC,CAAC,CAAC;AAEH,KAAK,UAAU,aAAa,CAAC,UAAkB;IAC7C,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAEvC,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;QAClB,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;IACtE,CAAC;IAED,yCAAyC;IACzC,MAAM,UAAU,GAAG;QACjB,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,OAAO,EAAE,YAAY,CAAC;QAC5C,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,YAAY,CAAC;KACpC,CAAC;IAEF,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,IAAI,CAAC;YACH,MAAM,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YAC3B,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC;QACvD,CAAC;QAAC,MAAM,CAAC;YACP,YAAY;QACd,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,KAAK,UAAU,kBAAkB,CAAC,OAAe,EAAE,SAAiB,EAAE,QAAQ,GAAG,CAAC;IAChF,MAAM,UAAU,GAAa,EAAE,CAAC;IAChC,MAAM,aAAa,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;IAExD,KAAK,UAAU,IAAI,CAAC,GAAW,EAAE,KAAa;QAC5C,IAAI,KAAK,GAAG,QAAQ;YAAE,OAAO;QAC7B,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QAE/D,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,IAAI,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;gBAAE,SAAS;YACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YAC5C,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;YAEtD,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;gBACxB,MAAM,IAAI,CAAC,QAAQ,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;YAClC,CAAC;iBAAM,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;gBAC1B,IAAI,YAAY,KAAK,aAAa;oBAAE,SAAS;gBAC7C,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;gBACnD,IAAI,kBAAkB,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;oBAChC,UAAU,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;gBAChC,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;IACvB,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,SAAS,eAAe,CAAC,OAAe;IACtC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACnC,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC;IAClE,MAAM,IAAI,GAAgB,IAAI,GAAG,EAAE,CAAC;IAEpC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YAC9B,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBAC3B,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC;oBAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACzD,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC;AACnB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,UAAmB;IAChD,MAAM,EAAE,CAAC;IAET,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,UAAU,IAAI,GAAG,CAAC,CAAC;IAEpE,uBAAuB;IACvB,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;IAChC,CAAC;IAAC,MAAM,CAAC;QACP,KAAK,CAAC,mBAAmB,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;QAC9C,OAAO,CAAC,GAAG,EAAE,CAAC;QACd,OAAO;IACT,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;IAEpC,IAAI,QAAQ,EAAE,CAAC;QACb,IAAI,CAAC,oBAAoB,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;IAC3D,CAAC;SAAM,CAAC;QACN,IAAI,CAAC,eAAe,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;IACrD,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,YAAY,CAAC,CAAC;IAEhD,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,CAAC,GAAG,EAAE,CAAC;QACd,KAAK,CAAC,qBAAqB,CAAC,CAAC;QAC7B,GAAG,CAAC,YAAY,CAAC,CAAC;QAClB,GAAG,CAAC,KAAK,YAAY,aAAa,CAAC,CAAC;QACpC,GAAG,CAAC,KAAK,YAAY,mBAAmB,CAAC,CAAC;QAC1C,OAAO,CAAC,GAAG,EAAE,CAAC;QACd,OAAO;IACT,CAAC;IAED,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,KAAK,CAAC;IACrC,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,CAAC,CAAC;IAC7D,OAAO,CAAC,SAAS,YAAY,EAAE,CAAC,CAAC;IAEjC,wCAAwC;IACxC,MAAM,UAAU,GAAG,QAAQ,CAAC,CAAC,CAAC,MAAM,kBAAkB,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAChF,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,OAAO,CAAC,GAAG,UAAU,CAAC,MAAM,kBAAkB,UAAU,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;QAC1F,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;YAC3B,GAAG,CAAC,CAAC,CAAC,CAAC;QACT,CAAC;IACH,CAAC;IAED,oBAAoB;IACpB,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IACtD,MAAM,MAAM,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;IAE1C,wBAAwB;IACxB,MAAM,QAAQ,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;IAC1C,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,OAAO,CAAC,GAAG,EAAE,CAAC;QACd,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,MAAM,KAAK,GAAa,EAAE,CAAC;QAE3B,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;YAC3B,IAAI,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC7B,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAClB,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACpB,CAAC;QACH,CAAC;QAED,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrB,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,mBAAmB,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;QACrF,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,IAAI,CAAC,GAAG,OAAO,CAAC,MAAM,mBAAmB,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;YACpF,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;gBACxB,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC1C,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,EAAE,CAAC;IAEd,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC;QACzC,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YAC9B,KAAK,CAAC,CAAC,CAAC,CAAC;QACX,CAAC;QACD,OAAO,CAAC,GAAG,EAAE,CAAC;IAChB,CAAC;IAED,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC/B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC;QAC9C,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YAChC,IAAI,CAAC,CAAC,CAAC,CAAC;QACV,CAAC;QACD,OAAO,CAAC,GAAG,EAAE,CAAC;IAChB,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;IACtC,OAAO,CAAC,GAAG,EAAE,CAAC;IACd,GAAG,CAAC,gBAAgB,YAAY,EAAE,CAAC,CAAC;IACpC,GAAG,CAAC,gBAAgB,YAAY,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,IAAI,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IAC/E,GAAG,CAAC,gBAAgB,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;IAExC,IAAI,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC;QAAE,GAAG,CAAC,gBAAgB,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;IACtF,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC;QAAE,GAAG,CAAC,gBAAgB,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAC1E,IAAI,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC;QAAE,GAAG,CAAC,gBAAgB,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;IAEhF,GAAG,CAAC,gBAAgB,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAE3E,OAAO,CAAC,GAAG,EAAE,CAAC;IAEd,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QACjB,OAAO,CAAC,mBAAmB,CAAC,CAAC;IAC/B,CAAC;SAAM,CAAC;QACN,KAAK,CAAC,mBAAmB,CAAC,CAAC;IAC7B,CAAC;IACD,OAAO,CAAC,GAAG,EAAE,CAAC;AAChB,CAAC"}

package/dist/format.d.ts

@@ -0,0 +1,8 @@
+export declare const VERSION = "0.1.0";
+export declare function header(): void;
+export declare function success(msg: string): void;
+export declare function error(msg: string): void;
+export declare function warn(msg: string): void;
+export declare function info(msg: string): void;
+export declare function dim(msg: string): void;
+export declare function table(rows: string[][]): void;

package/dist/format.js

@@ -0,0 +1,32 @@
+import chalk from "chalk";
+export const VERSION = "0.1.0";
+export function header() {
+    console.log(chalk.bold.cyan(`\nevalify`) + chalk.dim(` v${VERSION}\n`));
+}
+export function success(msg) {
+    console.log(chalk.green(`  ✓ ${msg}`));
+}
+export function error(msg) {
+    console.log(chalk.red(`  ✗ ${msg}`));
+}
+export function warn(msg) {
+    console.log(chalk.yellow(`  ⚠ ${msg}`));
+}
+export function info(msg) {
+    console.log(chalk.cyan(`  → ${msg}`));
+}
+export function dim(msg) {
+    console.log(chalk.dim(`    ${msg}`));
+}
+export function table(rows) {
+    if (rows.length === 0)
+        return;
+    const colWidths = rows[0].map((_, colIndex) => Math.max(...rows.map((row) => (row[colIndex] ?? "").length)));
+    for (const row of rows) {
+        const line = row
+            .map((cell, i) => cell.padEnd(colWidths[i]))
+            .join("  ");
+        console.log(`  ${line}`);
+    }
+}
+//# sourceMappingURL=format.js.map

package/dist/format.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"format.js","sourceRoot":"","sources":["../src/format.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAE1B,MAAM,CAAC,MAAM,OAAO,GAAG,OAAO,CAAC;AAE/B,MAAM,UAAU,MAAM;IACpB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,KAAK,OAAO,IAAI,CAAC,CAAC,CAAC;AAC1E,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,GAAW;IACjC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC,CAAC;AACzC,CAAC;AAED,MAAM,UAAU,KAAK,CAAC,GAAW;IAC/B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC,CAAC;AACvC,CAAC;AAED,MAAM,UAAU,IAAI,CAAC,GAAW;IAC9B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC,CAAC;AAC1C,CAAC;AAED,MAAM,UAAU,IAAI,CAAC,GAAW;IAC9B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC,CAAC;AACxC,CAAC;AAED,MAAM,UAAU,GAAG,CAAC,GAAW;IAC7B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC,CAAC;AACvC,CAAC;AAED,MAAM,UAAU,KAAK,CAAC,IAAgB;IACpC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAE9B,MAAM,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,CAC5C,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAC7D,CAAC;IAEF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,IAAI,GAAG,GAAG;aACb,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;aAC3C,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;IAC3B,CAAC;AACH,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/dist/index.js

@@ -0,0 +1,38 @@
+#!/usr/bin/env node
+import { Command } from "commander";
+import { VERSION } from "./format.js";
+import { pull } from "./commands/pull.js";
+import { publish } from "./commands/publish.js";
+import { search } from "./commands/search.js";
+import { validate } from "./commands/validate.js";
+const program = new Command();
+program
+    .name("evalify")
+    .description("CLI tool for the Evalify eval criteria registry")
+    .version(VERSION);
+program
+    .command("pull <slug>")
+    .description("Download eval criteria from the registry")
+    .action(async (slug) => {
+    await pull(slug);
+});
+program
+    .command("publish [path]")
+    .description("Publish eval criteria from a file or skill folder")
+    .action(async (targetPath) => {
+    await publish(targetPath);
+});
+program
+    .command("search <query>")
+    .description("Search the registry for eval criteria")
+    .action(async (query) => {
+    await search(query);
+});
+program
+    .command("validate [path]")
+    .description("Validate evals.json from a file or skill folder")
+    .action(async (targetPath) => {
+    await validate(targetPath);
+});
+program.parse();
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AACtC,OAAO,EAAE,IAAI,EAAE,MAAM,oBAAoB,CAAC;AAC1C,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAChD,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAElD,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;AAE9B,OAAO;KACJ,IAAI,CAAC,SAAS,CAAC;KACf,WAAW,CAAC,iDAAiD,CAAC;KAC9D,OAAO,CAAC,OAAO,CAAC,CAAC;AAEpB,OAAO;KACJ,OAAO,CAAC,aAAa,CAAC;KACtB,WAAW,CAAC,0CAA0C,CAAC;KACvD,MAAM,CAAC,KAAK,EAAE,IAAY,EAAE,EAAE;IAC7B,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,gBAAgB,CAAC;KACzB,WAAW,CAAC,mDAAmD,CAAC;KAChE,MAAM,CAAC,KAAK,EAAE,UAAmB,EAAE,EAAE;IACpC,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC;AAC5B,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,gBAAgB,CAAC;KACzB,WAAW,CAAC,uCAAuC,CAAC;KACpD,MAAM,CAAC,KAAK,EAAE,KAAa,EAAE,EAAE;IAC9B,MAAM,MAAM,CAAC,KAAK,CAAC,CAAC;AACtB,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,iBAAiB,CAAC;KAC1B,WAAW,CAAC,iDAAiD,CAAC;KAC9D,MAAM,CAAC,KAAK,EAAE,UAAmB,EAAE,EAAE;IACpC,MAAM,QAAQ,CAAC,UAAU,CAAC,CAAC;AAC7B,CAAC,CAAC,CAAC;AAEL,OAAO,CAAC,KAAK,EAAE,CAAC"}

package/dist/validator.d.ts

@@ -0,0 +1,9 @@
+export interface ValidationResult {
+    valid: boolean;
+    format: string;
+    errors: string[];
+    warnings: string[];
+    evalCount: number;
+    summary: Record<string, unknown>;
+}
+export declare function validateEvalsJson(content: string): ValidationResult;

package/dist/validator.js

@@ -0,0 +1,92 @@
+import { detectFramework } from "@evalify/frameworks";
+export function validateEvalsJson(content) {
+    const result = {
+        valid: true,
+        format: "unknown",
+        errors: [],
+        warnings: [],
+        evalCount: 0,
+        summary: {},
+    };
+    let parsed;
+    try {
+        parsed = JSON.parse(content);
+    }
+    catch {
+        result.valid = false;
+        result.errors.push("Invalid JSON — could not parse file");
+        return result;
+    }
+    // Accept both top-level array and { skill_name, evals: [...] } wrapper
+    let evals;
+    let hasSkillName = false;
+    if (Array.isArray(parsed)) {
+        evals = parsed;
+    }
+    else if (typeof parsed === "object" &&
+        parsed !== null &&
+        "evals" in parsed &&
+        Array.isArray(parsed.evals)) {
+        evals = parsed.evals;
+        const meta = parsed;
+        hasSkillName = typeof meta.skill_name === "string";
+        if (meta.skill_name)
+            result.summary["skill_name"] = meta.skill_name;
+        if (meta.name)
+            result.summary["name"] = meta.name;
+        if (meta.version)
+            result.summary["version"] = meta.version;
+        if (meta.description)
+            result.summary["description"] = meta.description;
+    }
+    else {
+        result.valid = false;
+        result.errors.push("Expected a JSON array or an object with an \"evals\" array");
+        return result;
+    }
+    result.evalCount = evals.length;
+    if (evals.length === 0) {
+        result.warnings.push("File contains zero eval entries");
+        return result;
+    }
+    // Detect format using framework registry
+    const framework = detectFramework(parsed);
+    if (framework) {
+        result.format = framework.meta.id;
+    }
+    else {
+        result.format = "unknown";
+        result.warnings.push("Could not detect format — expected {prompt, expectations} or Anthropic Skill Creator format");
+    }
+    // Validate each entry
+    for (let i = 0; i < evals.length; i++) {
+        const entry = evals[i];
+        if (typeof entry !== "object" || entry === null) {
+            result.errors.push(`Entry ${i}: not an object`);
+            result.valid = false;
+            continue;
+        }
+        const e = entry;
+        if (!("prompt" in e) || typeof e.prompt !== "string") {
+            result.errors.push(`Entry ${i}: missing or invalid "prompt" (expected string)`);
+            result.valid = false;
+        }
+        if (!("expectations" in e)) {
+            if (result.format === "anthropic/skillcreator/v2" && "expected_output" in e) {
+                // Skill Creator two-phase workflow: expected_output first, expectations added later
+                result.warnings.push(`Entry ${i}: has "expected_output" but no "expectations" — assertions may not have been added yet`);
+            }
+            else {
+                result.errors.push(`Entry ${i}: missing "expectations"`);
+                result.valid = false;
+            }
+        }
+        else if (!Array.isArray(e.expectations) &&
+            typeof e.expectations !== "string") {
+            result.errors.push(`Entry ${i}: "expectations" should be a string or array`);
+            result.valid = false;
+        }
+    }
+    return result;
+}
+//# sourceMappingURL=validator.js.map

package/dist/validator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"validator.js","sourceRoot":"","sources":["../src/validator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAWtD,MAAM,UAAU,iBAAiB,CAAC,OAAe;IAC/C,MAAM,MAAM,GAAqB;QAC/B,KAAK,EAAE,IAAI;QACX,MAAM,EAAE,SAAS;QACjB,MAAM,EAAE,EAAE;QACV,QAAQ,EAAE,EAAE;QACZ,SAAS,EAAE,CAAC;QACZ,OAAO,EAAE,EAAE;KACZ,CAAC;IAEF,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC;QACrB,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,qCAAqC,CAAC,CAAC;QAC1D,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,uEAAuE;IACvE,IAAI,KAAgB,CAAC;IACrB,IAAI,YAAY,GAAG,KAAK,CAAC;IAEzB,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAC1B,KAAK,GAAG,MAAM,CAAC;IACjB,CAAC;SAAM,IACL,OAAO,MAAM,KAAK,QAAQ;QAC1B,MAAM,KAAK,IAAI;QACf,OAAO,IAAI,MAAM;QACjB,KAAK,CAAC,OAAO,CAAE,MAAkC,CAAC,KAAK,CAAC,EACxD,CAAC;QACD,KAAK,GAAI,MAAkC,CAAC,KAAkB,CAAC;QAC/D,MAAM,IAAI,GAAG,MAAiC,CAAC;QAC/C,YAAY,GAAG,OAAO,IAAI,CAAC,UAAU,KAAK,QAAQ,CAAC;QACnD,IAAI,IAAI,CAAC,UAAU;YAAE,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC;QACpE,IAAI,IAAI,CAAC,IAAI;YAAE,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC;QAClD,IAAI,IAAI,CAAC,OAAO;YAAE,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC;QAC3D,IAAI,IAAI,CAAC,WAAW;YAAE,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC;IACzE,CAAC;SAAM,CAAC;QACN,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC;QACrB,MAAM,CAAC,MAAM,CAAC,IAAI,CAChB,4DAA4D,CAC7D,CAAC;QACF,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,MAAM,CAAC,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC;IAEhC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;QACxD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,yCAAyC;IACzC,MAAM,SAAS,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC;IAC1C,IAAI,SAAS,EAAE,CAAC;QACd,MAAM,CAAC,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;IACpC,CAAC;SAAM,CAAC;QACN,MAAM,CAAC,MAAM,GAAG,SAAS,CAAC;QAC1B,MAAM,CAAC,QAAQ,CAAC,IAAI,CAClB,6FAA6F,CAC9F,CAAC;IACJ,CAAC;IAED,sBAAsB;IACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACvB,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YAChD,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;YAChD,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC;YACrB,SAAS;QACX,CAAC;QAED,MAAM,CAAC,GAAG,KAAgC,CAAC;QAE3C,IAAI,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YACrD,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,iDAAiD,CAAC,CAAC;YAChF,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC;QACvB,CAAC;QAED,IAAI,CAAC,CAAC,cAAc,IAAI,CAAC,CAAC,EAAE,CAAC;YAC3B,IAAI,MAAM,CAAC,MAAM,KAAK,2BAA2B,IAAI,iBAAiB,IAAI,CAAC,EAAE,CAAC;gBAC5E,oFAAoF;gBACpF,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,wFAAwF,CAAC,CAAC;YAC3H,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,0BAA0B,CAAC,CAAC;gBACzD,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC;YACvB,CAAC;QACH,CAAC;aAAM,IACL,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,YAAY,CAAC;YAC9B,OAAO,CAAC,CAAC,YAAY,KAAK,QAAQ,EAClC,CAAC;YACD,MAAM,CAAC,MAAM,CAAC,IAAI,CAChB,SAAS,CAAC,8CAA8C,CACzD,CAAC;YACF,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC;QACvB,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/package.json

@@ -0,0 +1,25 @@
+{
+  "name": "evalify-cli",
+  "version": "0.1.0",
+  "description": "CLI tool for the Evalify eval criteria registry",
+  "homepage": "https://evalify.sh",
+  "repository": "https://github.com/AppVerse-cc/evalify",
+  "bin": {
+    "evalify": "./dist/index.js"
+  },
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsc --watch",
+    "start": "node dist/index.js"
+  },
+  "dependencies": {
+    "@evalify/frameworks": "workspace:*",
+    "commander": "^12.1.0",
+    "chalk": "^5.4.1"
+  },
+  "devDependencies": {
+    "@types/node": "^22.0.0",
+    "typescript": "^5.7.0"
+  },
+  "type": "module"
+}

package/src/commands/publish.ts

@@ -0,0 +1,100 @@
+import path from "node:path";
+import fs from "node:fs/promises";
+import chalk from "chalk";
+import { getFramework } from "@evalify/frameworks";
+import { header, success, info, dim, error, warn } from "../format.js";
+import { validateEvalsJson } from "../validator.js";
+
+async function findEvalsFile(targetPath: string): Promise<string | null> {
+  const stat = await fs.stat(targetPath);
+
+  if (stat.isFile()) return targetPath;
+
+  const candidates = [
+    path.join(targetPath, "evals", "evals.json"),
+    path.join(targetPath, "evals.json"),
+  ];
+
+  for (const candidate of candidates) {
+    try {
+      await fs.access(candidate);
+      return candidate;
+    } catch {
+      // not found, continue
+    }
+  }
+
+  return null;
+}
+
+export async function publish(targetPath?: string): Promise<void> {
+  header();
+
+  const resolvedPath = path.resolve(process.cwd(), targetPath || ".");
+
+  try {
+    await fs.access(resolvedPath);
+  } catch {
+    error(`Path not found: ${targetPath || "."}`);
+    console.log();
+    return;
+  }
+
+  info("Publishing eval criteria to registry...");
+  console.log();
+
+  const filePath = await findEvalsFile(resolvedPath);
+
+  if (!filePath) {
+    error("No evals.json found");
+    dim("Looked in:");
+    dim(`  ${resolvedPath}/evals.json`);
+    dim(`  ${resolvedPath}/evals/evals.json`);
+    console.log();
+    return;
+  }
+
+  success(`Found ${path.relative(process.cwd(), filePath)}`);
+
+  const content = await fs.readFile(filePath, "utf-8");
+  const result = validateEvalsJson(content);
+
+  if (!result.valid) {
+    console.log();
+    error("Validation failed — cannot publish");
+    for (const e of result.errors) {
+      error(e);
+    }
+    console.log();
+    return;
+  }
+
+  for (const w of result.warnings) {
+    warn(w);
+  }
+
+  console.log();
+  console.log(chalk.bold("  Publish summary:"));
+  console.log();
+
+  if (result.summary["skill_name"]) {
+    dim(`Skill:       ${result.summary["skill_name"]}`);
+  }
+  if (result.summary["name"]) {
+    dim(`Name:        ${result.summary["name"]}`);
+  }
+  if (result.summary["version"]) {
+    dim(`Version:     ${result.summary["version"]}`);
+  }
+  if (result.summary["description"]) {
+    dim(`Description: ${result.summary["description"]}`);
+  }
+  dim(`Format:      ${getFramework(result.format)?.meta.name ?? result.format}`);
+  dim(`Eval count:  ${result.evalCount}`);
+  dim(`File:        ${path.relative(process.cwd(), filePath)}`);
+
+  console.log();
+  warn("Dry run — publishing is not yet connected to the registry");
+  success("File is valid and ready to publish");
+  console.log();
+}

package/src/commands/pull.ts

@@ -0,0 +1,81 @@
+import path from "node:path";
+import fs from "node:fs/promises";
+import { header, success, info, dim, error } from "../format.js";
+
+const REGISTRY_URL = "https://evalify.sh/api/registry";
+
+export async function pull(slug: string): Promise<void> {
+  header();
+
+  if (!slug) {
+    error("Missing slug argument");
+    console.log();
+    return;
+  }
+
+  info(`Pulling eval criteria: ${slug}`);
+  console.log();
+
+  let pack: {
+    slug: string;
+    displayName: string;
+    version: string;
+    description: string;
+    domain: string;
+    author: string;
+    tags: string[];
+    evals: { prompt: string; expectations: string[] }[];
+  };
+
+  try {
+    const res = await fetch(`${REGISTRY_URL}/${slug}`);
+    if (res.status === 404) {
+      error(`Criteria not found: ${slug}`);
+      dim(`Check the registry at https://evalify.sh`);
+      console.log();
+      return;
+    }
+    if (!res.ok) {
+      error(`Registry returned ${res.status}`);
+      console.log();
+      return;
+    }
+    pack = await res.json();
+  } catch (err) {
+    error(`Failed to reach registry: ${(err as Error).message}`);
+    console.log();
+    return;
+  }
+
+  const targetDir = path.resolve(process.cwd(), "evals", slug);
+  const targetFile = path.join(targetDir, "evals.json");
+
+  try {
+    await fs.mkdir(targetDir, { recursive: true });
+
+    const output = {
+      slug: pack.slug,
+      displayName: pack.displayName,
+      version: pack.version,
+      description: pack.description,
+      domain: pack.domain,
+      author: pack.author,
+      tags: pack.tags,
+      evals: pack.evals,
+    };
+
+    await fs.writeFile(targetFile, JSON.stringify(output, null, 2) + "\n");
+
+    success(`Pulled ${pack.displayName} v${pack.version}`);
+    success(`Wrote ${pack.evals.length} eval${pack.evals.length !== 1 ? "s" : ""} to evals/${slug}/evals.json`);
+    console.log();
+    dim(`Author:   ${pack.author}`);
+    dim(`Domain:   ${pack.domain}`);
+    dim(`Location: ${targetFile}`);
+    dim(`To validate: evalify validate evals/${slug}`);
+  } catch (err) {
+    error(`Failed to write file: ${(err as Error).message}`);
+  }
+
+  console.log();
+}

package/src/commands/search.ts

@@ -0,0 +1,23 @@
+import chalk from "chalk";
+import { header, info, dim, table } from "../format.js";
+
+export async function search(query: string): Promise<void> {
+  header();
+  info(`Searching registry for: ${chalk.bold(query)}`);
+  console.log();
+
+  // Placeholder results for now
+  const results = [
+    [chalk.bold("Slug"), chalk.bold("Description"), chalk.bold("Evals")],
+    [chalk.dim("─".repeat(20)), chalk.dim("─".repeat(35)), chalk.dim("─".repeat(5))],
+    ["code-review", "Code review quality criteria", "12"],
+    ["summarization", "Text summarization accuracy", "8"],
+    ["safety-checks", "Safety and content policy evals", "24"],
+  ];
+
+  table(results);
+  console.log();
+  dim(`Showing placeholder results — registry search not yet connected`);
+  dim(`Use: evalify pull <slug> to download criteria`);
+  console.log();
+}

package/src/commands/validate.ts

@@ -0,0 +1,200 @@
+import path from "node:path";
+import fs from "node:fs/promises";
+import chalk from "chalk";
+import { getFramework } from "@evalify/frameworks";
+import { header, success, info, dim, error, warn } from "../format.js";
+import { validateEvalsJson } from "../validator.js";
+
+const ALLOWED_EXTENSIONS = new Set([
+  ".json", ".md", ".txt", ".csv", ".yaml", ".yml",
+  ".ts", ".tsx", ".js", ".jsx", ".py", ".sql",
+  ".html", ".css", ".xml", ".toml",
+]);
+
+async function findEvalsJson(targetPath: string): Promise<{ evalsPath: string; rootDir: string } | null> {
+  const stat = await fs.stat(targetPath);
+
+  if (stat.isFile()) {
+    return { evalsPath: targetPath, rootDir: path.dirname(targetPath) };
+  }
+
+  // It's a directory — scan for evals.json
+  const candidates = [
+    path.join(targetPath, "evals", "evals.json"),
+    path.join(targetPath, "evals.json"),
+  ];
+
+  for (const candidate of candidates) {
+    try {
+      await fs.access(candidate);
+      return { evalsPath: candidate, rootDir: targetPath };
+    } catch {
+      // not found
+    }
+  }
+
+  return null;
+}
+
+async function scanCompanionFiles(rootDir: string, evalsPath: string, maxDepth = 2): Promise<string[]> {
+  const companions: string[] = [];
+  const evalsRelative = path.relative(rootDir, evalsPath);
+
+  async function walk(dir: string, depth: number) {
+    if (depth > maxDepth) return;
+    const entries = await fs.readdir(dir, { withFileTypes: true });
+
+    for (const entry of entries) {
+      if (entry.name.startsWith(".")) continue;
+      const fullPath = path.join(dir, entry.name);
+      const relativePath = path.relative(rootDir, fullPath);
+
+      if (entry.isDirectory()) {
+        await walk(fullPath, depth + 1);
+      } else if (entry.isFile()) {
+        if (relativePath === evalsRelative) continue;
+        const ext = path.extname(entry.name).toLowerCase();
+        if (ALLOWED_EXTENSIONS.has(ext)) {
+          companions.push(relativePath);
+        }
+      }
+    }
+  }
+
+  await walk(rootDir, 0);
+  return companions;
+}
+
+function extractFileRefs(content: string): string[] {
+  const parsed = JSON.parse(content);
+  const evals = Array.isArray(parsed) ? parsed : parsed.evals || [];
+  const refs: Set<string> = new Set();
+
+  for (const item of evals) {
+    if (Array.isArray(item.files)) {
+      for (const f of item.files) {
+        if (typeof f === "string" && f.length > 0) refs.add(f);
+      }
+    }
+  }
+
+  return [...refs];
+}
+
+export async function validate(targetPath?: string): Promise<void> {
+  header();
+
+  const resolvedPath = path.resolve(process.cwd(), targetPath || ".");
+
+  // Check if path exists
+  try {
+    await fs.access(resolvedPath);
+  } catch {
+    error(`Path not found: ${targetPath || "."}`);
+    console.log();
+    return;
+  }
+
+  const stat = await fs.stat(resolvedPath);
+  const isFolder = stat.isDirectory();
+
+  if (isFolder) {
+    info(`Scanning folder: ${path.basename(resolvedPath)}/`);
+  } else {
+    info(`Validating: ${path.basename(resolvedPath)}`);
+  }
+
+  const found = await findEvalsJson(resolvedPath);
+
+  if (!found) {
+    console.log();
+    error("No evals.json found");
+    dim("Looked in:");
+    dim(`  ${resolvedPath}/evals.json`);
+    dim(`  ${resolvedPath}/evals/evals.json`);
+    console.log();
+    return;
+  }
+
+  const { evalsPath, rootDir } = found;
+  const relEvalsPath = path.relative(process.cwd(), evalsPath);
+  success(`Found ${relEvalsPath}`);
+
+  // Scan companion files if it's a folder
+  const companions = isFolder ? await scanCompanionFiles(rootDir, evalsPath) : [];
+  if (companions.length > 0) {
+    success(`${companions.length} companion file${companions.length !== 1 ? "s" : ""} found`);
+    for (const c of companions) {
+      dim(c);
+    }
+  }
+
+  // Read and validate
+  const content = await fs.readFile(evalsPath, "utf-8");
+  const result = validateEvalsJson(content);
+
+  // Check file references
+  const fileRefs = extractFileRefs(content);
+  if (fileRefs.length > 0) {
+    console.log();
+    const missing: string[] = [];
+    const found: string[] = [];
+
+    for (const ref of fileRefs) {
+      if (companions.includes(ref)) {
+        found.push(ref);
+      } else {
+        missing.push(ref);
+      }
+    }
+
+    if (found.length > 0) {
+      success(`${found.length} referenced file${found.length !== 1 ? "s" : ""} present`);
+    }
+    if (missing.length > 0) {
+      warn(`${missing.length} referenced file${missing.length !== 1 ? "s" : ""} missing`);
+      for (const m of missing) {
+        dim(`${chalk.yellow("missing:")} ${m}`);
+      }
+    }
+  }
+
+  console.log();
+
+  if (result.errors.length > 0) {
+    console.log(chalk.bold.red("  Errors:"));
+    for (const e of result.errors) {
+      error(e);
+    }
+    console.log();
+  }
+
+  if (result.warnings.length > 0) {
+    console.log(chalk.bold.yellow("  Warnings:"));
+    for (const w of result.warnings) {
+      warn(w);
+    }
+    console.log();
+  }
+
+  console.log(chalk.bold("  Summary:"));
+  console.log();
+  dim(`File:        ${relEvalsPath}`);
+  dim(`Format:      ${getFramework(result.format)?.meta.name ?? result.format}`);
+  dim(`Eval count:  ${result.evalCount}`);
+
+  if (result.summary["skill_name"]) dim(`Skill:       ${result.summary["skill_name"]}`);
+  if (result.summary["name"]) dim(`Name:        ${result.summary["name"]}`);
+  if (result.summary["version"]) dim(`Version:     ${result.summary["version"]}`);
+
+  dim(`Valid:       ${result.valid ? chalk.green("yes") : chalk.red("no")}`);
+
+  console.log();
+
+  if (result.valid) {
+    success("Validation passed");
+  } else {
+    error("Validation failed");
+  }
+  console.log();
+}

package/src/format.ts

@@ -0,0 +1,42 @@
+import chalk from "chalk";
+
+export const VERSION = "0.1.0";
+
+export function header(): void {
+  console.log(chalk.bold.cyan(`\nevalify`) + chalk.dim(` v${VERSION}\n`));
+}
+
+export function success(msg: string): void {
+  console.log(chalk.green(`  ✓ ${msg}`));
+}
+
+export function error(msg: string): void {
+  console.log(chalk.red(`  ✗ ${msg}`));
+}
+
+export function warn(msg: string): void {
+  console.log(chalk.yellow(`  ⚠ ${msg}`));
+}
+
+export function info(msg: string): void {
+  console.log(chalk.cyan(`  → ${msg}`));
+}
+
+export function dim(msg: string): void {
+  console.log(chalk.dim(`    ${msg}`));
+}
+
+export function table(rows: string[][]): void {
+  if (rows.length === 0) return;
+
+  const colWidths = rows[0].map((_, colIndex) =>
+    Math.max(...rows.map((row) => (row[colIndex] ?? "").length))
+  );
+
+  for (const row of rows) {
+    const line = row
+      .map((cell, i) => cell.padEnd(colWidths[i]))
+      .join("  ");
+    console.log(`  ${line}`);
+  }
+}

package/src/index.ts

@@ -0,0 +1,45 @@
+#!/usr/bin/env node
+
+import { Command } from "commander";
+import { VERSION } from "./format.js";
+import { pull } from "./commands/pull.js";
+import { publish } from "./commands/publish.js";
+import { search } from "./commands/search.js";
+import { validate } from "./commands/validate.js";
+
+const program = new Command();
+
+program
+  .name("evalify")
+  .description("CLI tool for the Evalify eval criteria registry")
+  .version(VERSION);
+
+program
+  .command("pull <slug>")
+  .description("Download eval criteria from the registry")
+  .action(async (slug: string) => {
+    await pull(slug);
+  });
+
+program
+  .command("publish [path]")
+  .description("Publish eval criteria from a file or skill folder")
+  .action(async (targetPath?: string) => {
+    await publish(targetPath);
+  });
+
+program
+  .command("search <query>")
+  .description("Search the registry for eval criteria")
+  .action(async (query: string) => {
+    await search(query);
+  });
+
+program
+  .command("validate [path]")
+  .description("Validate evals.json from a file or skill folder")
+  .action(async (targetPath?: string) => {
+    await validate(targetPath);
+  });
+
+program.parse();

package/src/validator.ts

@@ -0,0 +1,112 @@
+import { detectFramework } from "@evalify/frameworks";
+
+export interface ValidationResult {
+  valid: boolean;
+  format: string;
+  errors: string[];
+  warnings: string[];
+  evalCount: number;
+  summary: Record<string, unknown>;
+}
+
+export function validateEvalsJson(content: string): ValidationResult {
+  const result: ValidationResult = {
+    valid: true,
+    format: "unknown",
+    errors: [],
+    warnings: [],
+    evalCount: 0,
+    summary: {},
+  };
+
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(content);
+  } catch {
+    result.valid = false;
+    result.errors.push("Invalid JSON — could not parse file");
+    return result;
+  }
+
+  // Accept both top-level array and { skill_name, evals: [...] } wrapper
+  let evals: unknown[];
+  let hasSkillName = false;
+
+  if (Array.isArray(parsed)) {
+    evals = parsed;
+  } else if (
+    typeof parsed === "object" &&
+    parsed !== null &&
+    "evals" in parsed &&
+    Array.isArray((parsed as Record<string, unknown>).evals)
+  ) {
+    evals = (parsed as Record<string, unknown>).evals as unknown[];
+    const meta = parsed as Record<string, unknown>;
+    hasSkillName = typeof meta.skill_name === "string";
+    if (meta.skill_name) result.summary["skill_name"] = meta.skill_name;
+    if (meta.name) result.summary["name"] = meta.name;
+    if (meta.version) result.summary["version"] = meta.version;
+    if (meta.description) result.summary["description"] = meta.description;
+  } else {
+    result.valid = false;
+    result.errors.push(
+      "Expected a JSON array or an object with an \"evals\" array"
+    );
+    return result;
+  }
+
+  result.evalCount = evals.length;
+
+  if (evals.length === 0) {
+    result.warnings.push("File contains zero eval entries");
+    return result;
+  }
+
+  // Detect format using framework registry
+  const framework = detectFramework(parsed);
+  if (framework) {
+    result.format = framework.meta.id;
+  } else {
+    result.format = "unknown";
+    result.warnings.push(
+      "Could not detect format — expected {prompt, expectations} or Anthropic Skill Creator format"
+    );
+  }
+
+  // Validate each entry
+  for (let i = 0; i < evals.length; i++) {
+    const entry = evals[i];
+    if (typeof entry !== "object" || entry === null) {
+      result.errors.push(`Entry ${i}: not an object`);
+      result.valid = false;
+      continue;
+    }
+
+    const e = entry as Record<string, unknown>;
+
+    if (!("prompt" in e) || typeof e.prompt !== "string") {
+      result.errors.push(`Entry ${i}: missing or invalid "prompt" (expected string)`);
+      result.valid = false;
+    }
+
+    if (!("expectations" in e)) {
+      if (result.format === "anthropic/skillcreator/v2" && "expected_output" in e) {
+        // Skill Creator two-phase workflow: expected_output first, expectations added later
+        result.warnings.push(`Entry ${i}: has "expected_output" but no "expectations" — assertions may not have been added yet`);
+      } else {
+        result.errors.push(`Entry ${i}: missing "expectations"`);
+        result.valid = false;
+      }
+    } else if (
+      !Array.isArray(e.expectations) &&
+      typeof e.expectations !== "string"
+    ) {
+      result.errors.push(
+        `Entry ${i}: "expectations" should be a string or array`
+      );
+      result.valid = false;
+    }
+  }
+
+  return result;
+}

package/tsconfig.json

@@ -0,0 +1,17 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "Node16",
+    "moduleResolution": "Node16",
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "declaration": true,
+    "sourceMap": true
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist"]
+}