evalify-cli 0.1.2 → 0.1.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "evalify-cli",
-  "version": "0.1.2",
+  "version": "0.1.4",
   "description": "CLI tool for the Evalify eval criteria registry",
   "homepage": "https://evalify.sh",
   "repository": "https://github.com/AppVerse-cc/evalify",
@@ -14,11 +14,13 @@
   },
   "dependencies": {
     "chalk": "^5.4.1",
-    "commander": "^12.1.0"
+    "commander": "^12.1.0",
+    "prompts": "^2.4.2"
   },
   "devDependencies": {
     "@evalify/frameworks": "workspace:*",
     "@types/node": "^22.0.0",
+    "@types/prompts": "^2.4.9",
     "tsup": "^8.5.1",
     "typescript": "^5.7.0"
   }

package/src/commands/login.ts

@@ -0,0 +1,181 @@
+import http from "node:http";
+import crypto from "node:crypto";
+import { exec } from "node:child_process";
+import path from "node:path";
+import os from "node:os";
+import fs from "node:fs/promises";
+import { header, success, info, dim, error } from "../format.js";
+
+const AUTH_FILE = path.join(os.homedir(), ".evalify", "auth.json");
+const REGISTRY_BASE = "https://www.evalify.sh";
+
+export interface AuthData {
+  access_token: string;
+  refresh_token: string;
+  handle: string;
+}
+
+export async function readAuth(): Promise<AuthData | null> {
+  try {
+    const content = await fs.readFile(AUTH_FILE, "utf-8");
+    return JSON.parse(content) as AuthData;
+  } catch {
+    return null;
+  }
+}
+
+export async function logout(): Promise<void> {
+  header();
+  try {
+    await fs.rm(AUTH_FILE);
+    success("Logged out");
+  } catch {
+    error("Not logged in");
+  }
+  console.log();
+}
+
+function openBrowser(url: string): void {
+  const cmd =
+    process.platform === "darwin"
+      ? "open"
+      : process.platform === "win32"
+        ? "start"
+        : "xdg-open";
+  exec(`${cmd} "${url}"`);
+}
+
+/** Returns a valid (non-expired) access token, refreshing if needed. */
+export async function getValidToken(): Promise<string | null> {
+  const auth = await readAuth();
+  if (!auth) return null;
+
+  // Decode JWT expiry without a library
+  try {
+    const payload = JSON.parse(Buffer.from(auth.access_token.split(".")[1], "base64url").toString());
+    const expiresAt: number = payload.exp * 1000;
+    if (Date.now() < expiresAt - 60_000) return auth.access_token; // still valid
+  } catch {
+    return auth.access_token; // can't decode, try anyway
+  }
+
+  if (!auth.refresh_token) return null;
+
+  // Refresh via evalify.sh refresh endpoint
+  try {
+    const res = await fetch(`${REGISTRY_BASE}/api/auth/refresh`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ refresh_token: auth.refresh_token }),
+    });
+    if (!res.ok) return null;
+    const data = await res.json();
+    const updated: AuthData = {
+      access_token: data.access_token,
+      refresh_token: data.refresh_token ?? auth.refresh_token,
+      handle: auth.handle,
+    };
+    await fs.writeFile(AUTH_FILE, JSON.stringify(updated, null, 2));
+    return updated.access_token;
+  } catch {
+    return null;
+  }
+}
+
+export async function login(): Promise<void> {
+  header();
+  info("Logging in to Evalify...");
+  console.log();
+
+  const tokens = await waitForToken();
+  if (!tokens) return;
+
+  // Verify token and fetch handle
+  let handle = "";
+  try {
+    const res = await fetch(`${REGISTRY_BASE}/api/auth/me`, {
+      headers: { Authorization: `Bearer ${tokens.access_token}` },
+    });
+    if (res.ok) {
+      const data = await res.json();
+      handle = data.handle ?? "";
+    }
+  } catch {
+    // non-critical — save token anyway
+  }
+
+  // Save auth
+  const authDir = path.dirname(AUTH_FILE);
+  await fs.mkdir(authDir, { recursive: true });
+  await fs.writeFile(
+    AUTH_FILE,
+    JSON.stringify({ access_token: tokens.access_token, refresh_token: tokens.refresh_token, handle }, null, 2)
+  );
+
+  console.log();
+  success(`Logged in${handle ? ` as ${handle}` : ""}`);
+  dim(`Token saved to ${AUTH_FILE}`);
+  console.log();
+}
+
+function waitForToken(): Promise<{ access_token: string; refresh_token: string } | null> {
+  return new Promise((resolve) => {
+    const state = crypto.randomBytes(16).toString("hex");
+
+    const server = http.createServer((req, res) => {
+      const url = new URL(req.url ?? "/", "http://localhost");
+
+      if (url.pathname !== "/callback") {
+        res.writeHead(404);
+        res.end();
+        return;
+      }
+
+      const returnedState = url.searchParams.get("state");
+      const token = url.searchParams.get("token");
+      const refreshToken = url.searchParams.get("refresh_token");
+
+      const ok = returnedState === state && !!token;
+
+      res.writeHead(200, { "Content-Type": "text/html; charset=utf-8" });
+      res.end(`<!doctype html><html><body style="font-family:sans-serif;padding:2rem;background:#0a0a0a;color:#e4e4e7">
+${ok
+  ? `<h2 style="color:#22c55e">&#10003; Authentication successful</h2><p style="color:#71717a">You can close this tab and return to your terminal.</p>`
+  : `<h2 style="color:#ef4444">&#10007; Authentication failed</h2><p style="color:#71717a">Invalid state. Please try again.</p>`
+}
+</body></html>`);
+
+      req.socket.destroy();
+      server.close();
+
+      if (!ok) {
+        error(returnedState !== state ? "State mismatch — possible CSRF" : "No token received");
+        resolve(null);
+        return;
+      }
+
+      resolve({ access_token: token!, refresh_token: refreshToken ?? "" });
+    });
+
+    server.unref();
+    server.listen(0, "127.0.0.1", () => {
+      const port = (server.address() as { port: number }).port;
+      const authUrl = `${REGISTRY_BASE}/auth/cli?port=${port}&state=${state}`;
+      dim(`Opening ${authUrl}`);
+      dim("Waiting for authentication...");
+      openBrowser(authUrl);
+    });
+
+    server.on("error", (err) => {
+      error(`Local server error: ${err.message}`);
+      resolve(null);
+    });
+
+    // Timeout after 5 minutes
+    setTimeout(() => {
+      server.close();
+      error("Authentication timed out");
+      resolve(null);
+    }, 5 * 60 * 1000);
+  });
+}

package/src/commands/publish.ts

@@ -1,9 +1,13 @@
 import path from "node:path";
 import fs from "node:fs/promises";
 import chalk from "chalk";
+import prompts from "prompts";
 import { getFramework } from "@evalify/frameworks";
 import { header, success, info, dim, error, warn } from "../format.js";
 import { validateEvalsJson } from "../validator.js";
+import { readAuth, getValidToken, login } from "./login.js";
+
+const REGISTRY_URL = "https://www.evalify.sh/api/publish";
 
 async function findEvalsFile(targetPath: string): Promise<string | null> {
   const stat = await fs.stat(targetPath);
@@ -30,17 +34,47 @@ async function findEvalsFile(targetPath: string): Promise<string | null> {
 export async function publish(targetPath?: string): Promise<void> {
   header();
 
+  // Check auth
+  let auth = await readAuth();
+  if (!auth) {
+    error("Not logged in");
+    console.log();
+    const { action } = await prompts({
+      type: "select",
+      name: "action",
+      message: "What would you like to do?",
+      choices: [
+        { title: "Login now", value: "login" },
+        { title: "Later", value: "later" },
+      ],
+      initial: 0,
+    });
+    if (!action || action === "later") {
+      console.log();
+      dim("Run: evalify-cli login when ready.");
+      console.log();
+      return;
+    }
+    await login();
+    auth = await readAuth();
+    if (!auth) {
+      console.log();
+      return;
+    }
+  }
+
   const resolvedPath = path.resolve(process.cwd(), targetPath || ".");
 
   try {
     await fs.access(resolvedPath);
   } catch {
-    error(`Path not found: ${targetPath || "."}`);
+    error(`Path not found: ${resolvedPath}`);
+    dim("Pass a path to a folder containing evals.json, or run from that folder.");
     console.log();
     return;
   }
 
-  info("Publishing eval criteria to registry...");
+  info(`Publishing from ${path.relative(process.cwd(), resolvedPath) || "."}`);
   console.log();
 
   const filePath = await findEvalsFile(resolvedPath);
@@ -73,28 +107,84 @@ export async function publish(targetPath?: string): Promise<void> {
     warn(w);
   }
 
+  const parsed = JSON.parse(content);
+
+  const displayName = parsed.displayName ?? parsed.skill_name ?? parsed.name ?? "";
+  const slug =
+    parsed.slug ??
+    displayName
+      .toLowerCase()
+      .replace(/[^a-z0-9]+/g, "-")
+      .replace(/^-|-$/g, "");
+
   console.log();
   console.log(chalk.bold("  Publish summary:"));
   console.log();
-
-  if (result.summary["skill_name"]) {
-    dim(`Skill:       ${result.summary["skill_name"]}`);
-  }
-  if (result.summary["name"]) {
-    dim(`Name:        ${result.summary["name"]}`);
-  }
-  if (result.summary["version"]) {
-    dim(`Version:     ${result.summary["version"]}`);
-  }
-  if (result.summary["description"]) {
-    dim(`Description: ${result.summary["description"]}`);
-  }
+  dim(`Name:        ${displayName || "(unnamed)"}`);
+  dim(`Slug:        ${slug || "(auto)"}`);
+  dim(`Version:     ${parsed.version ?? "1.0.0"}`);
+  dim(`Domain:      ${parsed.domain ?? "general"}`);
   dim(`Format:      ${getFramework(result.format)?.meta.name ?? result.format}`);
   dim(`Eval count:  ${result.evalCount}`);
-  dim(`File:        ${path.relative(process.cwd(), filePath)}`);
-
-  console.log();
-  warn("Dry run — publishing is not yet connected to the registry");
-  success("File is valid and ready to publish");
+  dim(`Author:      ${auth.handle}`);
   console.log();
-}
+
+  const { confirm } = await prompts({
+    type: "confirm",
+    name: "confirm",
+    message: "Publish to evalify.sh?",
+    initial: true,
+  });
+
+  if (!confirm) {
+    console.log();
+    dim("Cancelled.");
+    console.log();
+    return;
+  }
+
+  info("Uploading...");
+
+  try {
+    const token = await getValidToken();
+    if (!token) {
+      error("Session expired — run: evalify-cli login");
+      return;
+    }
+
+    const res = await fetch(REGISTRY_URL, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        Authorization: `Bearer ${token}`,
+      },
+      body: JSON.stringify({
+        slug,
+        displayName,
+        domain: parsed.domain ?? "general",
+        version: parsed.version ?? "1.0.0",
+        tags: parsed.tags ?? [],
+        description: parsed.description ?? "",
+        evals: parsed.evals ?? [],
+      }),
+    });
+
+    const data = await res.json();
+
+    if (!res.ok) {
+      console.log();
+      error(data.error ?? `Server error ${res.status}`);
+      console.log();
+      return;
+    }
+
+    console.log();
+    success(`Published ${displayName || slug}`);
+    dim(`View at: https://evalify.sh/criteria/${data.slug}`);
+    console.log();
+  } catch (err) {
+    console.log();
+    error(`Failed to publish: ${(err as Error).message}`);
+    console.log();
+  }
+}

package/src/commands/pull.ts

@@ -1,8 +1,33 @@
 import path from "node:path";
+import os from "node:os";
 import fs from "node:fs/promises";
+import prompts from "prompts";
 import { header, success, info, dim, error } from "../format.js";
 
-const REGISTRY_URL = "https://evalify.sh/api/registry";
+const REGISTRY_URL = "https://www.evalify.sh/api/registry";
+
+async function detectSkills(skillsDir: string): Promise<{ name: string; evalCount: number }[]> {
+  try {
+    const entries = await fs.readdir(skillsDir, { withFileTypes: true });
+    const skills: { name: string; evalCount: number }[] = [];
+
+    for (const entry of entries) {
+      if (!entry.isDirectory()) continue;
+      const evalsFile = path.join(skillsDir, entry.name, "evals.json");
+      try {
+        const content = await fs.readFile(evalsFile, "utf-8");
+        const parsed = JSON.parse(content);
+        skills.push({ name: entry.name, evalCount: (parsed.evals ?? []).length });
+      } catch {
+        // Folder exists but no valid evals.json — skip
+      }
+    }
+
+    return skills;
+  } catch {
+    return [];
+  }
+}
 
 export async function pull(slug: string): Promise<void> {
   header();
@@ -47,9 +72,142 @@ export async function pull(slug: string): Promise<void> {
     return;
   }
 
-  const targetDir = path.resolve(process.cwd(), "evals", slug);
+  // Ask install location
+  const locationResponse = await prompts({
+    type: "select",
+    name: "location",
+    message: "Where do you want to install?",
+    choices: [
+      {
+        title: "Current directory",
+        description: `evals/${pack.slug}/evals.json`,
+        value: "current",
+      },
+      {
+        title: "Project",
+        description: ".claude/skills/<name>/evals.json",
+        value: "project",
+      },
+      {
+        title: "Global",
+        description: "~/.claude/skills/<name>/evals.json",
+        value: "global",
+      },
+    ],
+    initial: 0,
+  });
+
+  if (!locationResponse.location) {
+    console.log();
+    dim("Cancelled.");
+    console.log();
+    return;
+  }
+
+  const location: "current" | "project" | "global" = locationResponse.location;
+
+  let skillName = pack.slug;
+
+  if (location === "project" || location === "global") {
+    const skillsDir =
+      location === "project"
+        ? path.resolve(process.cwd(), ".claude", "skills")
+        : path.resolve(os.homedir(), ".claude", "skills");
+
+    const existing = await detectSkills(skillsDir);
+
+    const baseChoices = existing.map((s) => ({
+      title: s.name,
+      description: `${s.evalCount} eval${s.evalCount !== 1 ? "s" : ""}`,
+      value: s.name,
+    }));
+
+    const pickResponse = await prompts({
+      type: "autocomplete",
+      name: "skill",
+      message:
+        existing.length > 0
+          ? `Skill folder (${existing.length} found — type to filter or create new):`
+          : "Skill folder:",
+      choices: baseChoices,
+      initial: pack.slug,
+      suggest: async (input: string, choices: any[]) => {
+        const term = (input || "").toLowerCase();
+        const filtered = choices.filter((c) => c.title.toLowerCase().includes(term));
+        const exactMatch = choices.find((c) => c.title === (input || pack.slug));
+        if (!exactMatch) {
+          filtered.push({ title: input || pack.slug, value: input || pack.slug });
+        }
+        return filtered;
+      },
+    });
+
+    if (!pickResponse.skill) {
+      console.log();
+      dim("Cancelled.");
+      console.log();
+      return;
+    }
+
+    skillName = (pickResponse.skill as string).trim();
+  }
+
+  let targetDir: string;
+  if (location === "current") {
+    targetDir = path.resolve(process.cwd(), "evals", pack.slug);
+  } else if (location === "project") {
+    targetDir = path.resolve(process.cwd(), ".claude", "skills", skillName);
+  } else {
+    targetDir = path.resolve(os.homedir(), ".claude", "skills", skillName);
+  }
+
   const targetFile = path.join(targetDir, "evals.json");
 
+  // Check for existing evals.json and ask append vs override
+  let writeMode: "override" | "append" = "override";
+  let existingEvals: { prompt: string; expectations: string[] }[] = [];
+
+  try {
+    const existing = await fs.readFile(targetFile, "utf-8");
+    const parsed = JSON.parse(existing);
+    existingEvals = parsed.evals ?? [];
+
+    if (existingEvals.length > 0) {
+      const conflictResponse = await prompts({
+        type: "select",
+        name: "mode",
+        message: `Found ${existingEvals.length} existing eval${existingEvals.length !== 1 ? "s" : ""} in ${skillName}. What do you want to do?`,
+        choices: [
+          {
+            title: "Append",
+            description: `Add ${pack.evals.length} new eval${pack.evals.length !== 1 ? "s" : ""} to the existing ${existingEvals.length}`,
+            value: "append",
+          },
+          {
+            title: "Override",
+            description: "Replace all existing evals with the pulled set",
+            value: "override",
+          },
+        ],
+        initial: 0,
+      });
+
+      if (!conflictResponse.mode) {
+        console.log();
+        dim("Cancelled.");
+        console.log();
+        return;
+      }
+
+      writeMode = conflictResponse.mode;
+    }
+  } catch {
+    // No existing file — fresh write
+  }
+
+  const evalsToWrite =
+    writeMode === "append" ? [...existingEvals, ...pack.evals] : pack.evals;
+
   try {
     await fs.mkdir(targetDir, { recursive: true });
 
@@ -61,21 +219,26 @@ export async function pull(slug: string): Promise<void> {
       domain: pack.domain,
       author: pack.author,
       tags: pack.tags,
-      evals: pack.evals,
+      evals: evalsToWrite,
     };
 
     await fs.writeFile(targetFile, JSON.stringify(output, null, 2) + "\n");
 
+    const action = writeMode === "append" ? "Appended" : "Wrote";
+    console.log();
     success(`Pulled ${pack.displayName} v${pack.version}`);
-    success(`Wrote ${pack.evals.length} eval${pack.evals.length !== 1 ? "s" : ""} to evals/${slug}/evals.json`);
+    success(
+      `${action} ${pack.evals.length} eval${pack.evals.length !== 1 ? "s" : ""}` +
+        (writeMode === "append" ? ` (${evalsToWrite.length} total)` : "") +
+        ` to ${targetFile}`
+    );
     console.log();
     dim(`Author:   ${pack.author}`);
     dim(`Domain:   ${pack.domain}`);
-    dim(`Location: ${targetFile}`);
-    dim(`To validate: evalify validate evals/${slug}`);
+    dim(`To validate: evalify-cli validate ${targetDir}`);
   } catch (err) {
     error(`Failed to write file: ${(err as Error).message}`);
   }
 
   console.log();
-}
+}

package/src/commands/search.ts

@@ -18,6 +18,6 @@ export async function search(query: string): Promise<void> {
   table(results);
   console.log();
   dim(`Showing placeholder results — registry search not yet connected`);
-  dim(`Use: evalify pull <slug> to download criteria`);
+  dim(`Use: evalify-cli pull <slug> to download criteria`);
   console.log();
 }

package/src/format.ts

@@ -1,9 +1,10 @@
 import chalk from "chalk";
 
-export const VERSION = "0.1.0";
+declare const __PKG_VERSION__: string;
+export const VERSION = __PKG_VERSION__;
 
 export function header(): void {
-  console.log(chalk.bold.cyan(`\nevalify`) + chalk.dim(` v${VERSION}\n`));
+  console.log(chalk.bold.cyan(`\nevalify-cli`) + chalk.dim(` v${VERSION}\n`));
 }
 
 export function success(msg: string): void {

package/src/index.ts

@@ -4,16 +4,31 @@ import { Command } from "commander";
 import { VERSION } from "./format.js";
 import { pull } from "./commands/pull.js";
 import { publish } from "./commands/publish.js";
+import { login, logout } from "./commands/login.js";
 import { search } from "./commands/search.js";
 import { validate } from "./commands/validate.js";
 
 const program = new Command();
 
 program
-  .name("evalify")
+  .name("evalify-cli")
   .description("CLI tool for the Evalify eval criteria registry")
   .version(VERSION);
 
+program
+  .command("login")
+  .description("Authenticate with the Evalify registry")
+  .action(async () => {
+    await login();
+  });
+
+program
+  .command("logout")
+  .description("Remove saved credentials")
+  .action(async () => {
+    await logout();
+  });
+
 program
   .command("pull <slug>")
   .description("Download eval criteria from the registry")
@@ -42,4 +57,4 @@ program
     await validate(targetPath);
   });
 
-program.parse();
+program.parseAsync().then(() => process.exit(0));

package/tsup.config.ts

@@ -1,6 +1,10 @@
 import { defineConfig } from "tsup";
+import { readFileSync } from "node:fs";
+
+const pkg = JSON.parse(readFileSync("./package.json", "utf-8")) as { version: string };
 
 export default defineConfig({
+  define: { __PKG_VERSION__: JSON.stringify(pkg.version) },
   entry: ["src/index.ts"],
   format: ["cjs"],
   platform: "node",