evalify-cli 0.1.2 → 0.1.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "evalify-cli",
-  "version": "0.1.2",
+  "version": "0.1.3",
   "description": "CLI tool for the Evalify eval criteria registry",
   "homepage": "https://evalify.sh",
   "repository": "https://github.com/AppVerse-cc/evalify",
@@ -14,11 +14,13 @@
   },
   "dependencies": {
     "chalk": "^5.4.1",
-    "commander": "^12.1.0"
+    "commander": "^12.1.0",
+    "prompts": "^2.4.2"
   },
   "devDependencies": {
     "@evalify/frameworks": "workspace:*",
     "@types/node": "^22.0.0",
+    "@types/prompts": "^2.4.9",
     "tsup": "^8.5.1",
     "typescript": "^5.7.0"
   }

package/src/commands/pull.ts

@@ -1,9 +1,34 @@
 import path from "node:path";
+import os from "node:os";
 import fs from "node:fs/promises";
+import prompts from "prompts";
 import { header, success, info, dim, error } from "../format.js";
 
 const REGISTRY_URL = "https://evalify.sh/api/registry";
 
+async function detectSkills(skillsDir: string): Promise<{ name: string; evalCount: number }[]> {
+  try {
+    const entries = await fs.readdir(skillsDir, { withFileTypes: true });
+    const skills: { name: string; evalCount: number }[] = [];
+
+    for (const entry of entries) {
+      if (!entry.isDirectory()) continue;
+      const evalsFile = path.join(skillsDir, entry.name, "evals.json");
+      try {
+        const content = await fs.readFile(evalsFile, "utf-8");
+        const parsed = JSON.parse(content);
+        skills.push({ name: entry.name, evalCount: (parsed.evals ?? []).length });
+      } catch {
+        // Folder exists but no valid evals.json — skip
+      }
+    }
+
+    return skills;
+  } catch {
+    return [];
+  }
+}
+
 export async function pull(slug: string): Promise<void> {
   header();
 
@@ -47,9 +72,142 @@ export async function pull(slug: string): Promise<void> {
     return;
   }
 
-  const targetDir = path.resolve(process.cwd(), "evals", slug);
+  // Ask install location
+  const locationResponse = await prompts({
+    type: "select",
+    name: "location",
+    message: "Where do you want to install?",
+    choices: [
+      {
+        title: "Current directory",
+        description: `evals/${pack.slug}/evals.json`,
+        value: "current",
+      },
+      {
+        title: "Project",
+        description: ".claude/skills/<name>/evals.json",
+        value: "project",
+      },
+      {
+        title: "Global",
+        description: "~/.claude/skills/<name>/evals.json",
+        value: "global",
+      },
+    ],
+    initial: 0,
+  });
+
+  if (!locationResponse.location) {
+    console.log();
+    dim("Cancelled.");
+    console.log();
+    return;
+  }
+
+  const location: "current" | "project" | "global" = locationResponse.location;
+
+  let skillName = pack.slug;
+
+  if (location === "project" || location === "global") {
+    const skillsDir =
+      location === "project"
+        ? path.resolve(process.cwd(), ".claude", "skills")
+        : path.resolve(os.homedir(), ".claude", "skills");
+
+    const existing = await detectSkills(skillsDir);
+
+    const baseChoices = existing.map((s) => ({
+      title: s.name,
+      description: `${s.evalCount} eval${s.evalCount !== 1 ? "s" : ""}`,
+      value: s.name,
+    }));
+
+    const pickResponse = await prompts({
+      type: "autocomplete",
+      name: "skill",
+      message:
+        existing.length > 0
+          ? `Skill folder (${existing.length} found — type to filter or create new):`
+          : "Skill folder:",
+      choices: baseChoices,
+      initial: pack.slug,
+      suggest: async (input: string, choices: any[]) => {
+        const term = (input || "").toLowerCase();
+        const filtered = choices.filter((c) => c.title.toLowerCase().includes(term));
+        const exactMatch = choices.find((c) => c.title === (input || pack.slug));
+        if (!exactMatch) {
+          filtered.push({ title: input || pack.slug, value: input || pack.slug });
+        }
+        return filtered;
+      },
+    });
+
+    if (!pickResponse.skill) {
+      console.log();
+      dim("Cancelled.");
+      console.log();
+      return;
+    }
+
+    skillName = (pickResponse.skill as string).trim();
+  }
+
+  let targetDir: string;
+  if (location === "current") {
+    targetDir = path.resolve(process.cwd(), "evals", pack.slug);
+  } else if (location === "project") {
+    targetDir = path.resolve(process.cwd(), ".claude", "skills", skillName);
+  } else {
+    targetDir = path.resolve(os.homedir(), ".claude", "skills", skillName);
+  }
+
   const targetFile = path.join(targetDir, "evals.json");
 
+  // Check for existing evals.json and ask append vs override
+  let writeMode: "override" | "append" = "override";
+  let existingEvals: { prompt: string; expectations: string[] }[] = [];
+
+  try {
+    const existing = await fs.readFile(targetFile, "utf-8");
+    const parsed = JSON.parse(existing);
+    existingEvals = parsed.evals ?? [];
+
+    if (existingEvals.length > 0) {
+      const conflictResponse = await prompts({
+        type: "select",
+        name: "mode",
+        message: `Found ${existingEvals.length} existing eval${existingEvals.length !== 1 ? "s" : ""} in ${skillName}. What do you want to do?`,
+        choices: [
+          {
+            title: "Append",
+            description: `Add ${pack.evals.length} new eval${pack.evals.length !== 1 ? "s" : ""} to the existing ${existingEvals.length}`,
+            value: "append",
+          },
+          {
+            title: "Override",
+            description: "Replace all existing evals with the pulled set",
+            value: "override",
+          },
+        ],
+        initial: 0,
+      });
+
+      if (!conflictResponse.mode) {
+        console.log();
+        dim("Cancelled.");
+        console.log();
+        return;
+      }
+
+      writeMode = conflictResponse.mode;
+    }
+  } catch {
+    // No existing file — fresh write
+  }
+
+  const evalsToWrite =
+    writeMode === "append" ? [...existingEvals, ...pack.evals] : pack.evals;
+
   try {
     await fs.mkdir(targetDir, { recursive: true });
 
@@ -61,21 +219,26 @@ export async function pull(slug: string): Promise<void> {
       domain: pack.domain,
       author: pack.author,
       tags: pack.tags,
-      evals: pack.evals,
+      evals: evalsToWrite,
     };
 
     await fs.writeFile(targetFile, JSON.stringify(output, null, 2) + "\n");
 
+    const action = writeMode === "append" ? "Appended" : "Wrote";
+    console.log();
     success(`Pulled ${pack.displayName} v${pack.version}`);
-    success(`Wrote ${pack.evals.length} eval${pack.evals.length !== 1 ? "s" : ""} to evals/${slug}/evals.json`);
+    success(
+      `${action} ${pack.evals.length} eval${pack.evals.length !== 1 ? "s" : ""}` +
+        (writeMode === "append" ? ` (${evalsToWrite.length} total)` : "") +
+        ` to ${targetFile}`
+    );
     console.log();
     dim(`Author:   ${pack.author}`);
     dim(`Domain:   ${pack.domain}`);
-    dim(`Location: ${targetFile}`);
-    dim(`To validate: evalify validate evals/${slug}`);
+    dim(`To validate: evalify validate ${targetDir}`);
   } catch (err) {
     error(`Failed to write file: ${(err as Error).message}`);
   }
 
   console.log();
-}
+}