meow-skills-fetch-cli 0.2.1

package/README.md

@@ -0,0 +1,51 @@
+# meow-skills-fetch-cli
+
+Download matching `SKILL.md` files from the terminal.
+
+## Install
+
+```bash
+npm install -g meow-skills-fetch-cli
+```
+
+Or run without installing:
+
+```bash
+npx meow-skills-fetch-cli rust
+```
+
+## Usage
+
+```bash
+meow rust
+meow cpp --limit 5
+meow kotlin --output ./downloads/kotlin
+meow rust --limit 20 --concurrency 8
+```
+
+## Output
+
+By default, files are saved in:
+
+```bash
+./<query>_skills
+```
+
+Example:
+
+```bash
+meow rust
+```
+
+This creates:
+
+```bash
+./rust_skills
+```
+
+## Options
+
+- `--limit` limits how many matching skills are downloaded
+- `--output` sets a custom output folder
+- `--concurrency` controls parallel downloads
+- `--help` shows help

package/bin/meow.js

@@ -0,0 +1,8 @@
+#!/usr/bin/env node
+
+import { main } from "../src/cli.js";
+
+main().catch((error) => {
+  console.error(`Error: ${error.message}`);
+  process.exitCode = 1;
+});

package/package.json

@@ -0,0 +1,31 @@
+{
+  "name": "meow-skills-fetch-cli",
+  "version": "0.2.1",
+  "description": "CLI to search public skill indexes and download matching SKILL.md files with a terminal dashboard.",
+  "type": "module",
+  "bin": {
+    "meow": "./bin/meow.js"
+  },
+  "files": [
+    "bin",
+    "src",
+    "README.md"
+  ],
+  "scripts": {
+    "start": "node ./bin/meow.js",
+    "build:bin": "pkg . --targets node20-macos-arm64,node20-linux-x64,node20-win-x64 --out-path dist"
+  },
+  "keywords": [
+    "cli",
+    "skills",
+    "markdown",
+    "scraper"
+  ],
+  "license": "MIT",
+  "engines": {
+    "node": ">=20"
+  },
+  "devDependencies": {
+    "pkg": "^5.8.1"
+  }
+}

package/src/cli.js

@@ -0,0 +1,574 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import process from "node:process";
+
+const COLOR = {
+  black: "\u001b[30m",
+  green: "\u001b[92m",
+  yellow: "\u001b[93m",
+  red: "\u001b[91m",
+  cyan: "\u001b[96m",
+  white: "\u001b[97m",
+  reset: "\u001b[0m",
+  bold: "\u001b[1m",
+  dim: "\u001b[2m"
+};
+
+const FRAMES_NORMAL = [
+  ["o", "o"],
+  ["^", "^"],
+  ["-", "-"],
+  ["*", "*"],
+  ["@", "@"],
+  ["o", "O"],
+  [">", "<"],
+  ["O", "O"]
+];
+
+const FRAMES_SAD = [
+  ["T", "T"],
+  [";", ";"],
+  ["v", "v"],
+  ["x", "x"]
+];
+
+const FETCH_TIMEOUT_MS = 15_000;
+const FETCH_RETRIES = 3;
+const DEFAULT_CONCURRENCY = 8;
+const MAX_CONCURRENCY = 16;
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function xorDecode(values, key) {
+  return values
+    .map((value, index) => String.fromCharCode(value ^ key[index % key.length]))
+    .join("");
+}
+
+function searchEndpoint() {
+  // This only hides the endpoint from simple static greps; runtime traffic still reveals it.
+  return xorDecode(
+    [99, 105, 115, 103, 112, 41, 36, 50, 116, 124, 106, 127, 103, 110, 41, 100, 107, 60, 106, 109, 110, 56, 112, 118, 106, 111, 100, 127],
+    [11, 29, 7, 23, 3, 19]
+  );
+}
+
+function githubApiBase() {
+  return xorDecode(
+    [109, 101, 125, 101, 118, 43, 38, 58, 100, 97, 96, 59, 98, 120, 125, 125, 112, 115, 39, 118, 106, 124],
+    [5, 17, 9, 21]
+  );
+}
+
+function githubRawBase() {
+  return xorDecode(
+    [101, 119, 109, 119, 120, 55, 44, 54, 117, 106, 122, 45, 126, 110, 127, 101, 118, 123, 114, 120, 104, 113, 122, 104, 101, 121, 102, 119, 115, 37, 110, 108, 116],
+    [13, 3, 25, 7, 11]
+  );
+}
+
+function pbar(percent, width = 20) {
+  const filled = Math.floor((width * percent) / 100);
+  const color = percent === 100 ? COLOR.green : COLOR.cyan;
+  return `  ${COLOR.green}${"█".repeat(filled)}${COLOR.dim}${"░".repeat(
+    width - filled
+  )}${COLOR.reset} ${COLOR.bold}${color}${String(percent).padStart(3, " ")}%${COLOR.reset}`;
+}
+
+class CatUI {
+  static frameIndex = 0;
+  static sad = false;
+  static catHeight = 5;
+  static enabled = Boolean(process.stdout.isTTY);
+  static hiddenCursor = false;
+  static reserved = false;
+  static linesSinceCat = 0;
+
+  static init() {
+    if (!this.enabled) {
+      return;
+    }
+
+    process.stdout.write("\u001b[?25l");
+    this.hiddenCursor = true;
+    process.stdout.write("\n".repeat(this.catHeight));
+    this.reserved = true;
+    this.linesSinceCat = 0;
+  }
+
+  static cleanup() {
+    if (this.hiddenCursor) {
+      process.stdout.write("\u001b[?25h");
+      this.hiddenCursor = false;
+    }
+
+    this.reserved = false;
+    this.linesSinceCat = 0;
+  }
+
+  static setSad(sad) {
+    this.sad = sad;
+  }
+
+  static draw(status, percent = null) {
+    if (!this.enabled) {
+      return;
+    }
+
+    const pool = this.sad ? FRAMES_SAD : FRAMES_NORMAL;
+    this.frameIndex = (this.frameIndex + 1) % pool.length;
+    const [leftEye, rightEye] = pool[this.frameIndex];
+    const bar = percent === null ? `  ${COLOR.dim}${"░".repeat(20)}${COLOR.reset}   -%` : pbar(percent);
+    const statusColor = COLOR.black;
+
+    const totalUp = this.linesSinceCat + this.catHeight;
+    process.stdout.write(`\u001b[${totalUp}A\r`);
+    process.stdout.write(`\u001b[2K  ${COLOR.cyan}/\\_/\\\\${COLOR.reset}\n`);
+    process.stdout.write(
+      `\u001b[2K ${COLOR.cyan}(${COLOR.reset} ${COLOR.white}${leftEye}${COLOR.cyan}.${COLOR.white}${rightEye}${COLOR.cyan} )${COLOR.reset}  ${statusColor}Meowing: ${status}${COLOR.reset}\n`
+    );
+    process.stdout.write(`\u001b[2K  ${COLOR.cyan}> ^ <${COLOR.reset}\n`);
+    process.stdout.write(`\u001b[2K${bar}\n`);
+    process.stdout.write(`\u001b[2K${COLOR.dim}${"─".repeat(40)}${COLOR.reset}\n`);
+
+    if (this.linesSinceCat > 0) {
+      process.stdout.write(`\u001b[${this.linesSinceCat}B\r`);
+    }
+  }
+
+  static log(color, prefix, message) {
+    const line = `\r\u001b[2K ${color}${prefix}${COLOR.reset} ${message}\n`;
+    process.stdout.write(line);
+    if (this.enabled && this.reserved) {
+      this.linesSinceCat += 1;
+    }
+  }
+
+  static line(message = "") {
+    if (this.enabled && this.reserved) {
+      this.linesSinceCat += 1;
+    }
+
+    process.stdout.write(`\r\u001b[2K ${message}\n`);
+  }
+
+  static info(message) {
+    this.log(COLOR.cyan, "→", message);
+  }
+
+  static error(message) {
+    this.log(COLOR.red, "✕", message);
+  }
+}
+
+function usage() {
+  process.stdout.write(`${COLOR.bold}meow${COLOR.reset} search and download skills\n\n`);
+  process.stdout.write("Usage:\n");
+  process.stdout.write("  meow [query]\n");
+  process.stdout.write("  meow [query] --limit 10\n");
+  process.stdout.write("  meow [query] --output ./my-folder\n");
+  process.stdout.write("  meow [query] --concurrency 8\n\n");
+  process.stdout.write("Options:\n");
+  process.stdout.write("  -o, --output        Output directory (default: <query>_skills in current folder)\n");
+  process.stdout.write(`  -l, --limit         Limit number of skills to fetch\n`);
+  process.stdout.write(
+    `  -c, --concurrency   Parallel downloads (default: ${DEFAULT_CONCURRENCY}, max: ${MAX_CONCURRENCY})\n`
+  );
+  process.stdout.write("  -h, --help          Show help\n");
+}
+
+function parseArgs(argv) {
+  const args = {
+    query: "rust",
+    output: "",
+    limit: 0,
+    help: false,
+    concurrency: DEFAULT_CONCURRENCY
+  };
+  let querySet = false;
+
+  for (let index = 0; index < argv.length; index += 1) {
+    const token = argv[index];
+
+    if (token === "-h" || token === "--help") {
+      args.help = true;
+      continue;
+    }
+
+    if (token === "-o" || token === "--output") {
+      args.output = argv[index + 1] ?? "";
+      if (!args.output || args.output.startsWith("-")) {
+        throw new Error("--output requires a value");
+      }
+      index += 1;
+      continue;
+    }
+
+    if (token === "-l" || token === "--limit") {
+      const rawLimit = argv[index + 1] ?? "";
+      if (!rawLimit || rawLimit.startsWith("-")) {
+        throw new Error("--limit requires a value");
+      }
+      args.limit = Number(rawLimit);
+      index += 1;
+      continue;
+    }
+
+    if (token === "-c" || token === "--concurrency") {
+      const rawConcurrency = argv[index + 1] ?? "";
+      if (!rawConcurrency || rawConcurrency.startsWith("-")) {
+        throw new Error("--concurrency requires a value");
+      }
+      args.concurrency = Number(rawConcurrency);
+      index += 1;
+      continue;
+    }
+
+    if (!token.startsWith("-") && !querySet) {
+      args.query = token;
+      querySet = true;
+    }
+  }
+
+  if (Number.isNaN(args.limit) || args.limit < 0) {
+    throw new Error("--limit must be a positive number");
+  }
+
+  if (
+    Number.isNaN(args.concurrency) ||
+    args.concurrency < 1 ||
+    args.concurrency > MAX_CONCURRENCY
+  ) {
+    throw new Error(`--concurrency must be between 1 and ${MAX_CONCURRENCY}`);
+  }
+
+  return args;
+}
+
+function extractQuery(input) {
+  if (!input.startsWith("http://") && !input.startsWith("https://")) {
+    return input;
+  }
+
+  try {
+    const parsed = new URL(input);
+    return parsed.searchParams.get("q") || "rust";
+  } catch {
+    return input;
+  }
+}
+
+function parseSource(source) {
+  const parts = source.split("/");
+  if (parts.length >= 2) {
+    return {
+      owner: parts.slice(0, -1).join("/"),
+      repo: parts.at(-1)
+    };
+  }
+
+  return {
+    owner: source,
+    repo: ""
+  };
+}
+
+function sanitizeFilename(name) {
+  return name.replace(/[<>:"/\\|?*]+/g, "_").trim() || "unknown";
+}
+
+async function fetchWithRetry(url, accept, { nullable = false } = {}) {
+  for (let attempt = 1; attempt <= FETCH_RETRIES; attempt += 1) {
+    try {
+      const response = await fetch(url, {
+        headers: {
+          "user-agent": "meow-skills-fetch-cli/0.2.0",
+          accept
+        },
+        signal: AbortSignal.timeout(FETCH_TIMEOUT_MS)
+      });
+
+      if (nullable && response.status === 404) {
+        return null;
+      }
+
+      if (!response.ok) {
+        const retryable = response.status >= 500 || response.status === 429;
+        if (!retryable || attempt === FETCH_RETRIES) {
+          if (nullable) {
+            return null;
+          }
+          throw new Error(`Request failed with ${response.status}`);
+        }
+      } else {
+        return response;
+      }
+    } catch (error) {
+      const aborted = error?.name === "TimeoutError" || error?.name === "AbortError";
+      if (attempt === FETCH_RETRIES) {
+        if (nullable) {
+          return null;
+        }
+        throw new Error(aborted ? "request timed out" : error.message);
+      }
+    }
+
+    await sleep(220 * attempt);
+  }
+
+  return null;
+}
+
+async function getJson(url, options) {
+  const response = await fetchWithRetry(url, "application/json", options);
+  if (!response) {
+    return null;
+  }
+  return response.json();
+}
+
+async function getText(url, options) {
+  const response = await fetchWithRetry(url, "text/plain", options);
+  if (!response) {
+    return null;
+  }
+  return response.text();
+}
+
+async function searchSkills(query) {
+  const url = new URL(searchEndpoint());
+  url.searchParams.set("q", query);
+  const payload = await getJson(url);
+  return Array.isArray(payload?.skills) ? payload.skills : [];
+}
+
+const repoTreeCache = new Map();
+
+async function findSkillInTree(owner, repo, skillName, branch = "main") {
+  const cacheKey = `${owner}/${repo}/${branch}`;
+
+  if (!repoTreeCache.has(cacheKey)) {
+    const treeUrl = `${githubApiBase()}/repos/${owner}/${repo}/git/trees/${branch}?recursive=1`;
+    repoTreeCache.set(
+      cacheKey,
+      (async () => {
+        try {
+          const payload = await getJson(treeUrl, { nullable: true });
+          if (!payload?.tree) {
+            return [];
+          }
+          return payload.tree
+            .filter((item) => item.type === "blob" && item.path.endsWith("SKILL.md"))
+            .map((item) => item.path);
+        } catch {
+          return [];
+        }
+      })()
+    );
+  }
+
+  const cached = await repoTreeCache.get(cacheKey);
+  const paths = Array.isArray(cached) ? cached : [];
+  if (paths.length === 0 && branch === "main") {
+    return findSkillInTree(owner, repo, skillName, "master");
+  }
+  const normalized = skillName.toLowerCase();
+
+  for (const filePath of paths) {
+    if (!filePath.toLowerCase().includes(normalized)) {
+      continue;
+    }
+
+    const rawUrl = `${githubRawBase()}/${owner}/${repo}/${branch}/${filePath}`;
+    const content = await getText(rawUrl, { nullable: true });
+    if (content) {
+      return content;
+    }
+  }
+
+  return null;
+}
+
+function candidateSkillUrls(owner, repo, skillName) {
+  const base = `${githubRawBase()}/${owner}/${repo}`;
+  return [
+    `${base}/main/skills/${skillName}/SKILL.md`,
+    `${base}/main/.claude/skills/${skillName}/SKILL.md`,
+    `${base}/main/${skillName}/SKILL.md`,
+    `${base}/HEAD/main/skills/${skillName}/SKILL.md`,
+    `${base}/HEAD/.claude/skills/${skillName}/SKILL.md`,
+    `${base}/HEAD/${skillName}/SKILL.md`,
+    `${base}/master/skills/${skillName}/SKILL.md`,
+    `${base}/master/.claude/skills/${skillName}/SKILL.md`,
+    `${base}/master/${skillName}/SKILL.md`
+  ];
+}
+
+async function getSkillDetails(owner, repo, skillName) {
+  for (const candidate of candidateSkillUrls(owner, repo, skillName)) {
+    const content = await getText(candidate, { nullable: true });
+    if (content) {
+      return content;
+    }
+  }
+
+  return findSkillInTree(owner, repo, skillName);
+}
+
+function createPool(limit) {
+  let active = 0;
+  const queue = [];
+
+  const next = () => {
+    if (active >= limit || queue.length === 0) {
+      return;
+    }
+
+    active += 1;
+    const task = queue.shift();
+    task()
+      .catch(() => {})
+      .finally(() => {
+        active -= 1;
+        next();
+      });
+  };
+
+  return (runner) =>
+    new Promise((resolve, reject) => {
+      queue.push(async () => {
+        try {
+          resolve(await runner());
+        } catch (error) {
+          reject(error);
+        }
+      });
+      next();
+    });
+}
+
+function renderProgress(state, status, sad = false) {
+  const percent = state.total === 0 ? 0 : Math.floor((state.completed / state.total) * 100);
+  CatUI.setSad(sad);
+  CatUI.draw(status, percent);
+}
+
+async function saveSkills(skills, outputDir, concurrency) {
+  const outputPath = path.resolve(outputDir);
+  await fs.mkdir(outputPath, { recursive: true });
+
+  const state = {
+    total: skills.length,
+    completed: 0,
+    saved: 0,
+    skipped: 0,
+    active: 0
+  };
+  const pool = createPool(concurrency);
+
+  renderProgress(state, `Found ${skills.length} skills!`, false);
+  CatUI.info(`Downloading skill content with concurrency=${concurrency}...`);
+
+  const tasks = skills.map((skill, index) =>
+    pool(async () => {
+      const name = skill?.name || "Unknown";
+      const source = skill?.source || "";
+      const { owner, repo } = parseSource(source);
+      let logColor = COLOR.yellow;
+      let logPrefix = "⚠";
+      let logMessage = `Skipped: ${name}`;
+
+      state.active += 1;
+      renderProgress(
+        state,
+        `fetching ${name}... active:${state.active}`,
+        false
+      );
+
+      try {
+        const content = await getSkillDetails(owner, repo, name);
+
+        if (content) {
+          const fileName = `${sanitizeFilename(name)}.md`;
+          await fs.writeFile(path.join(outputPath, fileName), content, "utf8");
+          state.saved += 1;
+          logColor = COLOR.green;
+          logPrefix = "✓";
+          logMessage = `Saved:   ${name}`;
+        } else {
+          state.skipped += 1;
+          logMessage = `Skipped: ${name}`;
+        }
+      } catch (error) {
+        state.skipped += 1;
+        logMessage = `Skipped: ${name} (${error.message})`;
+      } finally {
+        state.active -= 1;
+        state.completed += 1;
+        const doneIndex = state.completed;
+        CatUI.log(logColor, `${logPrefix} [${doneIndex}/${state.total}]`, logMessage);
+        renderProgress(
+          state,
+          state.completed === state.total
+            ? "synced! ✓"
+            : `completed ${state.completed}/${state.total} active:${state.active}`,
+          state.completed > 0 && state.saved === 0
+        );
+      }
+    })
+  );
+
+  await Promise.all(tasks);
+  await sleep(300);
+  CatUI.setSad(false);
+  return { saved: state.saved, skipped: state.skipped, outputPath };
+}
+
+export async function main() {
+  const args = parseArgs(process.argv.slice(2));
+  if (args.help) {
+    usage();
+    return;
+  }
+
+  const query = extractQuery(args.query);
+  const outputDir = args.output || path.join(process.cwd(), `${query}_skills`);
+  let result = null;
+  let failure = null;
+
+  CatUI.init();
+
+  try {
+    renderProgress({ total: 1, completed: 0 }, "Searching...", false);
+
+    const skills = await searchSkills(query);
+    if (!skills.length) {
+      renderProgress({ total: 1, completed: 1 }, "No skills found!", true);
+      await sleep(900);
+      return;
+    }
+
+    const limitedSkills = args.limit > 0 ? skills.slice(0, args.limit) : skills;
+    result = await saveSkills(limitedSkills, outputDir, args.concurrency);
+  } catch (error) {
+    failure = error;
+  }
+
+  if (failure) {
+    CatUI.line(`${COLOR.red}Error: ${failure.message}${COLOR.reset}`);
+    CatUI.cleanup();
+    process.exitCode = 1;
+    return;
+  }
+
+  if (result) {
+    CatUI.line(`${COLOR.bold}${COLOR.green}DONE${COLOR.reset}`);
+    CatUI.line(`  Saved:   ${COLOR.green}${result.saved}${COLOR.reset}`);
+    CatUI.line(`  Skipped: ${COLOR.yellow}${result.skipped}${COLOR.reset}`);
+    CatUI.line(`  Output:  ${result.outputPath}/`);
+  }
+
+  CatUI.cleanup();
+}