crw-mcp 0.17.0 → 0.18.1

package/bin/agents.js

@@ -0,0 +1,149 @@
+// Shared agent registry + installers for `crw-mcp init` (skill only) and
+// `crw-mcp install` (skill + MCP server). MCP config shapes are per-agent and
+// were verified against each tool's official docs/source — DO NOT "normalize"
+// them: Codex is TOML with `mcp_servers`, OpenCode uses `mcp`/`type:local`/
+// `command:[]`/`environment`, the rest are JSON `mcpServers`.
+
+const fs = require("fs");
+const path = require("path");
+const os = require("os");
+const { spawnSync } = require("child_process");
+
+const home = os.homedir();
+
+// kind: how the MCP server is registered.
+//  - "claude-cli": shell out to `claude mcp add-json` (handles ~/.claude.json)
+//  - "json":  merge file[topKey].crw = { [type], command, env }
+//  - "toml-codex": append [mcp_servers.crw] to ~/.codex/config.toml
+//  - "opencode": merge file.mcp.crw = { type:local, command:[], environment }
+const AGENTS = [
+  { name: "Claude Code", flag: "--claude-code", configDir: ".claude",
+    mcp: { kind: "claude-cli" } },
+  { name: "Cursor", flag: "--cursor", configDir: ".cursor",
+    mcp: { kind: "json", file: ".cursor/mcp.json", withType: true } },
+  { name: "Gemini CLI", flag: "--gemini-cli", configDir: ".gemini",
+    mcp: { kind: "json", file: ".gemini/settings.json", withType: false } },
+  { name: "Codex", flag: "--codex", configDir: ".codex",
+    mcp: { kind: "toml-codex", file: ".codex/config.toml" } },
+  { name: "OpenCode", flag: "--opencode", configDir: ".opencode",
+    mcp: { kind: "opencode", file: path.join(".config", "opencode", "opencode.json") } },
+  { name: "Windsurf", flag: "--windsurf", configDir: ".codeium",
+    mcp: { kind: "json", file: path.join(".codeium", "windsurf", "mcp_config.json"), withType: false } },
+];
+
+function detectAgents(args) {
+  const has = (f) => args.includes(f);
+  const explicit = AGENTS.filter((a) => has(a.flag));
+  if (explicit.length) return explicit;
+  // --all or no flag → every agent whose config dir exists.
+  return AGENTS.filter((a) => fs.existsSync(path.join(home, a.configDir)));
+}
+
+function getApiKey(args) {
+  const i = args.indexOf("--api-key");
+  return i !== -1 && args[i + 1] ? args[i + 1] : process.env.CRW_API_KEY || null;
+}
+
+function readSkill() {
+  return fs.readFileSync(path.join(__dirname, "..", "skills", "SKILL.md"), "utf-8");
+}
+
+/** Write the SKILL.md into the agent's skills/crw/ dir. Returns the path. */
+function installSkill(agent, skillContent) {
+  const dir = path.join(home, agent.configDir, "skills", "crw");
+  fs.mkdirSync(dir, { recursive: true });
+  const file = path.join(dir, "SKILL.md");
+  fs.writeFileSync(file, skillContent, "utf-8");
+  return file;
+}
+
+function readJson(file) {
+  try {
+    return JSON.parse(fs.readFileSync(file, "utf-8"));
+  } catch {
+    return {};
+  }
+}
+
+function writeJson(file, obj) {
+  fs.mkdirSync(path.dirname(file), { recursive: true });
+  fs.writeFileSync(file, JSON.stringify(obj, null, 2) + "\n", "utf-8");
+}
+
+/**
+ * Register the crw MCP server for one agent. `env` is {} for local mode (the
+ * embedded binary needs no config) or {CRW_API_KEY, CRW_API_URL} for cloud.
+ * Returns a short human label of what was done.
+ */
+function installMcp(agent, env) {
+  const m = agent.mcp;
+  const hasEnv = Object.keys(env).length > 0;
+
+  if (m.kind === "claude-cli") {
+    // add-json avoids the `-e` variadic-name pitfall and merges ~/.claude.json
+    // (user scope) correctly. Idempotent-ish: remove a stale entry first.
+    const server = { command: "crw-mcp", ...(hasEnv ? { env } : {}) };
+    spawnSync("claude", ["mcp", "remove", "--scope", "user", "crw"], { stdio: "ignore" });
+    const r = spawnSync(
+      "claude",
+      ["mcp", "add-json", "--scope", "user", "crw", JSON.stringify(server)],
+      { stdio: "ignore" },
+    );
+    if (r.error || r.status !== 0) {
+      return "MCP: run `claude mcp add-json --scope user crw '" + JSON.stringify(server) + "'` (claude CLI not found)";
+    }
+    return "MCP via claude CLI (user scope)";
+  }
+
+  if (m.kind === "json") {
+    const file = path.join(home, m.file);
+    const cfg = readJson(file);
+    cfg.mcpServers = cfg.mcpServers || {};
+    cfg.mcpServers.crw = {
+      ...(m.withType ? { type: "stdio" } : {}),
+      command: "crw-mcp",
+      ...(hasEnv ? { env } : {}),
+    };
+    writeJson(file, cfg);
+    return `MCP → ${m.file}`;
+  }
+
+  if (m.kind === "opencode") {
+    const file = path.join(home, m.file);
+    const cfg = readJson(file);
+    if (!cfg.$schema) cfg.$schema = "https://opencode.ai/config.json";
+    cfg.mcp = cfg.mcp || {};
+    cfg.mcp.crw = {
+      type: "local",
+      command: ["crw-mcp"],
+      ...(hasEnv ? { environment: env } : {}),
+    };
+    writeJson(file, cfg);
+    return `MCP → ${m.file}`;
+  }
+
+  if (m.kind === "toml-codex") {
+    const file = path.join(home, m.file);
+    let toml = "";
+    try {
+      toml = fs.readFileSync(file, "utf-8");
+    } catch {
+      /* new file */
+    }
+    if (/\[mcp_servers\.crw\]/.test(toml)) {
+      return `MCP already in ${m.file} (left as-is)`;
+    }
+    const envLines = hasEnv
+      ? "\n[mcp_servers.crw.env]\n" +
+        Object.entries(env).map(([k, v]) => `${k} = "${v}"`).join("\n") + "\n"
+      : "";
+    const block = `\n[mcp_servers.crw]\ncommand = "crw-mcp"\n${envLines}`;
+    fs.mkdirSync(path.dirname(file), { recursive: true });
+    fs.appendFileSync(file, (toml && !toml.endsWith("\n") ? "\n" : "") + block, "utf-8");
+    return `MCP → ${m.file}`;
+  }
+
+  return "MCP: unsupported";
+}
+
+module.exports = { AGENTS, detectAgents, getApiKey, readSkill, installSkill, installMcp, home };

package/bin/crw-mcp.js

@@ -1,10 +1,15 @@
 #!/usr/bin/env node
 
-// Handle `init` subcommand before delegating to the Rust binary
+// Handle install/init subcommands before delegating to the Rust binary.
+// `init` = skill only; `install` = skill + MCP server.
 if (process.argv[2] === "init") {
   require("./init.js");
   process.exit(0);
 }
+if (process.argv[2] === "install") {
+  require("./install.js");
+  process.exit(0);
+}
 
 const { spawnSync } = require("child_process");
 const fs = require("fs");

package/bin/init.js

@@ -1,137 +1,78 @@
 #!/usr/bin/env node
 
-const fs = require("fs");
-const path = require("path");
-const os = require("os");
+// `crw-mcp init [--<agent>]` — installs the crw SKILL (SKILL.md) only.
+// For the skill AND the MCP server in one shot, use `crw-mcp install`.
+
+const {
+  AGENTS,
+  detectAgents,
+  getApiKey,
+  readSkill,
+  installSkill,
+  home,
+} = require("./agents.js");
 
-const AGENTS = [
-  { name: "Claude Code", dir: ".claude", flag: "--claude-code" },
-  { name: "Cursor", dir: ".cursor", flag: "--cursor" },
-  { name: "Gemini CLI", dir: ".gemini", flag: "--gemini-cli" },
-  { name: "Codex", dir: ".codex", flag: "--codex" },
-  { name: "OpenCode", dir: ".opencode", flag: "--opencode" },
-  { name: "Windsurf", dir: ".windsurf", flag: "--windsurf" },
-];
-
-const home = os.homedir();
 const args = process.argv.slice(2);
 
-function hasFlag(flag) {
-  return args.includes(flag);
-}
-
-function getApiKey() {
-  const idx = args.indexOf("--api-key");
-  return idx !== -1 && args[idx + 1] ? args[idx + 1] : null;
-}
-
-function readSkillFile() {
-  const skillPath = path.join(__dirname, "..", "skills", "SKILL.md");
-  return fs.readFileSync(skillPath, "utf-8");
-}
-
-function detectAgents() {
-  const all = hasFlag("--all");
-  const specificFlags = AGENTS.filter((a) => hasFlag(a.flag));
-
-  if (!all && specificFlags.length === 0) {
-    // Auto-detect: install to all agents whose config dirs exist
-    return AGENTS.filter((a) =>
-      fs.existsSync(path.join(home, a.dir))
-    );
-  }
-
-  if (all) {
-    return AGENTS.filter((a) =>
-      fs.existsSync(path.join(home, a.dir))
-    );
-  }
-
-  return specificFlags;
-}
-
-function deploy(agents, skillContent) {
-  const installed = [];
-
-  for (const agent of agents) {
-    const targetDir = path.join(home, agent.dir, "skills", "crw");
-    const targetFile = path.join(targetDir, "SKILL.md");
-
-    try {
-      fs.mkdirSync(targetDir, { recursive: true });
-      fs.writeFileSync(targetFile, skillContent, "utf-8");
-      installed.push({ name: agent.name, path: targetFile });
-    } catch (err) {
-      console.error(`  Failed to install for ${agent.name}: ${err.message}`);
-    }
-  }
-
-  return installed;
-}
-
-function main() {
-  if (hasFlag("--help") || hasFlag("-h")) {
-    console.log(`
-crw-mcp init — Install CRW agent skill to your AI coding agents
+if (args.includes("--help") || args.includes("-h")) {
+  console.log(`
+crw-mcp init — Install the CRW agent SKILL (skill only, no MCP server)
 
 Usage:
-  npx crw-mcp@latest init [options]
+  npx crw-mcp@latest init [options]      # skill only
+  npx crw-mcp@latest install [options]   # skill + MCP server
 
 Options:
   --all            Install to all detected agents
-  --claude-code    Install to Claude Code only
-  --cursor         Install to Cursor only
-  --gemini-cli     Install to Gemini CLI only
-  --codex          Install to Codex only
-  --opencode       Install to OpenCode only
-  --windsurf       Install to Windsurf only
-  --api-key <key>  Set your fastcrw.com API key
-  -h, --help       Show this help message
+  --claude-code    Claude Code      --codex      Codex
+  --cursor         Cursor           --opencode   OpenCode
+  --gemini-cli     Gemini CLI       --windsurf   Windsurf
+  --api-key <key>  Your fastcrw.com API key
+  -h, --help       Show this help
 
-Without flags, auto-detects installed agents and installs to all of them.
+Without flags, auto-detects installed agents.
 `);
-    process.exit(0);
-  }
-
-  const skillContent = readSkillFile();
-  const agents = detectAgents();
-
-  if (agents.length === 0) {
-    console.log("No supported AI agents detected.");
-    console.log("Supported agents: " + AGENTS.map((a) => a.name).join(", "));
-    console.log("\nManually install by copying the skill file to your agent's skills directory.");
-    process.exit(1);
-  }
-
-  const installed = deploy(agents, skillContent);
+  process.exit(0);
+}
 
-  if (installed.length === 0) {
-    console.log("Failed to install to any agent.");
-    process.exit(1);
-  }
+const skill = readSkill();
+const agents = detectAgents(args);
 
-  console.log("crw skill installed:\n");
-  const maxName = Math.max(...installed.map((i) => i.name.length));
-  for (const i of installed) {
-    console.log(`  ${i.name.padEnd(maxName + 2)} ${i.path}`);
-  }
+if (agents.length === 0) {
+  console.log("No supported AI agents detected.");
+  console.log("Supported: " + AGENTS.map((a) => a.name).join(", "));
+  process.exit(1);
+}
 
-  const apiKey = getApiKey();
-  if (apiKey) {
-    console.log(`\nAPI key configured: ${apiKey.slice(0, 6)}...`);
-    console.log("Set it in your environment:");
-    console.log(`  export CRW_API_KEY=${apiKey}`);
-  } else {
-    console.log("\nCloud mode (fastcrw.com):");
-    console.log("  500 free one-time credits, managed infra — https://fastcrw.com");
-    console.log("  export CRW_API_KEY=crw_live_xxx");
-    console.log("  export CRW_API_URL=https://api.fastcrw.com");
-    console.log("  Terms of Service: https://fastcrw.com/terms");
+const installed = [];
+for (const agent of agents) {
+  try {
+    installed.push({ name: agent.name, path: installSkill(agent, skill) });
+  } catch (err) {
+    console.error(`  Failed for ${agent.name}: ${err.message}`);
   }
+}
+if (installed.length === 0) {
+  console.log("Failed to install to any agent.");
+  process.exit(1);
+}
 
-  console.log("\nLocal mode (free, same binary, fully capable):");
-  console.log("  npx crw-mcp");
-  console.log("\nDocs: https://fastcrw.com/docs");
+console.log("crw SKILL installed (skill only — this does NOT add the MCP server):\n");
+const maxName = Math.max(...installed.map((i) => i.name.length));
+for (const i of installed) {
+  console.log(`  ${i.name.padEnd(maxName + 2)} ${i.path.replace(home, "~")}`);
 }
 
-main();
+console.log("\nWant the MCP tools (crw_scrape / crawl / map / search) too?");
+console.log("  npx crw-mcp@latest install        # same agents, skill + MCP server");
+
+const apiKey = getApiKey(args);
+if (apiKey) {
+  console.log(`\nAPI key set: ${apiKey.slice(0, 10)}… — export it for cloud mode:`);
+  console.log(`  export CRW_API_KEY=${apiKey}`);
+  console.log("  export CRW_API_URL=https://api.fastcrw.com");
+} else {
+  console.log("\nThe skill works in local mode out of the box (free, embedded).");
+  console.log("Cloud mode (managed, 500 free credits): https://fastcrw.com");
+}
+console.log("\nDocs: https://fastcrw.com/docs");

package/bin/install.js

@@ -0,0 +1,90 @@
+#!/usr/bin/env node
+
+// `crw-mcp install [--<agent>] [--api-key <key>] [--api-url <url>]`
+// Installs BOTH the crw skill AND the crw MCP server into the target agents.
+// (`crw-mcp init` installs the skill only.) Without an agent flag it targets
+// every detected agent; with no API key it wires local mode (free, embedded).
+
+const {
+  AGENTS,
+  detectAgents,
+  getApiKey,
+  readSkill,
+  installSkill,
+  installMcp,
+  home,
+} = require("./agents.js");
+
+const args = process.argv.slice(2);
+
+function argValue(flag) {
+  const i = args.indexOf(flag);
+  return i !== -1 && args[i + 1] ? args[i + 1] : null;
+}
+
+if (args.includes("--help") || args.includes("-h")) {
+  console.log(`
+crw-mcp install — Install the CRW skill AND MCP server into your AI agents
+
+Usage:
+  npx crw-mcp@latest install [options]
+
+Options:
+  --all            Install to all detected agents
+  --claude-code    Claude Code      --codex      Codex
+  --cursor         Cursor           --opencode   OpenCode
+  --gemini-cli     Gemini CLI       --windsurf   Windsurf
+  --api-key <key>  fastcrw.com API key → cloud mode (else local/embedded)
+  --api-url <url>  API base (default https://api.fastcrw.com when a key is set)
+  -h, --help       Show this help
+
+Without flags, auto-detects installed agents. Installs the skill (SKILL.md) and
+registers the "crw" MCP server. Run \`crw-mcp init\` for the skill only.
+`);
+  process.exit(0);
+}
+
+const apiKey = getApiKey(args);
+const apiUrl = argValue("--api-url") || (apiKey ? "https://api.fastcrw.com" : null);
+const env = apiKey ? { CRW_API_KEY: apiKey, CRW_API_URL: apiUrl } : {};
+
+const agents = detectAgents(args);
+if (agents.length === 0) {
+  console.log("No supported AI agents detected.");
+  console.log("Supported: " + AGENTS.map((a) => a.name).join(", "));
+  console.log("\nPass an agent flag (e.g. --claude-code) to install anyway.");
+  process.exit(1);
+}
+
+const skill = readSkill();
+const results = [];
+for (const agent of agents) {
+  try {
+    const skillPath = installSkill(agent, skill);
+    const mcpLabel = installMcp(agent, env);
+    results.push({ name: agent.name, skillPath, mcpLabel });
+  } catch (err) {
+    console.error(`  ${agent.name}: ${err.message}`);
+  }
+}
+
+if (results.length === 0) {
+  console.log("Failed to install to any agent.");
+  process.exit(1);
+}
+
+console.log("crw installed — skill + MCP:\n");
+for (const r of results) {
+  console.log(`  ${r.name}`);
+  console.log(`    skill: ${r.skillPath.replace(home, "~")}`);
+  console.log(`    ${r.mcpLabel.replace(home, "~")}`);
+}
+
+if (apiKey) {
+  console.log(`\nMode: cloud — CRW_API_KEY ${apiKey.slice(0, 10)}… → ${apiUrl}`);
+} else {
+  console.log("\nMode: local (free, embedded binary — no key needed).");
+  console.log("  Cloud mode: re-run with --api-key crw_live_… (500 free credits at https://fastcrw.com).");
+}
+console.log("\nRestart your agent to load the MCP server.");
+console.log("Docs: https://fastcrw.com/docs");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crw-mcp",
-  "version": "0.17.0",
+  "version": "0.18.1",
   "description": "MCP server for CRW web scraper — scrape, crawl, map, search, and PDF-parse tools for AI agents",
   "license": "AGPL-3.0",
   "homepage": "https://github.com/us/crw",
@@ -27,12 +27,14 @@
   "files": [
     "bin/crw-mcp.js",
     "bin/init.js",
+    "bin/install.js",
+    "bin/agents.js",
     "skills/SKILL.md"
   ],
   "optionalDependencies": {
-    "crw-mcp-darwin-x64": "0.17.0",
-    "crw-mcp-darwin-arm64": "0.17.0",
-    "crw-mcp-linux-x64": "0.17.0",
-    "crw-mcp-linux-arm64": "0.17.0"
+    "crw-mcp-darwin-x64": "0.18.1",
+    "crw-mcp-darwin-arm64": "0.18.1",
+    "crw-mcp-linux-x64": "0.18.1",
+    "crw-mcp-linux-arm64": "0.18.1"
   }
 }