crw-mcp 0.16.0 → 0.18.0

package/bin/crw-mcp.js

@@ -1,10 +1,15 @@
 #!/usr/bin/env node
 
-// Handle `init` subcommand before delegating to the Rust binary
+// Handle install/init subcommands before delegating to the Rust binary.
+// `init` = skill only; `install` = skill + MCP server.
 if (process.argv[2] === "init") {
   require("./init.js");
   process.exit(0);
 }
+if (process.argv[2] === "install") {
+  require("./install.js");
+  process.exit(0);
+}
 
 const { spawnSync } = require("child_process");
 const fs = require("fs");

package/bin/init.js

@@ -1,137 +1,78 @@
 #!/usr/bin/env node
 
-const fs = require("fs");
-const path = require("path");
-const os = require("os");
+// `crw-mcp init [--<agent>]` — installs the crw SKILL (SKILL.md) only.
+// For the skill AND the MCP server in one shot, use `crw-mcp install`.
+
+const {
+  AGENTS,
+  detectAgents,
+  getApiKey,
+  readSkill,
+  installSkill,
+  home,
+} = require("./agents.js");
 
-const AGENTS = [
-  { name: "Claude Code", dir: ".claude", flag: "--claude-code" },
-  { name: "Cursor", dir: ".cursor", flag: "--cursor" },
-  { name: "Gemini CLI", dir: ".gemini", flag: "--gemini-cli" },
-  { name: "Codex", dir: ".codex", flag: "--codex" },
-  { name: "OpenCode", dir: ".opencode", flag: "--opencode" },
-  { name: "Windsurf", dir: ".windsurf", flag: "--windsurf" },
-];
-
-const home = os.homedir();
 const args = process.argv.slice(2);
 
-function hasFlag(flag) {
-  return args.includes(flag);
-}
-
-function getApiKey() {
-  const idx = args.indexOf("--api-key");
-  return idx !== -1 && args[idx + 1] ? args[idx + 1] : null;
-}
-
-function readSkillFile() {
-  const skillPath = path.join(__dirname, "..", "skills", "SKILL.md");
-  return fs.readFileSync(skillPath, "utf-8");
-}
-
-function detectAgents() {
-  const all = hasFlag("--all");
-  const specificFlags = AGENTS.filter((a) => hasFlag(a.flag));
-
-  if (!all && specificFlags.length === 0) {
-    // Auto-detect: install to all agents whose config dirs exist
-    return AGENTS.filter((a) =>
-      fs.existsSync(path.join(home, a.dir))
-    );
-  }
-
-  if (all) {
-    return AGENTS.filter((a) =>
-      fs.existsSync(path.join(home, a.dir))
-    );
-  }
-
-  return specificFlags;
-}
-
-function deploy(agents, skillContent) {
-  const installed = [];
-
-  for (const agent of agents) {
-    const targetDir = path.join(home, agent.dir, "skills", "crw");
-    const targetFile = path.join(targetDir, "SKILL.md");
-
-    try {
-      fs.mkdirSync(targetDir, { recursive: true });
-      fs.writeFileSync(targetFile, skillContent, "utf-8");
-      installed.push({ name: agent.name, path: targetFile });
-    } catch (err) {
-      console.error(`  Failed to install for ${agent.name}: ${err.message}`);
-    }
-  }
-
-  return installed;
-}
-
-function main() {
-  if (hasFlag("--help") || hasFlag("-h")) {
-    console.log(`
-crw-mcp init — Install CRW agent skill to your AI coding agents
+if (args.includes("--help") || args.includes("-h")) {
+  console.log(`
+crw-mcp init — Install the CRW agent SKILL (skill only, no MCP server)
 
 Usage:
-  npx crw-mcp@latest init [options]
+  npx crw-mcp@latest init [options]      # skill only
+  npx crw-mcp@latest install [options]   # skill + MCP server
 
 Options:
   --all            Install to all detected agents
-  --claude-code    Install to Claude Code only
-  --cursor         Install to Cursor only
-  --gemini-cli     Install to Gemini CLI only
-  --codex          Install to Codex only
-  --opencode       Install to OpenCode only
-  --windsurf       Install to Windsurf only
-  --api-key <key>  Set your fastcrw.com API key
-  -h, --help       Show this help message
+  --claude-code    Claude Code      --codex      Codex
+  --cursor         Cursor           --opencode   OpenCode
+  --gemini-cli     Gemini CLI       --windsurf   Windsurf
+  --api-key <key>  Your fastcrw.com API key
+  -h, --help       Show this help
 
-Without flags, auto-detects installed agents and installs to all of them.
+Without flags, auto-detects installed agents.
 `);
-    process.exit(0);
-  }
-
-  const skillContent = readSkillFile();
-  const agents = detectAgents();
-
-  if (agents.length === 0) {
-    console.log("No supported AI agents detected.");
-    console.log("Supported agents: " + AGENTS.map((a) => a.name).join(", "));
-    console.log("\nManually install by copying the skill file to your agent's skills directory.");
-    process.exit(1);
-  }
-
-  const installed = deploy(agents, skillContent);
+  process.exit(0);
+}
 
-  if (installed.length === 0) {
-    console.log("Failed to install to any agent.");
-    process.exit(1);
-  }
+const skill = readSkill();
+const agents = detectAgents(args);
 
-  console.log("crw skill installed:\n");
-  const maxName = Math.max(...installed.map((i) => i.name.length));
-  for (const i of installed) {
-    console.log(`  ${i.name.padEnd(maxName + 2)} ${i.path}`);
-  }
+if (agents.length === 0) {
+  console.log("No supported AI agents detected.");
+  console.log("Supported: " + AGENTS.map((a) => a.name).join(", "));
+  process.exit(1);
+}
 
-  const apiKey = getApiKey();
-  if (apiKey) {
-    console.log(`\nAPI key configured: ${apiKey.slice(0, 6)}...`);
-    console.log("Set it in your environment:");
-    console.log(`  export CRW_API_KEY=${apiKey}`);
-  } else {
-    console.log("\nCloud mode (fastcrw.com):");
-    console.log("  500 free one-time credits, managed infra — https://fastcrw.com");
-    console.log("  export CRW_API_KEY=fc-your-key");
-    console.log("  export CRW_API_URL=https://api.fastcrw.com");
-    console.log("  Terms of Service: https://fastcrw.com/terms");
+const installed = [];
+for (const agent of agents) {
+  try {
+    installed.push({ name: agent.name, path: installSkill(agent, skill) });
+  } catch (err) {
+    console.error(`  Failed for ${agent.name}: ${err.message}`);
   }
+}
+if (installed.length === 0) {
+  console.log("Failed to install to any agent.");
+  process.exit(1);
+}
 
-  console.log("\nLocal mode (free, same binary, fully capable):");
-  console.log("  npx crw-mcp");
-  console.log("\nDocs: https://fastcrw.com/docs");
+console.log("crw SKILL installed (skill only — this does NOT add the MCP server):\n");
+const maxName = Math.max(...installed.map((i) => i.name.length));
+for (const i of installed) {
+  console.log(`  ${i.name.padEnd(maxName + 2)} ${i.path.replace(home, "~")}`);
 }
 
-main();
+console.log("\nWant the MCP tools (crw_scrape / crawl / map / search) too?");
+console.log("  npx crw-mcp@latest install        # same agents, skill + MCP server");
+
+const apiKey = getApiKey(args);
+if (apiKey) {
+  console.log(`\nAPI key set: ${apiKey.slice(0, 10)}… — export it for cloud mode:`);
+  console.log(`  export CRW_API_KEY=${apiKey}`);
+  console.log("  export CRW_API_URL=https://api.fastcrw.com");
+} else {
+  console.log("\nThe skill works in local mode out of the box (free, embedded).");
+  console.log("Cloud mode (managed, 500 free credits): https://fastcrw.com");
+}
+console.log("\nDocs: https://fastcrw.com/docs");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crw-mcp",
-  "version": "0.16.0",
+  "version": "0.18.0",
   "description": "MCP server for CRW web scraper — scrape, crawl, map, search, and PDF-parse tools for AI agents",
   "license": "AGPL-3.0",
   "homepage": "https://github.com/us/crw",
@@ -30,9 +30,9 @@
     "skills/SKILL.md"
   ],
   "optionalDependencies": {
-    "crw-mcp-darwin-x64": "0.16.0",
-    "crw-mcp-darwin-arm64": "0.16.0",
-    "crw-mcp-linux-x64": "0.16.0",
-    "crw-mcp-linux-arm64": "0.16.0"
+    "crw-mcp-darwin-x64": "0.18.0",
+    "crw-mcp-darwin-arm64": "0.18.0",
+    "crw-mcp-linux-x64": "0.18.0",
+    "crw-mcp-linux-arm64": "0.18.0"
   }
 }

package/skills/SKILL.md

@@ -32,7 +32,7 @@ This installs the CRW skill and MCP server to all detected AI agents (Claude Cod
 ## Authentication
 
 - **Embedded mode** (default): No key needed — the MCP server runs a self-contained scraper in ~6 MB RAM. No server required.
-- **Cloud mode** (fastcrw.com): Set `CRW_API_KEY=crw_live_...` and `CRW_API_URL=https://fastcrw.com/api`. Get a free key at https://fastcrw.com with 500 credits/month.
+- **Cloud mode** (fastcrw.com): Set `CRW_API_KEY=crw_live_...` and `CRW_API_URL=https://api.fastcrw.com`. Get a free key at https://fastcrw.com with 500 one-time lifetime credits (never resets, not monthly).
 
 ## MCP Tools
 
@@ -76,7 +76,9 @@ Parameters:
 - `id` (required) — The crawl job ID from `crw_crawl`
 - `maxLength` — Truncate each page's content fields to this many chars. `0` = unbounded. Default: ~15 000
 
-Returns: `{ "status": "pending|running|completed|failed", "data": [...] }`
+Returns: `{ "status": "scraping|completed|failed", "data": [...] }`
+
+> **Browser Automation:** Full interactive browser control (JavaScript rendering, click, fill, etc.) requires the separate **crw-browse** MCP server binary (`command: crw-browse`). It exposes its own tools (`goto`, `tree`, and others) and is not part of this MCP server. Do not call `crw_browse` here — it is not a tool in crw-mcp and will return a JSON-RPC -32602 "Unknown tool" error.
 
 ### crw_search
 
@@ -86,7 +88,6 @@ Parameters:
 - `query` (required) — The search query
 - `limit` — Maximum number of results to return. Default: `5`
 - `lang` — Language code for results (e.g. `"en"`, `"tr"`)
-- `country` — Country code for results (e.g. `"us"`, `"tr"`)
 - `tbs` — Time filter: `qdr:h|qdr:d|qdr:w|qdr:m|qdr:y` (past hour/day/week/month/year)
 - `sources` — If set, group results by source: `web`, `news`, `images`
 - `categories` — Bias toward a category (e.g. `"pdf"`, `"github"`, `"research"`, or a native SearXNG category)
@@ -137,6 +138,18 @@ crw_check_crawl_status(id="...")  → poll until completed
 crw_search(query="your search query", limit=5)
 ```
 
+**Search from the CLI (one-shot LLM-ready output):**
+
+When the `crw` binary is available, prefer the native field projection
+over piping through `jq` — it's one call instead of two:
+
+```bash
+crw search "renewable energy 2024" --json --fields title,url,snippet --limit 3
+```
+
+Available fields: `title`, `url`, `description`, `snippet`, `position`,
+`score`, `category`. `--json` is shorthand for `--format json`.
+
 ## Common Edge Cases
 
 - **JavaScript-heavy sites**: Set `renderJs: true` if the page is blank or returns a loading skeleton
@@ -146,7 +159,7 @@ crw_search(query="your search query", limit=5)
 
 ## Links
 
-- Cloud API: https://fastcrw.com — 500 free credits/month
+- Cloud API: https://fastcrw.com — 500 one-time lifetime free credits (never resets, not monthly)
 - Docs: https://docs.fastcrw.com
 - GitHub: https://github.com/us/crw
 - Firecrawl-compatible: same REST endpoints at `/v1/scrape`, `/v1/crawl`, `/v1/map`, `/v1/search`