crw-mcp 0.3.0 → 0.3.2

package/bin/crw-mcp.js

@@ -1,5 +1,11 @@
 #!/usr/bin/env node
 
+// Handle `init` subcommand before delegating to the Rust binary
+if (process.argv[2] === "init") {
+  require("./init.js");
+  process.exit(0);
+}
+
 const { spawnSync } = require("child_process");
 const path = require("path");
 

package/bin/init.js

@@ -0,0 +1,135 @@
+#!/usr/bin/env node
+
+const fs = require("fs");
+const path = require("path");
+const os = require("os");
+
+const AGENTS = [
+  { name: "Claude Code", dir: ".claude", flag: "--claude-code" },
+  { name: "Cursor", dir: ".cursor", flag: "--cursor" },
+  { name: "Gemini CLI", dir: ".gemini", flag: "--gemini-cli" },
+  { name: "Codex", dir: ".codex", flag: "--codex" },
+  { name: "OpenCode", dir: ".opencode", flag: "--opencode" },
+  { name: "Windsurf", dir: ".windsurf", flag: "--windsurf" },
+];
+
+const home = os.homedir();
+const args = process.argv.slice(2);
+
+function hasFlag(flag) {
+  return args.includes(flag);
+}
+
+function getApiKey() {
+  const idx = args.indexOf("--api-key");
+  return idx !== -1 && args[idx + 1] ? args[idx + 1] : null;
+}
+
+function readSkillFile() {
+  const skillPath = path.join(__dirname, "..", "skills", "SKILL.md");
+  return fs.readFileSync(skillPath, "utf-8");
+}
+
+function detectAgents() {
+  const all = hasFlag("--all");
+  const specificFlags = AGENTS.filter((a) => hasFlag(a.flag));
+
+  if (!all && specificFlags.length === 0) {
+    // Auto-detect: install to all agents whose config dirs exist
+    return AGENTS.filter((a) =>
+      fs.existsSync(path.join(home, a.dir))
+    );
+  }
+
+  if (all) {
+    return AGENTS.filter((a) =>
+      fs.existsSync(path.join(home, a.dir))
+    );
+  }
+
+  return specificFlags;
+}
+
+function deploy(agents, skillContent) {
+  const installed = [];
+
+  for (const agent of agents) {
+    const targetDir = path.join(home, agent.dir, "skills", "crw");
+    const targetFile = path.join(targetDir, "SKILL.md");
+
+    try {
+      fs.mkdirSync(targetDir, { recursive: true });
+      fs.writeFileSync(targetFile, skillContent, "utf-8");
+      installed.push({ name: agent.name, path: targetFile });
+    } catch (err) {
+      console.error(`  Failed to install for ${agent.name}: ${err.message}`);
+    }
+  }
+
+  return installed;
+}
+
+function main() {
+  if (hasFlag("--help") || hasFlag("-h")) {
+    console.log(`
+crw-mcp init — Install CRW agent skill to your AI coding agents
+
+Usage:
+  npx crw-mcp@latest init [options]
+
+Options:
+  --all            Install to all detected agents
+  --claude-code    Install to Claude Code only
+  --cursor         Install to Cursor only
+  --gemini-cli     Install to Gemini CLI only
+  --codex          Install to Codex only
+  --opencode       Install to OpenCode only
+  --windsurf       Install to Windsurf only
+  --api-key <key>  Set your fastcrw.com API key
+  -h, --help       Show this help message
+
+Without flags, auto-detects installed agents and installs to all of them.
+`);
+    process.exit(0);
+  }
+
+  const skillContent = readSkillFile();
+  const agents = detectAgents();
+
+  if (agents.length === 0) {
+    console.log("No supported AI agents detected.");
+    console.log("Supported agents: " + AGENTS.map((a) => a.name).join(", "));
+    console.log("\nManually install by copying the skill file to your agent's skills directory.");
+    process.exit(1);
+  }
+
+  const installed = deploy(agents, skillContent);
+
+  if (installed.length === 0) {
+    console.log("Failed to install to any agent.");
+    process.exit(1);
+  }
+
+  console.log("crw skill installed:\n");
+  const maxName = Math.max(...installed.map((i) => i.name.length));
+  for (const i of installed) {
+    console.log(`  ${i.name.padEnd(maxName + 2)} ${i.path}`);
+  }
+
+  const apiKey = getApiKey();
+  if (apiKey) {
+    console.log(`\nAPI key configured: ${apiKey.slice(0, 6)}...`);
+    console.log("Set it in your environment:");
+    console.log(`  export CRW_API_KEY=${apiKey}`);
+  } else {
+    console.log("\nCloud mode (fastcrw.com):");
+    console.log("  export CRW_API_KEY=fc-your-key");
+    console.log("  export CRW_API_URL=https://fastcrw.com/api");
+  }
+
+  console.log("\nLocal mode (no key needed):");
+  console.log("  npx crw-mcp");
+  console.log("\nDocs: https://fastcrw.com/docs");
+}
+
+main();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crw-mcp",
-  "version": "0.3.0",
+  "version": "0.3.2",
   "description": "MCP server for CRW web scraper — scrape, crawl, and map tools for AI agents",
   "license": "AGPL-3.0",
   "homepage": "https://github.com/us/crw",
@@ -22,14 +22,16 @@
     "crw-mcp": "bin/crw-mcp.js"
   },
   "files": [
-    "bin/crw-mcp.js"
+    "bin/crw-mcp.js",
+    "bin/init.js",
+    "skills/SKILL.md"
   ],
   "optionalDependencies": {
-    "crw-mcp-darwin-x64": "0.2.0",
-    "crw-mcp-darwin-arm64": "0.2.0",
-    "crw-mcp-linux-x64": "0.2.0",
-    "crw-mcp-linux-arm64": "0.2.0",
-    "crw-mcp-win32-x64": "0.2.0",
-    "crw-mcp-win32-arm64": "0.2.0"
+    "crw-mcp-darwin-x64": "0.3.0",
+    "crw-mcp-darwin-arm64": "0.3.0",
+    "crw-mcp-linux-x64": "0.3.0",
+    "crw-mcp-linux-arm64": "0.3.0",
+    "crw-mcp-win32-x64": "0.3.0",
+    "crw-mcp-win32-arm64": "0.3.0"
   }
 }

package/skills/SKILL.md

@@ -0,0 +1,114 @@
+---
+name: crw
+description: "Scrape, crawl, map, and search the web using fastCRW. Use when the user needs web page content, site-wide extraction, URL discovery, or web search results. Single binary, 6 MB RAM, Firecrawl-compatible API."
+license: AGPL-3.0
+metadata:
+  author: us
+  version: "0.3.0"
+  homepage: https://fastcrw.com
+  repository: https://github.com/us/crw
+allowed-tools: Bash(npx:crw-mcp*) Bash(curl:*) Read
+---
+
+# fastCRW — Web Data Toolkit for AI Agents
+
+## When to use this skill
+
+Use this skill when:
+- The user asks you to read, scrape, or fetch a web page
+- You need to extract content from a URL for context or research
+- The user wants to crawl an entire website or discover its pages
+- You need to search the web and get page content (cloud mode)
+- The user mentions Firecrawl — CRW is a drop-in replacement
+
+## Installation
+
+```bash
+npx crw-mcp@latest init
+```
+
+This installs the CRW MCP server to all detected AI agents (Claude Code, Cursor, Gemini CLI, Codex, OpenCode, Windsurf, Roo Code).
+
+## Authentication
+
+- **Embedded mode** (default): No key needed — the MCP server runs a self-contained scraper in ~6 MB RAM. No server required.
+- **Cloud mode** (fastcrw.com): Set `CRW_API_KEY=crw_live_...` and `CRW_API_URL=https://fastcrw.com/api`. Get a free key at https://fastcrw.com with 500 credits/month.
+
+## MCP Tools
+
+### crw_scrape
+
+Scrape a single URL and return clean content.
+
+Parameters:
+- `url` (required) — The URL to scrape
+- `formats` — Output formats: `markdown` (default), `html`, `rawHtml`, `plainText`, `links`, `json`
+- `onlyMainContent` — Strip navs/footers/sidebars. Default: `true`
+- `renderJs` — Force JavaScript rendering. Default: auto-detect (null)
+- `cssSelector` — Extract only elements matching this CSS selector
+- `xpath` — Extract only elements matching this XPath
+- `includeTags` — Only include these HTML tags (e.g. `["article", "main"]`)
+- `excludeTags` — Remove these HTML tags (e.g. `["nav", "footer"]`)
+
+### crw_crawl
+
+Start an async BFS crawl from a URL. Returns a job ID — poll with `crw_check_crawl_status`.
+
+Parameters:
+- `url` (required) — Starting URL
+- `maxDepth` — Maximum link depth. Default: `2`, max: `10`
+- `limit` — Maximum pages to crawl. Default: `10`, max: `1000`
+
+Returns: `{ "id": "job-uuid" }` — use this ID with crw_check_crawl_status.
+
+### crw_check_crawl_status
+
+Poll an async crawl job for results.
+
+Parameters:
+- `id` (required) — The crawl job ID from `crw_crawl`
+
+Returns: `{ "status": "pending|running|completed|failed", "data": [...] }`
+
+### crw_map
+
+Discover all URLs on a website via sitemap + link extraction, without scraping content.
+
+Parameters:
+- `url` (required) — The URL to map
+- `maxDepth` — Discovery depth. Default: `2`
+- `useSitemap` — Check sitemap.xml. Default: `true`
+
+Returns: `{ "links": ["url1", "url2", ...] }` — up to 5000 URLs.
+
+## Common Patterns
+
+**Scrape a page for context:**
+```
+crw_scrape(url="https://example.com", formats=["markdown"])
+```
+
+**Crawl docs for RAG:**
+First discover URLs, then crawl:
+```
+crw_map(url="https://docs.example.com")  → get URL list
+crw_crawl(url="https://docs.example.com", limit=50)  → extract all content
+crw_check_crawl_status(id="...")  → poll until completed
+```
+
+**Search the web (cloud mode only):**
+Use the REST API directly — `POST /v1/search` with `{"query": "...", "limit": 5}`. Requires `CRW_API_URL` and `CRW_API_KEY`.
+
+## Common Edge Cases
+
+- **JavaScript-heavy sites**: Set `renderJs: true` if the page is blank or returns a loading skeleton
+- **Rate limiting**: Cloud mode has per-plan rate limits. Check response headers for `X-RateLimit-*`
+- **Large crawls**: Use `crw_map` first to estimate site size before committing to a large `crw_crawl`
+- **Timeout**: Crawl jobs expire after 1 hour. Poll `crw_check_crawl_status` regularly
+
+## Links
+
+- Cloud API: https://fastcrw.com — 500 free credits/month
+- Docs: https://docs.fastcrw.com
+- GitHub: https://github.com/us/crw
+- Firecrawl-compatible: same REST endpoints at `/v1/scrape`, `/v1/crawl`, `/v1/map`, `/v1/search`