crw-mcp 0.6.0 → 0.16.0

package/bin/crw-mcp.js

@@ -7,41 +7,133 @@ if (process.argv[2] === "init") {
 }
 
 const { spawnSync } = require("child_process");
+const fs = require("fs");
 const path = require("path");
+const os = require("os");
+const https = require("https");
 
+const VERSION = require("../package.json").version;
+const REPO = "us/crw";
+
+// Each platform has a prebuilt npm package (fast path, installed via
+// optionalDependencies) AND a matching GitHub release asset (fallback path,
+// downloaded on first run). The fallback makes the launcher robust to a
+// missing/unpublishable platform package (e.g. npm security-held names) — no
+// single package can freeze the channel, mirroring the PyPI launcher.
 const PLATFORMS = {
-  "darwin-x64": "crw-mcp-darwin-x64",
-  "darwin-arm64": "crw-mcp-darwin-arm64",
-  "linux-x64": "crw-mcp-linux-x64",
-  "linux-arm64": "crw-mcp-linux-arm64",
-  "win32-x64": "crw-mcp-win32-x64",
-  "win32-arm64": "crw-mcp-win32-arm64",
+  "darwin-x64": { pkg: "crw-mcp-darwin-x64", asset: "crw-mcp-darwin-x64.tar.gz" },
+  "darwin-arm64": { pkg: "crw-mcp-darwin-arm64", asset: "crw-mcp-darwin-arm64.tar.gz" },
+  "linux-x64": { pkg: "crw-mcp-linux-x64", asset: "crw-mcp-linux-x64.tar.gz" },
+  "linux-arm64": { pkg: "crw-mcp-linux-arm64", asset: "crw-mcp-linux-arm64.tar.gz" },
+  "win32-x64": { pkg: "crw-mcp-win32-x64", asset: "crw-mcp-win32-x64.zip" },
+  "win32-arm64": { pkg: "crw-mcp-win32-arm64", asset: "crw-mcp-win32-arm64.zip" },
 };
 
 const key = `${process.platform}-${process.arch}`;
-const pkg = PLATFORMS[key];
+const plat = PLATFORMS[key];
 
-if (!pkg) {
+if (!plat) {
   console.error(
     `crw-mcp: unsupported platform ${key}. Supported: ${Object.keys(PLATFORMS).join(", ")}`
   );
   process.exit(1);
 }
 
-const ext = process.platform === "win32" ? ".exe" : "";
+const binName = `crw-mcp${process.platform === "win32" ? ".exe" : ""}`;
 
-let bin;
-try {
-  bin = path.join(
-    path.dirname(require.resolve(`${pkg}/package.json`)),
-    `crw-mcp${ext}`
-  );
-} catch {
-  console.error(
-    `crw-mcp: platform package "${pkg}" not found. Try reinstalling:\n  npm install crw-mcp`
-  );
-  process.exit(1);
+// 1. Explicit override.
+function fromEnv() {
+  const p = process.env.CRW_MCP_BINARY || process.env.CRW_BINARY;
+  return p && fs.existsSync(p) ? p : null;
+}
+
+// 2. Prebuilt platform package (fast path — no download).
+function fromPackage() {
+  try {
+    const bin = path.join(
+      path.dirname(require.resolve(`${plat.pkg}/package.json`)),
+      binName
+    );
+    return fs.existsSync(bin) ? bin : null;
+  } catch {
+    return null;
+  }
+}
+
+function cacheDir() {
+  const base =
+    process.platform === "win32"
+      ? process.env.LOCALAPPDATA || path.join(os.homedir(), "AppData", "Local")
+      : process.env.XDG_CACHE_HOME || path.join(os.homedir(), ".cache");
+  return path.join(base, "crw-mcp", `v${VERSION}`);
+}
+
+function httpsGet(url, dest, redirects = 0) {
+  return new Promise((resolve, reject) => {
+    https
+      .get(url, { headers: { "User-Agent": `crw-mcp/${VERSION}` } }, (res) => {
+        if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
+          res.resume();
+          if (redirects > 5) return reject(new Error("too many redirects"));
+          return resolve(httpsGet(res.headers.location, dest, redirects + 1));
+        }
+        if (res.statusCode !== 200) {
+          res.resume();
+          return reject(new Error(`HTTP ${res.statusCode} for ${url}`));
+        }
+        const file = fs.createWriteStream(dest);
+        res.pipe(file);
+        file.on("finish", () => file.close(() => resolve()));
+        file.on("error", reject);
+      })
+      .on("error", reject);
+  });
+}
+
+// 3. Download the matching release asset and extract it (cached per version).
+async function fromDownload() {
+  const dir = cacheDir();
+  const bin = path.join(dir, binName);
+  if (fs.existsSync(bin)) return bin;
+
+  fs.mkdirSync(dir, { recursive: true });
+  const archive = path.join(dir, plat.asset);
+  const url = `https://github.com/${REPO}/releases/download/v${VERSION}/${plat.asset}`;
+
+  // stderr only: stdout is the MCP (JSON-RPC) channel.
+  console.error(`crw-mcp: downloading ${plat.asset} (v${VERSION})...`);
+  await httpsGet(url, archive);
+
+  const tarArgs = plat.asset.endsWith(".zip")
+    ? ["-xf", archive, "-C", dir]
+    : ["-xzf", archive, "-C", dir];
+  const x = spawnSync("tar", tarArgs, { stdio: "inherit" });
+  if (x.status !== 0) {
+    throw new Error(`failed to extract ${plat.asset} (tar exit ${x.status})`);
+  }
+  fs.rmSync(archive, { force: true });
+
+  if (!fs.existsSync(bin)) {
+    throw new Error(`binary ${binName} not found in ${plat.asset}`);
+  }
+  if (process.platform !== "win32") fs.chmodSync(bin, 0o755);
+  return bin;
+}
+
+async function resolveBinary() {
+  return fromEnv() || fromPackage() || (await fromDownload());
 }
 
-const result = spawnSync(bin, process.argv.slice(2), { stdio: "inherit" });
-process.exit(result.status ?? 1);
+resolveBinary()
+  .then((bin) => {
+    const result = spawnSync(bin, process.argv.slice(2), { stdio: "inherit" });
+    process.exit(result.status ?? 1);
+  })
+  .catch((err) => {
+    console.error(
+      `crw-mcp: could not locate or download the ${key} binary.\n  ${err.message}\n` +
+        `  Set CRW_MCP_BINARY=/path/to/crw-mcp to use a local build, or install\n` +
+        `  from https://github.com/${REPO}/releases.`
+    );
+    process.exit(1);
+  });

package/bin/init.js

@@ -123,11 +123,13 @@ Without flags, auto-detects installed agents and installs to all of them.
     console.log(`  export CRW_API_KEY=${apiKey}`);
   } else {
     console.log("\nCloud mode (fastcrw.com):");
+    console.log("  500 free one-time credits, managed infra — https://fastcrw.com");
     console.log("  export CRW_API_KEY=fc-your-key");
-    console.log("  export CRW_API_URL=https://fastcrw.com/api");
+    console.log("  export CRW_API_URL=https://api.fastcrw.com");
+    console.log("  Terms of Service: https://fastcrw.com/terms");
   }
 
-  console.log("\nLocal mode (no key needed):");
+  console.log("\nLocal mode (free, same binary, fully capable):");
   console.log("  npx crw-mcp");
   console.log("\nDocs: https://fastcrw.com/docs");
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "crw-mcp",
-  "version": "0.6.0",
-  "description": "MCP server for CRW web scraper — scrape, crawl, and map tools for AI agents",
+  "version": "0.16.0",
+  "description": "MCP server for CRW web scraper — scrape, crawl, map, search, and PDF-parse tools for AI agents",
   "license": "AGPL-3.0",
   "homepage": "https://github.com/us/crw",
   "repository": {
@@ -21,17 +21,18 @@
   "bin": {
     "crw-mcp": "bin/crw-mcp.js"
   },
+  "engines": {
+    "node": ">=18"
+  },
   "files": [
     "bin/crw-mcp.js",
     "bin/init.js",
     "skills/SKILL.md"
   ],
   "optionalDependencies": {
-    "crw-mcp-darwin-x64": "0.3.5",
-    "crw-mcp-darwin-arm64": "0.3.5",
-    "crw-mcp-linux-x64": "0.3.5",
-    "crw-mcp-linux-arm64": "0.3.5",
-    "crw-mcp-win32-x64": "0.3.5",
-    "crw-mcp-win32-arm64": "0.3.5"
+    "crw-mcp-darwin-x64": "0.16.0",
+    "crw-mcp-darwin-arm64": "0.16.0",
+    "crw-mcp-linux-x64": "0.16.0",
+    "crw-mcp-linux-arm64": "0.16.0"
   }
 }

package/skills/SKILL.md

@@ -36,19 +36,22 @@ This installs the CRW skill and MCP server to all detected AI agents (Claude Cod
 
 ## MCP Tools
 
+> **Output bounds:** By default, content is truncated to ~15 000 chars (`crw_scrape`, `crw_check_crawl_status`, `crw_parse_file`) and `crw_map` returns ≤ 100 URLs. Truncated results carry a `truncated: true` marker (`crw_map` also adds `totalDiscovered`). Pass `maxLength: 0` or `limit: 0` to opt out of bounding.
+
 ### crw_scrape
 
 Scrape a single URL and return clean content.
 
 Parameters:
 - `url` (required) — The URL to scrape
-- `formats` — Output formats: `markdown` (default), `html`, `rawHtml`, `plainText`, `links`, `json`
+- `formats` — Output formats: `markdown` (default), `html`, `links`
 - `onlyMainContent` — Strip navs/footers/sidebars. Default: `true`
+- `includeTags` — Only include content matching these CSS selectors (e.g. `["article", "main"]`)
+- `excludeTags` — Exclude content matching these CSS selectors (e.g. `["nav", "footer"]`)
 - `renderJs` — Force JavaScript rendering. Default: auto-detect (null)
-- `cssSelector` — Extract only elements matching this CSS selector
-- `xpath` — Extract only elements matching this XPath
-- `includeTags` — Only include these HTML tags (e.g. `["article", "main"]`)
-- `excludeTags` — Remove these HTML tags (e.g. `["nav", "footer"]`)
+- `waitFor` — Milliseconds to wait after page load before capturing
+- `renderer` — Renderer override (e.g. `"playwright"`)
+- `maxLength` — Truncate output to this many chars. `0` = unbounded. Default: ~15 000
 
 ### crw_crawl
 
@@ -56,8 +59,12 @@ Start an async BFS crawl from a URL. Returns a job ID — poll with `crw_check_c
 
 Parameters:
 - `url` (required) — Starting URL
-- `maxDepth` — Maximum link depth. Default: `2`, max: `10`
-- `limit` — Maximum pages to crawl. Default: `10`, max: `1000`
+- `maxDepth` — Maximum link depth. Default: `2`
+- `maxPages` — Maximum pages to crawl
+- `jsonSchema` — JSON schema for structured extraction per page
+- `renderJs` — Force JavaScript rendering
+- `waitFor` — Milliseconds to wait after page load before capturing
+- `renderer` — Renderer override
 
 Returns: `{ "id": "job-uuid" }` — use this ID with crw_check_crawl_status.
 
@@ -67,18 +74,22 @@ Poll an async crawl job for results.
 
 Parameters:
 - `id` (required) — The crawl job ID from `crw_crawl`
+- `maxLength` — Truncate each page's content fields to this many chars. `0` = unbounded. Default: ~15 000
 
 Returns: `{ "status": "pending|running|completed|failed", "data": [...] }`
 
 ### crw_search
 
-Search the web and return relevant results with titles, URLs, and descriptions.
+Search the web and return relevant results with titles, URLs, and descriptions. Always available in proxy/cloud mode; in embedded mode only when a SearXNG backend is configured.
 
 Parameters:
 - `query` (required) — The search query
 - `limit` — Maximum number of results to return. Default: `5`
 - `lang` — Language code for results (e.g. `"en"`, `"tr"`)
 - `country` — Country code for results (e.g. `"us"`, `"tr"`)
+- `tbs` — Time filter: `qdr:h|qdr:d|qdr:w|qdr:m|qdr:y` (past hour/day/week/month/year)
+- `sources` — If set, group results by source: `web`, `news`, `images`
+- `categories` — Bias toward a category (e.g. `"pdf"`, `"github"`, `"research"`, or a native SearXNG category)
 - `scrapeOptions` — Options for scraping each result page (e.g. `{"formats": ["markdown"]}`)
 
 ### crw_map
@@ -89,8 +100,22 @@ Parameters:
 - `url` (required) — The URL to map
 - `maxDepth` — Discovery depth. Default: `2`
 - `useSitemap` — Check sitemap.xml. Default: `true`
+- `crawlFallback` — Supplement sitemap discovery with a short BFS crawl. Default: `true` (`false` = sitemap-only)
+- `limit` — Maximum URLs to return. `0` = unbounded. Default: `100`
+
+Returns: `{ "links": ["url1", "url2", ...] }`
+
+### crw_parse_file
 
-Returns: `{ "links": ["url1", "url2", ...] }` — up to 5000 URLs.
+Parse a local file (PDF) into markdown or structured output without fetching from the web.
+
+Parameters:
+- `contentBase64` (required) — Base64-encoded file contents
+- `filename` — Original filename (optional, e.g. `"report.pdf"`)
+- `formats` — Output formats: `markdown` (default), `plainText`, `links`, `json`, `summary` (json/summary need a server LLM)
+- `jsonSchema` — JSON schema for LLM extraction (when `formats` includes `json`)
+- `parsers` — Document parsers to apply. Default: `["pdf"]`
+- `maxLength` — Truncate output to this many chars. `0` = unbounded. Default: ~15 000
 
 ## Common Patterns
 
@@ -103,7 +128,7 @@ crw_scrape(url="https://example.com", formats=["markdown"])
 First discover URLs, then crawl:
 ```
 crw_map(url="https://docs.example.com")  → get URL list
-crw_crawl(url="https://docs.example.com", limit=50)  → extract all content
+crw_crawl(url="https://docs.example.com", maxPages=50)  → extract all content
 crw_check_crawl_status(id="...")  → poll until completed
 ```