crw-mcp 0.15.2 → 0.17.0

package/bin/init.js

@@ -124,7 +124,7 @@ Without flags, auto-detects installed agents and installs to all of them.
   } else {
     console.log("\nCloud mode (fastcrw.com):");
     console.log("  500 free one-time credits, managed infra — https://fastcrw.com");
-    console.log("  export CRW_API_KEY=fc-your-key");
+    console.log("  export CRW_API_KEY=crw_live_xxx");
     console.log("  export CRW_API_URL=https://api.fastcrw.com");
     console.log("  Terms of Service: https://fastcrw.com/terms");
   }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "crw-mcp",
-  "version": "0.15.2",
-  "description": "MCP server for CRW web scraper — scrape, crawl, and map tools for AI agents",
+  "version": "0.17.0",
+  "description": "MCP server for CRW web scraper — scrape, crawl, map, search, and PDF-parse tools for AI agents",
   "license": "AGPL-3.0",
   "homepage": "https://github.com/us/crw",
   "repository": {
@@ -30,9 +30,9 @@
     "skills/SKILL.md"
   ],
   "optionalDependencies": {
-    "crw-mcp-darwin-x64": "0.15.2",
-    "crw-mcp-darwin-arm64": "0.15.2",
-    "crw-mcp-linux-x64": "0.15.2",
-    "crw-mcp-linux-arm64": "0.15.2"
+    "crw-mcp-darwin-x64": "0.17.0",
+    "crw-mcp-darwin-arm64": "0.17.0",
+    "crw-mcp-linux-x64": "0.17.0",
+    "crw-mcp-linux-arm64": "0.17.0"
   }
 }

package/skills/SKILL.md

@@ -32,23 +32,26 @@ This installs the CRW skill and MCP server to all detected AI agents (Claude Cod
 ## Authentication
 
 - **Embedded mode** (default): No key needed — the MCP server runs a self-contained scraper in ~6 MB RAM. No server required.
-- **Cloud mode** (fastcrw.com): Set `CRW_API_KEY=crw_live_...` and `CRW_API_URL=https://fastcrw.com/api`. Get a free key at https://fastcrw.com with 500 credits/month.
+- **Cloud mode** (fastcrw.com): Set `CRW_API_KEY=crw_live_...` and `CRW_API_URL=https://api.fastcrw.com`. Get a free key at https://fastcrw.com with 500 one-time lifetime credits (never resets, not monthly).
 
 ## MCP Tools
 
+> **Output bounds:** By default, content is truncated to ~15 000 chars (`crw_scrape`, `crw_check_crawl_status`, `crw_parse_file`) and `crw_map` returns ≤ 100 URLs. Truncated results carry a `truncated: true` marker (`crw_map` also adds `totalDiscovered`). Pass `maxLength: 0` or `limit: 0` to opt out of bounding.
+
 ### crw_scrape
 
 Scrape a single URL and return clean content.
 
 Parameters:
 - `url` (required) — The URL to scrape
-- `formats` — Output formats: `markdown` (default), `html`, `rawHtml`, `plainText`, `links`, `json`
+- `formats` — Output formats: `markdown` (default), `html`, `links`
 - `onlyMainContent` — Strip navs/footers/sidebars. Default: `true`
+- `includeTags` — Only include content matching these CSS selectors (e.g. `["article", "main"]`)
+- `excludeTags` — Exclude content matching these CSS selectors (e.g. `["nav", "footer"]`)
 - `renderJs` — Force JavaScript rendering. Default: auto-detect (null)
-- `cssSelector` — Extract only elements matching this CSS selector
-- `xpath` — Extract only elements matching this XPath
-- `includeTags` — Only include these HTML tags (e.g. `["article", "main"]`)
-- `excludeTags` — Remove these HTML tags (e.g. `["nav", "footer"]`)
+- `waitFor` — Milliseconds to wait after page load before capturing
+- `renderer` — Renderer override (e.g. `"playwright"`)
+- `maxLength` — Truncate output to this many chars. `0` = unbounded. Default: ~15 000
 
 ### crw_crawl
 
@@ -56,8 +59,12 @@ Start an async BFS crawl from a URL. Returns a job ID — poll with `crw_check_c
 
 Parameters:
 - `url` (required) — Starting URL
-- `maxDepth` — Maximum link depth. Default: `2`, max: `10`
-- `limit` — Maximum pages to crawl. Default: `10`, max: `1000`
+- `maxDepth` — Maximum link depth. Default: `2`
+- `maxPages` — Maximum pages to crawl
+- `jsonSchema` — JSON schema for structured extraction per page
+- `renderJs` — Force JavaScript rendering
+- `waitFor` — Milliseconds to wait after page load before capturing
+- `renderer` — Renderer override
 
 Returns: `{ "id": "job-uuid" }` — use this ID with crw_check_crawl_status.
 
@@ -67,18 +74,23 @@ Poll an async crawl job for results.
 
 Parameters:
 - `id` (required) — The crawl job ID from `crw_crawl`
+- `maxLength` — Truncate each page's content fields to this many chars. `0` = unbounded. Default: ~15 000
+
+Returns: `{ "status": "scraping|completed|failed", "data": [...] }`
 
-Returns: `{ "status": "pending|running|completed|failed", "data": [...] }`
+> **Browser Automation:** Full interactive browser control (JavaScript rendering, click, fill, etc.) requires the separate **crw-browse** MCP server binary (`command: crw-browse`). It exposes its own tools (`goto`, `tree`, and others) and is not part of this MCP server. Do not call `crw_browse` here — it is not a tool in crw-mcp and will return a JSON-RPC -32602 "Unknown tool" error.
 
 ### crw_search
 
-Search the web and return relevant results with titles, URLs, and descriptions.
+Search the web and return relevant results with titles, URLs, and descriptions. Always available in proxy/cloud mode; in embedded mode only when a SearXNG backend is configured.
 
 Parameters:
 - `query` (required) — The search query
 - `limit` — Maximum number of results to return. Default: `5`
 - `lang` — Language code for results (e.g. `"en"`, `"tr"`)
-- `country` — Country code for results (e.g. `"us"`, `"tr"`)
+- `tbs` — Time filter: `qdr:h|qdr:d|qdr:w|qdr:m|qdr:y` (past hour/day/week/month/year)
+- `sources` — If set, group results by source: `web`, `news`, `images`
+- `categories` — Bias toward a category (e.g. `"pdf"`, `"github"`, `"research"`, or a native SearXNG category)
 - `scrapeOptions` — Options for scraping each result page (e.g. `{"formats": ["markdown"]}`)
 
 ### crw_map
@@ -89,8 +101,22 @@ Parameters:
 - `url` (required) — The URL to map
 - `maxDepth` — Discovery depth. Default: `2`
 - `useSitemap` — Check sitemap.xml. Default: `true`
+- `crawlFallback` — Supplement sitemap discovery with a short BFS crawl. Default: `true` (`false` = sitemap-only)
+- `limit` — Maximum URLs to return. `0` = unbounded. Default: `100`
+
+Returns: `{ "links": ["url1", "url2", ...] }`
 
-Returns: `{ "links": ["url1", "url2", ...] }` — up to 5000 URLs.
+### crw_parse_file
+
+Parse a local file (PDF) into markdown or structured output without fetching from the web.
+
+Parameters:
+- `contentBase64` (required) — Base64-encoded file contents
+- `filename` — Original filename (optional, e.g. `"report.pdf"`)
+- `formats` — Output formats: `markdown` (default), `plainText`, `links`, `json`, `summary` (json/summary need a server LLM)
+- `jsonSchema` — JSON schema for LLM extraction (when `formats` includes `json`)
+- `parsers` — Document parsers to apply. Default: `["pdf"]`
+- `maxLength` — Truncate output to this many chars. `0` = unbounded. Default: ~15 000
 
 ## Common Patterns
 
@@ -103,7 +129,7 @@ crw_scrape(url="https://example.com", formats=["markdown"])
 First discover URLs, then crawl:
 ```
 crw_map(url="https://docs.example.com")  → get URL list
-crw_crawl(url="https://docs.example.com", limit=50)  → extract all content
+crw_crawl(url="https://docs.example.com", maxPages=50)  → extract all content
 crw_check_crawl_status(id="...")  → poll until completed
 ```
 
@@ -112,6 +138,18 @@ crw_check_crawl_status(id="...")  → poll until completed
 crw_search(query="your search query", limit=5)
 ```
 
+**Search from the CLI (one-shot LLM-ready output):**
+
+When the `crw` binary is available, prefer the native field projection
+over piping through `jq` — it's one call instead of two:
+
+```bash
+crw search "renewable energy 2024" --json --fields title,url,snippet --limit 3
+```
+
+Available fields: `title`, `url`, `description`, `snippet`, `position`,
+`score`, `category`. `--json` is shorthand for `--format json`.
+
 ## Common Edge Cases
 
 - **JavaScript-heavy sites**: Set `renderJs: true` if the page is blank or returns a loading skeleton
@@ -121,7 +159,7 @@ crw_search(query="your search query", limit=5)
 
 ## Links
 
-- Cloud API: https://fastcrw.com — 500 free credits/month
+- Cloud API: https://fastcrw.com — 500 one-time lifetime free credits (never resets, not monthly)
 - Docs: https://docs.fastcrw.com
 - GitHub: https://github.com/us/crw
 - Firecrawl-compatible: same REST endpoints at `/v1/scrape`, `/v1/crawl`, `/v1/map`, `/v1/search`