crw-mcp 0.30.0 → 0.32.0

package/bin/agents.js

@@ -53,21 +53,51 @@ function installSkill(agent, skillContent) {
   const dir = path.join(home, agent.configDir, "skills", "crw");
   fs.mkdirSync(dir, { recursive: true });
   const file = path.join(dir, "SKILL.md");
-  fs.writeFileSync(file, skillContent, "utf-8");
+  atomicWriteText(file, skillContent);
   return file;
 }
 
 function readJson(file) {
+  if (!fs.existsSync(file)) return {};
   try {
-    return JSON.parse(fs.readFileSync(file, "utf-8"));
-  } catch {
-    return {};
+    const value = JSON.parse(fs.readFileSync(file, "utf-8"));
+    if (!value || typeof value !== "object" || Array.isArray(value)) {
+      throw new Error("top-level value must be an object");
+    }
+    return value;
+  } catch (error) {
+    throw new Error(`refusing to overwrite malformed JSON in ${file}: ${error.message}`);
   }
 }
 
-function writeJson(file, obj) {
+function ensureObject(value, label, file) {
+  if (value === undefined) return {};
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    throw new Error(`refusing to replace non-object ${label} in ${file}`);
+  }
+  return value;
+}
+
+function backupOnce(file) {
+  if (!fs.existsSync(file)) return;
+  const backup = `${file}.crw-backup`;
+  if (!fs.existsSync(backup)) fs.copyFileSync(file, backup);
+}
+
+function atomicWriteText(file, content) {
   fs.mkdirSync(path.dirname(file), { recursive: true });
-  fs.writeFileSync(file, JSON.stringify(obj, null, 2) + "\n", "utf-8");
+  backupOnce(file);
+  const tmp = `${file}.crw-tmp-${process.pid}-${Date.now()}`;
+  try {
+    fs.writeFileSync(tmp, content, { encoding: "utf-8", mode: 0o600 });
+    fs.renameSync(tmp, file);
+  } finally {
+    try { fs.unlinkSync(tmp); } catch { /* already renamed or never created */ }
+  }
+}
+
+function writeJson(file, obj) {
+  atomicWriteText(file, JSON.stringify(obj, null, 2) + "\n");
 }
 
 /**
@@ -90,8 +120,15 @@ function installMcp(agent, env) {
 
   if (m.kind === "claude-cli") {
     // add-json avoids the `-e` variadic-name pitfall and merges ~/.claude.json
-    // (user scope) correctly. Idempotent-ish: remove a stale entry first.
+    // (user scope) correctly. Probe before touching state, then restore the
+    // exact prior user config if replacement fails.
     const server = { command: MCP_CMD, args: MCP_ARGS, ...(hasEnv ? { env } : {}) };
+    const probe = spawnSync("claude", ["--version"], { stdio: "ignore" });
+    if (probe.error || probe.status !== 0) {
+      throw new Error("Claude Code CLI was detected but is not launchable");
+    }
+    const configFile = path.join(home, ".claude.json");
+    const original = fs.existsSync(configFile) ? fs.readFileSync(configFile) : null;
     spawnSync("claude", ["mcp", "remove", "--scope", "user", "crw"], { stdio: "ignore" });
     const r = spawnSync(
       "claude",
@@ -99,7 +136,12 @@ function installMcp(agent, env) {
       { stdio: "ignore" },
     );
     if (r.error || r.status !== 0) {
-      return "MCP: run `claude mcp add-json --scope user crw '" + JSON.stringify(server) + "'` (claude CLI not found)";
+      if (original) {
+        fs.writeFileSync(configFile, original);
+      } else {
+        try { fs.unlinkSync(configFile); } catch { /* no file was created */ }
+      }
+      throw new Error("Claude Code rejected the MCP registration; previous config restored");
     }
     return "MCP via claude CLI (user scope)";
   }
@@ -107,7 +149,7 @@ function installMcp(agent, env) {
   if (m.kind === "json") {
     const file = path.join(home, m.file);
     const cfg = readJson(file);
-    cfg.mcpServers = cfg.mcpServers || {};
+    cfg.mcpServers = ensureObject(cfg.mcpServers, "mcpServers", file);
     cfg.mcpServers.crw = {
       ...(m.withType ? { type: "stdio" } : {}),
       command: MCP_CMD,
@@ -122,7 +164,7 @@ function installMcp(agent, env) {
     const file = path.join(home, m.file);
     const cfg = readJson(file);
     if (!cfg.$schema) cfg.$schema = "https://opencode.ai/config.json";
-    cfg.mcp = cfg.mcp || {};
+    cfg.mcp = ensureObject(cfg.mcp, "mcp", file);
     cfg.mcp.crw = {
       type: "local",
       command: [MCP_CMD, ...MCP_ARGS],
@@ -140,20 +182,38 @@ function installMcp(agent, env) {
     } catch {
       /* new file */
     }
-    if (/\[mcp_servers\.crw\]/.test(toml)) {
-      return `MCP already in ${m.file} (left as-is)`;
-    }
+    // The installer owns the crw section. Replace it so a previous install
+    // with embedded env credentials cannot override a later `--from-config`
+    // setup (and so re-runs remain idempotent).
+    toml = removeOwnedCodexSections(toml);
     const envLines = hasEnv
       ? "\n[mcp_servers.crw.env]\n" +
         Object.entries(env).map(([k, v]) => `${k} = "${v}"`).join("\n") + "\n"
       : "";
     const block = `\n[mcp_servers.crw]\ncommand = "${MCP_CMD}"\nargs = [${MCP_ARGS.map((a) => `"${a}"`).join(", ")}]\n${envLines}`;
-    fs.mkdirSync(path.dirname(file), { recursive: true });
-    fs.appendFileSync(file, (toml && !toml.endsWith("\n") ? "\n" : "") + block, "utf-8");
+    const next = toml + (toml && !toml.endsWith("\n") ? "\n" : "") + block;
+    atomicWriteText(file, next);
     return `MCP → ${m.file}`;
   }
 
   return "MCP: unsupported";
 }
 
+function removeOwnedCodexSections(toml) {
+  const lines = toml.split("\n");
+  const kept = [];
+  let skipping = false;
+  for (const line of lines) {
+    const isSection = /^\s*\[[^\]]+\]/.test(line);
+    const isOwned = /^\s*\[mcp_servers\.crw(?:\.[^\]]+)?\]\s*(?:#.*)?$/.test(line);
+    if (isOwned) {
+      skipping = true;
+      continue;
+    }
+    if (isSection) skipping = false;
+    if (!skipping) kept.push(line);
+  }
+  return kept.join("\n").replace(/\n{3,}/g, "\n\n").trimEnd();
+}
+
 module.exports = { AGENTS, detectAgents, getApiKey, readSkill, installSkill, installMcp, home };

package/bin/init.js

@@ -73,6 +73,6 @@ if (apiKey) {
   console.log("  export CRW_API_URL=https://api.fastcrw.com");
 } else {
   console.log("\nThe skill works in local mode out of the box (free, embedded).");
-  console.log("Cloud mode (managed, 500 free credits): https://fastcrw.com");
+  console.log("Cloud mode (managed, 1000 free credits): https://fastcrw.com");
 }
 console.log("\nDocs: https://fastcrw.com/docs");

package/bin/install.js

@@ -36,6 +36,7 @@ Options:
   --gemini-cli     Gemini CLI       --windsurf   Windsurf
   --api-key <key>  fastcrw.com API key → cloud mode (else local/embedded)
   --api-url <url>  API base (default https://api.fastcrw.com when a key is set)
+  --from-config    Read mode from ~/.config/crw/config.toml at runtime
   -h, --help       Show this help
 
 Without flags, auto-detects installed agents. Installs the skill (SKILL.md) and
@@ -44,7 +45,8 @@ registers the "crw" MCP server. Run \`crw-mcp init\` for the skill only.
   process.exit(0);
 }
 
-const apiKey = getApiKey(args);
+const fromConfig = args.includes("--from-config");
+const apiKey = fromConfig ? null : getApiKey(args);
 const apiUrl = argValue("--api-url") || (apiKey ? "https://api.fastcrw.com" : null);
 const env = apiKey ? { CRW_API_KEY: apiKey, CRW_API_URL: apiUrl } : {};
 
@@ -58,6 +60,7 @@ if (agents.length === 0) {
 
 const skill = readSkill();
 const results = [];
+const failures = [];
 for (const agent of agents) {
   try {
     const skillPath = installSkill(agent, skill);
@@ -65,6 +68,7 @@ for (const agent of agents) {
     results.push({ name: agent.name, skillPath, mcpLabel });
   } catch (err) {
     console.error(`  ${agent.name}: ${err.message}`);
+    failures.push(agent.name);
   }
 }
 
@@ -80,11 +84,17 @@ for (const r of results) {
   console.log(`    ${r.mcpLabel.replace(home, "~")}`);
 }
 
-if (apiKey) {
+if (fromConfig) {
+  console.log("\nMode: follows ~/.config/crw/config.toml (credentials are not copied into agent configs).");
+} else if (apiKey) {
   console.log(`\nMode: cloud — CRW_API_KEY ${apiKey.slice(0, 10)}… → ${apiUrl}`);
 } else {
   console.log("\nMode: local (free, embedded binary — no key needed).");
-  console.log("  Cloud mode: re-run with --api-key crw_live_… (500 free credits at https://fastcrw.com).");
+  console.log("  Cloud mode: re-run with --api-key crw_live_… (1000 free credits at https://fastcrw.com).");
 }
 console.log("\nRestart your agent to load the MCP server.");
 console.log("Docs: https://fastcrw.com/docs");
+if (failures.length > 0) {
+  console.error(`\nFailed targets: ${failures.join(", ")}. Existing configs were preserved.`);
+  process.exitCode = 1;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crw-mcp",
-  "version": "0.30.0",
+  "version": "0.32.0",
   "description": "MCP server for CRW web scraper — scrape, crawl, map, extract, search, and PDF-parse tools for AI agents",
   "license": "AGPL-3.0",
   "author": "fastCRW <hello@fastcrw.com>",
@@ -38,9 +38,9 @@
     "skills/SKILL.md"
   ],
   "optionalDependencies": {
-    "crw-mcp-darwin-x64": "0.30.0",
-    "crw-mcp-darwin-arm64": "0.30.0",
-    "crw-mcp-linux-x64": "0.30.0",
-    "crw-mcp-linux-arm64": "0.30.0"
+    "crw-mcp-darwin-x64": "0.32.0",
+    "crw-mcp-darwin-arm64": "0.32.0",
+    "crw-mcp-linux-x64": "0.32.0",
+    "crw-mcp-linux-arm64": "0.32.0"
   }
 }

package/skills/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: crw
-description: "Scrape, crawl, map, and search the web using fastCRW's native /v1 API. Use when the user needs web page content, site-wide extraction, URL discovery, or web search results. Single binary, 6 MB RAM; /v2 exists separately for Firecrawl migration."
+description: "Scrape, crawl, map, and search the web using fastCRW's native /v1 API. Use when the user needs web page content, site-wide extraction, URL discovery, or web search results. Single binary, 14 MB RAM; /v2 exists separately for Firecrawl migration."
 license: AGPL-3.0
 metadata:
   author: us
@@ -32,12 +32,12 @@ npx crw-mcp@latest init      # skill only
 
 ## Authentication
 
-- **Embedded mode** (default): No key needed — the MCP server runs a self-contained scraper in ~6 MB RAM. No server required.
-- **Cloud mode** (fastcrw.com): Set `CRW_API_KEY=crw_live_...` and `CRW_API_URL=https://api.fastcrw.com`. Get a free key at https://fastcrw.com with 500 one-time lifetime credits (never resets, not monthly).
+- **Embedded mode** (default): No key needed — the MCP server runs a self-contained scraper in ~14 MB RAM. No server required.
+- **Cloud mode** (fastcrw.com): Set `CRW_API_KEY=crw_live_...` and `CRW_API_URL=https://api.fastcrw.com`. Get a free key at https://fastcrw.com with 1000 one-time lifetime credits (never resets, not monthly).
 
 ## MCP Tools
 
-> **Output bounds:** By default, content is truncated to ~15 000 chars (`crw_scrape`, `crw_check_crawl_status`, `crw_parse_file`, and any page content inlined into `crw_search` results via `scrapeOptions`) and `crw_map` returns ≤ 100 URLs. Truncated results carry a `truncated: true` marker (`crw_map` also adds `totalDiscovered`). Pass `maxLength: 0` or `limit: 0` to opt out of bounding. `crw_search` does not advertise `maxLength`, so an agent will not discover it, but a hand-written client may still pass it.
+> **Output bounds:** By default, content is truncated to ~15 000 chars (`crw_scrape`, `crw_check_crawl_status`, `crw_parse_file`, and any page content inlined into `crw_search` results via `scrapeOptions`) and `crw_map` returns ≤ 100 URLs and ≤ 100 sitemaps. Truncated results carry a `truncated: true` marker (`crw_map` also adds `totalDiscovered` for links and `totalSitemaps` for sitemaps). Pass `maxLength: 0` or `limit: 0` to opt out of bounding. `crw_search` does not advertise `maxLength`, so an agent will not discover it, but a hand-written client may still pass it.
 
 ### crw_scrape
 
@@ -194,7 +194,7 @@ Available fields: `title`, `url`, `description`, `snippet`, `position`,
 
 ## Links
 
-- Cloud API: https://fastcrw.com — 500 one-time lifetime free credits (never resets, not monthly)
+- Cloud API: https://fastcrw.com — 1000 one-time lifetime free credits (never resets, not monthly)
 - Docs: https://docs.fastcrw.com
 - GitHub: https://github.com/us/crw
 - Native API: `/v1/scrape`, `/v1/crawl`, `/v1/map`, and `/v1/search` are the recommended routes for new CRW integrations