crw-mcp 0.27.0 → 0.28.0

package/bin/crw-mcp.js

@@ -1,17 +1,7 @@
 #!/usr/bin/env node
 
-// Handle install/init subcommands before delegating to the Rust binary.
-// `init` = skill only; `install` = skill + MCP server.
-if (process.argv[2] === "init") {
-  require("./init.js");
-  process.exit(0);
-}
-if (process.argv[2] === "install") {
-  require("./install.js");
-  process.exit(0);
-}
-
 const { spawnSync } = require("child_process");
+const crypto = require("crypto");
 const fs = require("fs");
 const path = require("path");
 const os = require("os");
@@ -37,13 +27,6 @@ const PLATFORMS = {
 const key = `${process.platform}-${process.arch}`;
 const plat = PLATFORMS[key];
 
-if (!plat) {
-  console.error(
-    `crw-mcp: unsupported platform ${key}. Supported: ${Object.keys(PLATFORMS).join(", ")}`
-  );
-  process.exit(1);
-}
-
 const binName = `crw-mcp${process.platform === "win32" ? ".exe" : ""}`;
 
 // 1. Explicit override.
@@ -73,28 +56,41 @@ function cacheDir() {
   return path.join(base, "crw-mcp", `v${VERSION}`);
 }
 
-function httpsGet(url, dest, redirects = 0) {
+// Always resolves a Buffer: the archive is verified in memory, so an unverified
+// byte never reaches disk at all.
+function httpsGet(url, redirects = 0) {
   return new Promise((resolve, reject) => {
     https
       .get(url, { headers: { "User-Agent": `crw-mcp/${VERSION}` } }, (res) => {
         if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
           res.resume();
           if (redirects > 5) return reject(new Error("too many redirects"));
-          return resolve(httpsGet(res.headers.location, dest, redirects + 1));
+          return resolve(httpsGet(res.headers.location, redirects + 1));
         }
         if (res.statusCode !== 200) {
           res.resume();
           return reject(new Error(`HTTP ${res.statusCode} for ${url}`));
         }
-        const file = fs.createWriteStream(dest);
-        res.pipe(file);
-        file.on("finish", () => file.close(() => resolve()));
-        file.on("error", reject);
+        const chunks = [];
+        res.on("data", (c) => chunks.push(c));
+        res.on("end", () => resolve(Buffer.concat(chunks)));
+        res.on("error", reject);
       })
       .on("error", reject);
   });
 }
 
+// Parse one entry out of a coreutils-style SHA256SUMS. Written on Linux, read
+// here on every platform, so tolerate CRLF and the "*" binary-mode marker.
+function digestFor(sumsText, asset) {
+  for (const line of sumsText.split(/\r?\n/)) {
+    const parts = line.trim().split(/\s+/);
+    if (parts.length === 2 && parts[1].replace(/^\*/, "") === asset)
+      return parts[0].toLowerCase();
+  }
+  return null;
+}
+
 // 3. Download the matching release asset and extract it (cached per version).
 async function fromDownload() {
   const dir = cacheDir();
@@ -102,26 +98,69 @@ async function fromDownload() {
   if (fs.existsSync(bin)) return bin;
 
   fs.mkdirSync(dir, { recursive: true });
-  const archive = path.join(dir, plat.asset);
-  const url = `https://github.com/${REPO}/releases/download/v${VERSION}/${plat.asset}`;
+  const base = `https://github.com/${REPO}/releases/download/v${VERSION}`;
+  const url = `${base}/${plat.asset}`;
+
+  // Fail closed: fetch the expected digest first, so a release without one is
+  // refused before anything is downloaded rather than after.
+  let sums;
+  try {
+    sums = (await httpsGet(`${base}/SHA256SUMS`)).toString("utf8");
+  } catch (e) {
+    // Only a 404 means the release genuinely lacks checksums. Reporting a
+    // proxy or DNS failure as "our release is broken" sends users to file
+    // issues against a release that is fine.
+    throw new Error(
+      /^HTTP 404 /.test(e.message)
+        ? `release v${VERSION} publishes no SHA256SUMS, so ${plat.asset} cannot be verified`
+        : `could not fetch SHA256SUMS for v${VERSION}: ${e.message}`
+    );
+  }
+  const expected = digestFor(sums, plat.asset);
+  if (!expected) {
+    throw new Error(`${plat.asset} is not listed in SHA256SUMS for v${VERSION}`);
+  }
 
   // stderr only: stdout is the MCP (JSON-RPC) channel.
   console.error(`crw-mcp: downloading ${plat.asset} (v${VERSION})...`);
-  await httpsGet(url, archive);
-
-  const tarArgs = plat.asset.endsWith(".zip")
-    ? ["-xf", archive, "-C", dir]
-    : ["-xzf", archive, "-C", dir];
-  const x = spawnSync("tar", tarArgs, { stdio: "inherit" });
-  if (x.status !== 0) {
-    throw new Error(`failed to extract ${plat.asset} (tar exit ${x.status})`);
-  }
-  fs.rmSync(archive, { force: true });
+  const data = await httpsGet(url);
 
-  if (!fs.existsSync(bin)) {
-    throw new Error(`binary ${binName} not found in ${plat.asset}`);
+  const actual = crypto.createHash("sha256").update(data).digest("hex");
+  if (actual !== expected) {
+    throw new Error(
+      `checksum mismatch for ${plat.asset}: expected ${expected}, got ${actual}. ` +
+        `Refusing to run it.`
+    );
+  }
+  // Stage per process, so two cold starts (MCP clients do spawn several at
+  // once on first configure) cannot truncate each other's archive or delete
+  // each other's extraction. The binary is moved in only once it is whole:
+  // `tar` applies the archive's mode from the first byte and the cache hit
+  // above gates on bare existsSync, so an interrupted extraction (Ctrl-C, disk
+  // full, OOM) would otherwise leave a truncated executable that every later
+  // run spawns. Python needs no staging: it writes the member itself, so the
+  // exec bit only arrives after the copy and its os.access(X_OK) gate rejects
+  // a partial file and re-downloads.
+  const stage = fs.mkdtempSync(path.join(dir, ".stage-"));
+  const archive = path.join(stage, plat.asset);
+  try {
+    fs.writeFileSync(archive, data);
+    const tarArgs = plat.asset.endsWith(".zip")
+      ? ["-xf", archive, "-C", stage]
+      : ["-xzf", archive, "-C", stage];
+    const x = spawnSync("tar", tarArgs, { stdio: "inherit" });
+    if (x.status !== 0) {
+      throw new Error(`failed to extract ${plat.asset} (tar exit ${x.status})`);
+    }
+    const staged = path.join(stage, binName);
+    if (!fs.existsSync(staged)) {
+      throw new Error(`binary ${binName} not found in ${plat.asset}`);
+    }
+    if (process.platform !== "win32") fs.chmodSync(staged, 0o755);
+    fs.renameSync(staged, bin);
+  } finally {
+    fs.rmSync(stage, { recursive: true, force: true });
   }
-  if (process.platform !== "win32") fs.chmodSync(bin, 0o755);
   return bin;
 }
 
@@ -129,7 +168,29 @@ async function resolveBinary() {
   return fromEnv() || fromPackage() || (await fromDownload());
 }
 
-resolveBinary()
+module.exports = { digestFor, fromDownload, cacheDir, plat, binName };
+
+// Only run when invoked as the CLI, so tests can require this file.
+if (require.main === module) {
+  // Handle install/init subcommands before delegating to the Rust binary.
+  // `init` = skill only; `install` = skill + MCP server. Both are file copies
+  // that need no binary, so they must run before the platform check: a 32-bit
+  // Pi or FreeBSD user can still install the skill.
+  if (process.argv[2] === "init") {
+    require("./init.js");
+    process.exit(0);
+  }
+  if (process.argv[2] === "install") {
+    require("./install.js");
+    process.exit(0);
+  }
+  if (!plat) {
+    console.error(
+      `crw-mcp: unsupported platform ${key}. Supported: ${Object.keys(PLATFORMS).join(", ")}`
+    );
+    process.exit(1);
+  }
+  resolveBinary()
   .then((bin) => {
     const result = spawnSync(bin, process.argv.slice(2), { stdio: "inherit" });
     process.exit(result.status ?? 1);
@@ -142,3 +203,4 @@ resolveBinary()
     );
     process.exit(1);
   });
+}

package/package.json

@@ -1,20 +1,23 @@
 {
   "name": "crw-mcp",
-  "version": "0.27.0",
+  "version": "0.28.0",
   "description": "MCP server for CRW web scraper — scrape, crawl, map, extract, search, and PDF-parse tools for AI agents",
   "license": "AGPL-3.0",
-  "homepage": "https://github.com/us/crw",
+  "author": "fastCRW <hello@fastcrw.com>",
+  "homepage": "https://fastcrw.com",
   "repository": {
     "type": "git",
     "url": "https://github.com/us/crw.git"
   },
+  "bugs": {
+    "url": "https://github.com/us/crw/issues"
+  },
   "keywords": [
     "mcp",
     "web-scraping",
     "ai-agent",
     "claude",
     "cursor",
-    "firecrawl",
     "crawl",
     "scrape"
   ],
@@ -24,6 +27,9 @@
   "engines": {
     "node": ">=18"
   },
+  "scripts": {
+    "test": "node --test test/*.test.js"
+  },
   "files": [
     "bin/crw-mcp.js",
     "bin/init.js",
@@ -32,9 +38,9 @@
     "skills/SKILL.md"
   ],
   "optionalDependencies": {
-    "crw-mcp-darwin-x64": "0.27.0",
-    "crw-mcp-darwin-arm64": "0.27.0",
-    "crw-mcp-linux-x64": "0.27.0",
-    "crw-mcp-linux-arm64": "0.27.0"
+    "crw-mcp-darwin-x64": "0.28.0",
+    "crw-mcp-darwin-arm64": "0.28.0",
+    "crw-mcp-linux-x64": "0.28.0",
+    "crw-mcp-linux-arm64": "0.28.0"
   }
 }