mcpscraper-cli 0.1.0

package/README.md

@@ -0,0 +1,97 @@
+# mcpscraper-cli
+
+Command-line interface for [mcpscraper.dev](https://mcpscraper.dev) and [memory.mcpscraper.dev](https://memory.mcpscraper.dev).
+
+This is a **curated subset** of the full API — search, scrape, crawl, map, Maps search, and a couple of memory commands. It is not full API coverage. For every operation (40 scraper endpoints, 85 memory tools), use [`mcpscraper-sdk`](../scraper) and [`mcpscraper-memory-sdk`](../memory) directly.
+
+## Install
+
+```bash
+npm install -g mcpscraper-cli
+```
+
+## Auth
+
+```bash
+export MCPSCRAPER_API_KEY=sk_live_your_key
+```
+
+or pass `--api-key <key>` on any command.
+
+## Commands
+
+### `mcpscraper search <query>`
+
+```bash
+$ mcpscraper search "best roofing companies denver"
+1. TOP 10 BEST Roofing Companies in Denver, CO
+   https://www.yelp.com/search?find_desc=Roofing+Companies&find_loc=Denver%2C+CO
+2. Recomendations for roofing company in Denver - ASAP
+   https://www.reddit.com/r/Denver/comments/1kh2gd3/...
+```
+
+Options: `--location <location>`, `--json`.
+
+### `mcpscraper scrape <url>`
+
+```bash
+$ mcpscraper scrape https://example.com --deposit-to-vault --vault research
+Example Domain
+This domain is for use in illustrative examples...
+
+Vault deposit: saved to research
+```
+
+Options: `--screenshot`, `--deposit-to-vault`, `--vault <name>`, `--json`.
+
+### `mcpscraper crawl <url>`
+
+Crawls and extracts every page of a site. Options: `--max-pages <n>` (default 100), `--json`.
+
+### `mcpscraper map <url>`
+
+Discovers all URLs on a site (no extraction). Options: `--json`.
+
+### `mcpscraper maps-search <query> --location <location>`
+
+```bash
+$ mcpscraper maps-search "roofers" --location "Denver, CO"
+1. Forest Roofs (4.9★)
+   701 Birch Street
+2. Mike & Sons Roofing (4.8★)
+   ...
+```
+
+Options: `--json`.
+
+### `mcpscraper memory search <query>`
+
+Semantic search across your mcp-memory vaults, dispatched through your mcpscraper.dev API key (no separate memory key needed).
+
+```bash
+$ mcpscraper memory search "roofing warranty terms"
+[0.83] note:competitor-research/roofing-warranties.md
+  Standard roofing warranty terms cover materials for 25-30 years...
+```
+
+### `mcpscraper memory list-vaults`
+
+```bash
+$ mcpscraper memory list-vaults
+Ideas (2 notes, owner)
+Inspiration (2 notes, owner)
+Knowledge (3 notes, owner)
+```
+
+## Every command supports `--json`
+
+```bash
+$ mcpscraper map https://example.com --json
+{
+  "urls": ["https://example.com/", "https://example.com/about"]
+}
+```
+
+## See also
+
+[Repo README](../../README.md) (multi-language examples with real sample output) · [`mcpscraper-sdk`](../scraper) · [`mcpscraper-memory-sdk`](../memory) · Python: [`mcpscraper-sdk`](../scraper-python) · [`mcpscraper-memory-sdk`](../memory-python)

package/dist/bin/cli.js

@@ -0,0 +1,172 @@
+#!/usr/bin/env node
+
+// src/cli.ts
+import { Command } from "commander";
+import { fileURLToPath } from "url";
+import { ScraperClient, ScraperApiError } from "mcpscraper-sdk";
+
+// src/version.ts
+var CLI_VERSION = "0.1.0";
+
+// src/cli.ts
+function printResult(result, json, summarize) {
+  if (json) {
+    console.log(JSON.stringify(result, null, 2));
+  } else {
+    summarize(result);
+  }
+}
+async function run(fn) {
+  try {
+    await fn();
+  } catch (err) {
+    if (err instanceof ScraperApiError) {
+      console.error(`Error${err.code ? ` [${err.code}]` : ""}: ${err.message}`);
+    } else {
+      console.error(err instanceof Error ? err.message : String(err));
+    }
+    process.exitCode = 1;
+  }
+}
+function createProgram(fetchImpl = globalThis.fetch) {
+  function resolveApiKey(cmd) {
+    const opts = cmd.optsWithGlobals();
+    const key = opts.apiKey ?? process.env.MCPSCRAPER_API_KEY;
+    if (!key) {
+      console.error("Missing API key. Pass --api-key <key> or set MCPSCRAPER_API_KEY.");
+      process.exit(1);
+    }
+    return key;
+  }
+  function client(cmd) {
+    return new ScraperClient({ apiKey: resolveApiKey(cmd), fetch: fetchImpl });
+  }
+  const program = new Command();
+  program.name("mcpscraper").description("CLI for mcpscraper.dev and memory.mcpscraper.dev \u2014 a curated subset of the full API, see mcpscraper-sdk for full coverage").version(CLI_VERSION).option("--api-key <key>", "mcpscraper.dev API key (or set MCPSCRAPER_API_KEY)");
+  program.command("search <query>").description("Run a Google SERP search").option("--location <location>", "geographic location to search from").option("--json", "print raw JSON").action(async (query, opts, cmd) => {
+    await run(async () => {
+      const result = await client(cmd).searchSerp({
+        query,
+        location: opts.location,
+        depth: 4,
+        maxQuestions: 30,
+        gl: "us",
+        hl: "en",
+        device: "desktop",
+        proxyMode: "none",
+        pages: 1,
+        debug: false
+      });
+      printResult(result, Boolean(opts.json), (r) => {
+        const organic = r.result?.organicResults ?? [];
+        for (const item of organic) {
+          console.log(`${item.position}. ${item.title}
+   ${item.url}`);
+        }
+      });
+    });
+  });
+  program.command("scrape <url>").description("Extract a single page (markdown, headings, optional screenshot/vault deposit)").option("--screenshot", "capture a full-page screenshot").option("--deposit-to-vault", "save the full page into your mcp-memory vault").option("--vault <name>", "vault name for --deposit-to-vault").option("--json", "print raw JSON").action(async (url, opts, cmd) => {
+    await run(async () => {
+      const result = await client(cmd).extractUrl({
+        url,
+        screenshot: Boolean(opts.screenshot),
+        screenshotDevice: "desktop",
+        extractBranding: false,
+        downloadMedia: false,
+        allowLocal: false,
+        depositToVault: Boolean(opts.depositToVault),
+        vaultName: opts.vault
+      });
+      printResult(result, Boolean(opts.json), (r) => {
+        const page = r;
+        console.log(page.title ?? "(untitled)");
+        console.log((page.bodyMarkdown ?? "").slice(0, 500));
+        if (page.memory) console.log(`
+Vault deposit: ${page.memory.deposited ? `saved to ${page.memory.vault}` : "failed"}`);
+      });
+    });
+  });
+  program.command("crawl <url>").description("Crawl and extract every page of a site").option("--max-pages <n>", "maximum pages to crawl", (v) => parseInt(v, 10)).option("--json", "print raw JSON").action(async (url, opts, cmd) => {
+    await run(async () => {
+      const result = await client(cmd).extractSite({
+        url,
+        maxPages: opts.maxPages ?? 100,
+        background: false,
+        rotateProxies: false,
+        rotateProxyEvery: 30,
+        browserFallback: false,
+        kernelFallback: false
+      });
+      printResult(result, Boolean(opts.json), (r) => {
+        const pages = r.pages ?? [];
+        console.log(`Crawled ${pages.length} page(s).`);
+      });
+    });
+  });
+  program.command("map <url>").description("Discover all URLs on a site").option("--json", "print raw JSON").action(async (url, opts, cmd) => {
+    await run(async () => {
+      const result = await client(cmd).mapSiteUrls({
+        url,
+        maxUrls: 500,
+        concurrency: 12,
+        browserFallback: false,
+        kernelFallback: false
+      });
+      printResult(result, Boolean(opts.json), (r) => {
+        const urls = r.urls ?? [];
+        for (const u of urls) console.log(u);
+      });
+    });
+  });
+  program.command("maps-search <query>").description("Search Google Maps for local businesses").requiredOption("--location <location>", 'e.g. "Denver, CO"').option("--json", "print raw JSON").action(async (query, opts, cmd) => {
+    await run(async () => {
+      const result = await client(cmd).maps.search({
+        query,
+        location: opts.location,
+        gl: "us",
+        hl: "en",
+        maxResults: 10,
+        proxyMode: "location",
+        debug: false
+      });
+      printResult(result, Boolean(opts.json), (r) => {
+        const results = r.results ?? [];
+        for (const item of results) {
+          console.log(`${item.position}. ${item.name}${item.rating ? ` (${item.rating}\u2605)` : ""}
+   ${item.address ?? ""}`);
+        }
+      });
+    });
+  });
+  const memory = program.command("memory").description("Memory tools, dispatched through your mcpscraper.dev API key (see mcpscraper-memory-sdk for the full 85-tool surface)");
+  memory.command("search <query>").description("Semantic search across your mcp-memory vaults").option("--json", "print raw JSON").action(async (query, opts, cmd) => {
+    await run(async () => {
+      const result = await client(cmd).memoryTools.memory.search({ query });
+      printResult(result, Boolean(opts.json), (r) => {
+        const results = r.results ?? [];
+        for (const item of results) {
+          console.log(`[${item.score.toFixed(2)}] ${item.source}
+  ${item.text.slice(0, 160)}`);
+        }
+      });
+    });
+  });
+  memory.command("list-vaults").description("List your mcp-memory vaults").option("--json", "print raw JSON").action(async (opts, cmd) => {
+    await run(async () => {
+      const result = await client(cmd).memoryTools.vaults.listVaults({});
+      printResult(result, Boolean(opts.json), (r) => {
+        const vaults = r.vaults ?? [];
+        for (const v of vaults) console.log(`${v.vault} (${v.notes} notes, ${v.role})`);
+      });
+    });
+  });
+  return program;
+}
+var isMain = process.argv[1] === fileURLToPath(import.meta.url);
+if (isMain) {
+  await createProgram().parseAsync(process.argv);
+}
+export {
+  createProgram
+};

package/package.json

@@ -0,0 +1,34 @@
+{
+  "name": "mcpscraper-cli",
+  "version": "0.1.0",
+  "description": "Command-line interface for mcpscraper.dev and memory.mcpscraper.dev",
+  "license": "MIT",
+  "type": "module",
+  "bin": {
+    "mcpscraper": "dist/bin/cli.js"
+  },
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "scripts": {
+    "build": "tsup src/cli.ts --format esm --out-dir dist/bin",
+    "typecheck": "tsc --noEmit",
+    "test": "tsx --test test/*.test.ts"
+  },
+  "dependencies": {
+    "mcpscraper-sdk": "^0.3.0",
+    "commander": "^12.0.0"
+  },
+  "devDependencies": {
+    "@types/node": "^22.0.0"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/VilovietaSEO/mcpscraper-sdk.git",
+    "directory": "packages/cli"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}