mcpscraper-sdk 0.3.0

package/README.md

@@ -0,0 +1,77 @@
+# mcpscraper-sdk
+
+Official TypeScript/JavaScript client for the [mcpscraper.dev](https://mcpscraper.dev) REST API — SERP search, People-Also-Ask harvesting, single-page and whole-site extraction, YouTube, Facebook/Google Ads Transparency, Instagram, Reddit, video breakdown, Google Maps, and directory/rank-tracking workflows.
+
+This is a thin HTTP client generated against [`../../contracts/scraper.openapi.yaml`](../../contracts/scraper.openapi.yaml), the public contract for the hosted API. It contains no scraping, proxy, or billing logic — only typed request/response plumbing.
+
+## Install
+
+```bash
+npm install mcpscraper-sdk
+```
+
+## Usage
+
+```ts
+import { ScraperClient, ScraperApiError } from 'mcpscraper-sdk'
+
+const client = new ScraperClient({ apiKey: process.env.MCPSCRAPER_API_KEY! })
+
+const serp = await client.searchSerp({ query: 'roof repair Denver' })
+
+try {
+  const places = await client.maps.search({ query: 'roofers', location: 'Denver, CO' })
+  console.log(places)
+} catch (err) {
+  if (err instanceof ScraperApiError && err.isInsufficientBalance()) {
+    console.error(`Need ${err.body.required_credits} credits, have ${err.body.balance_credits}. Top up: ${err.body.topup_url}`)
+  } else {
+    throw err
+  }
+}
+```
+
+## Errors
+
+Every non-2xx response throws a `ScraperApiError` with `status`, `code`, and the raw response `body`. Two cases have typed narrowing helpers:
+
+- `err.isInsufficientBalance()` — narrows `err.body` to `{ balance_credits, required_credits, topup_url, ... }`.
+- `err.isConcurrencyLimitExceeded()` — narrows `err.body` to `{ active, limit, retryable, ... }`.
+
+## API surface
+
+Core operations are flat on the client: `searchSerp`, `harvestPaa`, `extractUrl`, `mapSiteUrls`, `extractSite`, `auditSite`, `getExtractSiteStatus`, `listJobs`, `getJob`, `getHistory`, `getLedger`.
+
+Everything else is namespaced by product area, matching the OpenAPI spec's tags: `client.youtube`, `client.screenshot`, `client.facebook`, `client.googleAds`, `client.instagram`, `client.reddit`, `client.video`, `client.maps`, `client.directory`, `client.serpIntelligence`, `client.workflows`.
+
+## Scrape → memory vault
+
+`extractUrl` accepts `depositToVault: true` (optionally with `vaultName`) to embed the full scraped body server-side into your mcp-memory vault instead of returning it inline — the response's `memory` field reports `{ deposited, vault, noteId, path, chunks }` (or falls back to a temporary `fileUrl` download if the vault deposit itself fails):
+
+```ts
+const page = await client.extractUrl({ url: 'https://example.com/pricing', depositToVault: true, vaultName: 'competitors' })
+console.log(page.memory) // { deposited: true, vault: 'competitors', noteId: '...', path: '...', chunks: 4 }
+```
+
+## Memory tools, using only this API key
+
+`client.memoryTools` exposes every tool from [`mcpscraper-memory-sdk`](../memory) — access/keys, channels, memory search/CRUD, tables, vaults, facts, schedule, webhooks, video — dispatched through `POST /memory/mcp-call` with your mcpscraper.dev API key. No separate memory key needed; a memory identity is auto-provisioned for your account on first use:
+
+```ts
+const hits = await client.memoryTools.memory.search({ query: 'competitor pricing pages' })
+const vaults = await client.memoryTools.vaults.listVaults({})
+```
+
+Use `mcpscraper-memory-sdk`'s own `MemoryClient` instead if you already have a dedicated `mk_...` memory key and want to talk to `memory.mcpscraper.dev` directly.
+
+## Regenerating types
+
+`src/schema.ts` is generated from the OpenAPI spec and checked in. After editing `../../contracts/scraper.openapi.yaml`, regenerate with:
+
+```bash
+npm run generate
+```
+
+## See also
+
+[Repo README](../../README.md) (multi-language examples with real sample output) · [`mcpscraper-memory-sdk`](../memory) (Node, full 85-tool typed surface) · [`mcpscraper-sdk` on PyPI](../scraper-python) (Python) · [`mcpscraper-cli`](../cli)

package/dist/index.cjs

@@ -0,0 +1,371 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/index.ts
+var index_exports = {};
+__export(index_exports, {
+  ScraperApiError: () => ScraperApiError,
+  ScraperClient: () => ScraperClient
+});
+module.exports = __toCommonJS(index_exports);
+
+// src/errors.ts
+function readString(body, key) {
+  if (body && typeof body === "object" && key in body) {
+    const value = body[key];
+    if (typeof value === "string") return value;
+  }
+  return void 0;
+}
+var ScraperApiError = class extends Error {
+  status;
+  code;
+  body;
+  constructor(status, body) {
+    super(readString(body, "message") ?? `mcpscraper.dev API request failed with status ${status}`);
+    this.name = "ScraperApiError";
+    this.status = status;
+    this.code = readString(body, "error_code") ?? readString(body, "error");
+    this.body = body;
+  }
+  isInsufficientBalance() {
+    return this.code === "insufficient_balance";
+  }
+  isConcurrencyLimitExceeded() {
+    return this.code === "concurrency_limit_exceeded";
+  }
+  isStructuredError() {
+    return typeof readString(this.body, "error_type") === "string";
+  }
+};
+
+// src/client.ts
+var import_mcpscraper_memory_sdk = require("mcpscraper-memory-sdk");
+var Requester = class {
+  constructor(apiKey, baseUrl, fetchImpl) {
+    this.apiKey = apiKey;
+    this.baseUrl = baseUrl;
+    this.fetchImpl = fetchImpl;
+  }
+  apiKey;
+  baseUrl;
+  fetchImpl;
+  async call(method, path, body) {
+    const res = await this.fetchImpl(`${this.baseUrl}${path}`, {
+      method,
+      headers: {
+        "x-api-key": this.apiKey,
+        ...body !== void 0 ? { "content-type": "application/json" } : {}
+      },
+      body: body !== void 0 ? JSON.stringify(body) : void 0
+    });
+    const data = await res.json().catch(() => void 0);
+    if (!res.ok) throw new ScraperApiError(res.status, data);
+    return data;
+  }
+  async callRaw(method, path) {
+    const res = await this.fetchImpl(`${this.baseUrl}${path}`, { method, headers: { "x-api-key": this.apiKey } });
+    if (!res.ok) {
+      const data = await res.json().catch(() => void 0);
+      throw new ScraperApiError(res.status, data);
+    }
+    return res.arrayBuffer();
+  }
+};
+var YoutubeNamespace = class {
+  constructor(r) {
+    this.r = r;
+  }
+  r;
+  harvest(params) {
+    return this.r.call("POST", "/youtube/harvest", params);
+  }
+  transcribe(params) {
+    return this.r.call("POST", "/youtube/transcribe", params);
+  }
+};
+var ScreenshotNamespace = class {
+  constructor(r) {
+    this.r = r;
+  }
+  r;
+  capture(params) {
+    return this.r.call("POST", "/screenshot", params);
+  }
+};
+var FacebookNamespace = class {
+  constructor(r) {
+    this.r = r;
+  }
+  r;
+  ad(params) {
+    return this.r.call("POST", "/facebook/ad", params);
+  }
+  pageIntel(params) {
+    return this.r.call("POST", "/facebook/page-intel", params);
+  }
+  adTranscribe(params) {
+    return this.r.call("POST", "/facebook/transcribe", params);
+  }
+  videoTranscribe(params) {
+    return this.r.call("POST", "/facebook/video-transcribe", params);
+  }
+  search(params) {
+    return this.r.call("POST", "/facebook/search", params);
+  }
+  media(params) {
+    return this.r.call("POST", "/facebook/media", params);
+  }
+};
+var GoogleAdsNamespace = class {
+  constructor(r) {
+    this.r = r;
+  }
+  r;
+  search(params) {
+    return this.r.call("POST", "/google-ads/search", params);
+  }
+  pageIntel(params) {
+    return this.r.call("POST", "/google-ads/page-intel", params);
+  }
+  transcribe(params) {
+    return this.r.call("POST", "/google-ads/transcribe", params);
+  }
+};
+var InstagramNamespace = class {
+  constructor(r) {
+    this.r = r;
+  }
+  r;
+  profileContent(params) {
+    return this.r.call("POST", "/instagram/profile-content", params);
+  }
+  mediaDownload(params) {
+    return this.r.call("POST", "/instagram/media-download", params);
+  }
+};
+var RedditNamespace = class {
+  constructor(r) {
+    this.r = r;
+  }
+  r;
+  thread(params) {
+    return this.r.call("POST", "/reddit/thread", params);
+  }
+};
+var VideoNamespace = class {
+  constructor(r) {
+    this.r = r;
+  }
+  r;
+  analyze(params) {
+    return this.r.call("POST", "/video/analyze", params);
+  }
+  status(params) {
+    return this.r.call("POST", "/video/status", params);
+  }
+};
+var MapsNamespace = class {
+  constructor(r) {
+    this.r = r;
+  }
+  r;
+  search(params) {
+    return this.r.call("POST", "/maps/search", params);
+  }
+  place(params) {
+    return this.r.call("POST", "/maps/place", params);
+  }
+};
+var DirectoryNamespace = class {
+  constructor(r) {
+    this.r = r;
+  }
+  r;
+  run(params) {
+    return this.r.call("POST", "/directory/run", params);
+  }
+};
+var SerpIntelligenceNamespace = class {
+  constructor(r) {
+    this.r = r;
+  }
+  r;
+  capture(params) {
+    return this.r.call("POST", "/serp-intelligence/capture", params);
+  }
+  pageSnapshots(params) {
+    return this.r.call("POST", "/serp-intelligence/page-snapshots", params);
+  }
+};
+var WorkflowsNamespace = class {
+  constructor(r) {
+    this.r = r;
+  }
+  r;
+  listDefinitions() {
+    return this.r.call("GET", "/workflows/definitions");
+  }
+  run(params) {
+    return this.r.call("POST", "/workflows/run", params);
+  }
+  advanceStep(id) {
+    return this.r.call("POST", `/workflows/runs/${encodeURIComponent(id)}/step`);
+  }
+  listRuns() {
+    return this.r.call("GET", "/workflows/runs");
+  }
+  getRun(id) {
+    return this.r.call("GET", `/workflows/runs/${encodeURIComponent(id)}`);
+  }
+  getRunArtifact(id, artifactId) {
+    return this.r.callRaw(
+      "GET",
+      `/workflows/runs/${encodeURIComponent(id)}/artifacts/${encodeURIComponent(artifactId)}`
+    );
+  }
+  createSchedule(params) {
+    return this.r.call("POST", "/workflows/schedules", params);
+  }
+  listSchedules() {
+    return this.r.call("GET", "/workflows/schedules");
+  }
+  patchSchedule(id, params) {
+    return this.r.call("PATCH", `/workflows/schedules/${encodeURIComponent(id)}`, params);
+  }
+  deleteSchedule(id) {
+    return this.r.call("DELETE", `/workflows/schedules/${encodeURIComponent(id)}`);
+  }
+  runScheduleNow(id) {
+    return this.r.call("POST", `/workflows/schedules/${encodeURIComponent(id)}/run`);
+  }
+};
+var MemoryTools = class {
+  access;
+  capture;
+  channels;
+  facts;
+  graph;
+  library;
+  memory;
+  recall;
+  schedule;
+  storage;
+  tables;
+  tags;
+  vaults;
+  video;
+  webhooks;
+  constructor(callTool) {
+    this.access = new import_mcpscraper_memory_sdk.AccessNamespace(callTool);
+    this.capture = new import_mcpscraper_memory_sdk.CaptureNamespace(callTool);
+    this.channels = new import_mcpscraper_memory_sdk.ChannelsNamespace(callTool);
+    this.facts = new import_mcpscraper_memory_sdk.FactsNamespace(callTool);
+    this.graph = new import_mcpscraper_memory_sdk.GraphNamespace(callTool);
+    this.library = new import_mcpscraper_memory_sdk.LibraryNamespace(callTool);
+    this.memory = new import_mcpscraper_memory_sdk.MemoryNamespace(callTool);
+    this.recall = new import_mcpscraper_memory_sdk.RecallNamespace(callTool);
+    this.schedule = new import_mcpscraper_memory_sdk.ScheduleNamespace(callTool);
+    this.storage = new import_mcpscraper_memory_sdk.StorageNamespace(callTool);
+    this.tables = new import_mcpscraper_memory_sdk.TablesNamespace(callTool);
+    this.tags = new import_mcpscraper_memory_sdk.TagsNamespace(callTool);
+    this.vaults = new import_mcpscraper_memory_sdk.VaultsNamespace(callTool);
+    this.video = new import_mcpscraper_memory_sdk.VideoNamespace(callTool);
+    this.webhooks = new import_mcpscraper_memory_sdk.WebhooksNamespace(callTool);
+  }
+};
+var ScraperClient = class {
+  r;
+  youtube;
+  screenshot;
+  facebook;
+  googleAds;
+  instagram;
+  reddit;
+  video;
+  maps;
+  directory;
+  serpIntelligence;
+  workflows;
+  memoryTools;
+  constructor(options) {
+    this.r = new Requester(options.apiKey, options.baseUrl ?? "https://mcpscraper.dev", options.fetch ?? globalThis.fetch);
+    this.youtube = new YoutubeNamespace(this.r);
+    this.screenshot = new ScreenshotNamespace(this.r);
+    this.facebook = new FacebookNamespace(this.r);
+    this.googleAds = new GoogleAdsNamespace(this.r);
+    this.instagram = new InstagramNamespace(this.r);
+    this.reddit = new RedditNamespace(this.r);
+    this.video = new VideoNamespace(this.r);
+    this.maps = new MapsNamespace(this.r);
+    this.directory = new DirectoryNamespace(this.r);
+    this.serpIntelligence = new SerpIntelligenceNamespace(this.r);
+    this.workflows = new WorkflowsNamespace(this.r);
+    this.memoryTools = new MemoryTools(this.callMemoryTool.bind(this));
+  }
+  async callMemoryTool(toolName, args) {
+    const result = await this.r.call("POST", "/memory/mcp-call", {
+      toolName,
+      args: args ?? {}
+    });
+    if (result && typeof result === "object" && result.ok === false) {
+      const failure = result;
+      throw new ScraperApiError(200, { message: failure.error, ...failure });
+    }
+    return result;
+  }
+  searchSerp(params) {
+    return this.r.call("POST", "/harvest/sync", { ...params, serpOnly: true });
+  }
+  harvestPaa(params) {
+    return this.r.call("POST", "/harvest/sync", params);
+  }
+  extractUrl(params) {
+    return this.r.call("POST", "/extract-url", params);
+  }
+  mapSiteUrls(params) {
+    return this.r.call("POST", "/map-urls", params);
+  }
+  extractSite(params) {
+    return this.r.call("POST", "/extract-site", params);
+  }
+  auditSite(params) {
+    return this.extractSite(params);
+  }
+  getExtractSiteStatus(id) {
+    return this.r.call("GET", `/extract-site/status/${encodeURIComponent(id)}`);
+  }
+  listJobs() {
+    return this.r.call("GET", "/jobs");
+  }
+  getJob(id) {
+    return this.r.call("GET", `/jobs/${encodeURIComponent(id)}`);
+  }
+  getHistory() {
+    return this.r.call("GET", "/history");
+  }
+  getLedger() {
+    return this.r.call("GET", "/ledger");
+  }
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  ScraperApiError,
+  ScraperClient
+});