@scrapecreators/cli 1.0.0

package/bin/check-lockfile-sync.js

@@ -0,0 +1,42 @@
+#!/usr/bin/env node
+import { readFileSync } from "fs";
+
+function readJson(path) {
+  return JSON.parse(readFileSync(path, "utf-8"));
+}
+
+function normalizeDeps(deps) {
+  return Object.fromEntries(Object.entries(deps || {}).sort(([a], [b]) => a.localeCompare(b)));
+}
+
+const pkg = readJson(new URL("../package.json", import.meta.url));
+const lock = readJson(new URL("../package-lock.json", import.meta.url));
+
+const pkgDeps = normalizeDeps(pkg.dependencies);
+const lockDeps = normalizeDeps(lock?.packages?.[""]?.dependencies);
+
+const errors = [];
+
+for (const [name, version] of Object.entries(pkgDeps)) {
+  if (!(name in lockDeps)) {
+    errors.push(`missing in lockfile root deps: ${name}`);
+    continue;
+  }
+  if (lockDeps[name] !== version) {
+    errors.push(`version mismatch for ${name}: package.json=${version}, package-lock.json=${lockDeps[name]}`);
+  }
+}
+
+for (const name of Object.keys(lockDeps)) {
+  if (!(name in pkgDeps)) {
+    errors.push(`extra lockfile root dependency not in package.json: ${name}`);
+  }
+}
+
+if (errors.length > 0) {
+  console.error("lockfile check failed:");
+  for (const err of errors) console.error(`- ${err}`);
+  process.exit(1);
+}
+
+console.log("lockfile check passed.");

package/bin/scrapecreators.js

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+import { run } from "../src/cli.js";
+
+run(process.argv);

package/package.json

@@ -0,0 +1,48 @@
+{
+  "name": "@scrapecreators/cli",
+  "version": "1.0.0",
+  "description": "CLI for the ScrapeCreators API — scrape 27+ social media platforms from the terminal",
+  "type": "module",
+  "bin": {
+    "scrapecreators": "./bin/scrapecreators.js"
+  },
+  "scripts": {
+    "start": "node bin/scrapecreators.js",
+    "test": "vitest run",
+    "check:lockfile": "node bin/check-lockfile-sync.js"
+  },
+  "dependencies": {
+    "@clack/prompts": "0.10.1",
+    "chalk": "5.6.2",
+    "cli-table3": "0.6.5",
+    "commander": "13.1.0",
+    "conf": "13.1.0",
+    "ora": "8.2.0"
+  },
+  "engines": {
+    "node": ">=20"
+  },
+  "files": [
+    "bin/",
+    "src/",
+    "api-config/",
+    "!api-config/withIcons.js"
+  ],
+  "keywords": [
+    "scraping",
+    "social-media",
+    "tiktok",
+    "instagram",
+    "youtube",
+    "cli"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/ScrapeCreators/scrapecreators-cli"
+  },
+  "homepage": "https://scrapecreators.com/",
+  "license": "MIT",
+  "devDependencies": {
+    "vitest": "^4.1.4"
+  }
+}

package/src/api-client.js

@@ -0,0 +1,71 @@
+const API_BASE = "https://api.scrapecreators.com";
+
+// 50 MB — generous for JSON API responses, prevents memory exhaustion
+const MAX_RESPONSE_BYTES = 50 * 1024 * 1024;
+
+export { MAX_RESPONSE_BYTES };
+
+export async function readBodyWithLimit(res) {
+  const contentLength = parseInt(res.headers.get("content-length"), 10);
+  if (contentLength > MAX_RESPONSE_BYTES) {
+    throw new Error(`Response Content-Length (${contentLength}) exceeds ${MAX_RESPONSE_BYTES} byte limit`);
+  }
+
+  const chunks = [];
+  let total = 0;
+  for await (const chunk of res.body) {
+    total += chunk.length;
+    if (total > MAX_RESPONSE_BYTES) {
+      throw new Error(`Response body exceeded ${MAX_RESPONSE_BYTES} byte limit`);
+    }
+    chunks.push(chunk);
+  }
+  return Buffer.concat(chunks).toString("utf-8");
+}
+
+export async function callApi(apiKey, method, path, params = {}) {
+  const url = new URL(`${API_BASE}${path}`);
+
+  if (method === "GET" && params) {
+    for (const [key, value] of Object.entries(params)) {
+      if (value !== undefined && value !== null && value !== "") {
+        url.searchParams.set(key, String(value));
+      }
+    }
+  }
+
+  const options = {
+    method,
+    headers: {
+      "x-api-key": apiKey,
+      accept: "application/json",
+    },
+  };
+
+  if (method !== "GET" && params && Object.keys(params).length > 0) {
+    options.headers["content-type"] = "application/json";
+    options.body = JSON.stringify(params);
+  }
+
+  options.signal = AbortSignal.timeout(60_000);
+
+  const start = Date.now();
+  const res = await fetch(url.toString(), options);
+  const elapsed = Date.now() - start;
+  const text = await readBodyWithLimit(res);
+
+  let data;
+  try {
+    data = JSON.parse(text);
+  } catch {
+    data = text;
+  }
+
+  return {
+    ok: res.ok,
+    status: res.status,
+    data,
+    elapsed,
+    url: url.toString(),
+  };
+}

package/src/auth.js

@@ -0,0 +1,27 @@
+import config from "./config.js";
+
+export function resolveApiKey(opts = {}) {
+  if (opts.apiKey) return opts.apiKey;
+  const stored = config.get("apiKey");
+  if (stored) return stored;
+  if (process.env.SCRAPECREATORS_API_KEY) return process.env.SCRAPECREATORS_API_KEY;
+  return null;
+}
+
+export function storeApiKey(key) {
+  config.set("apiKey", key);
+}
+
+export function clearApiKey() {
+  config.delete("apiKey");
+}
+
+export function getStoredApiKey() {
+  return config.get("apiKey") || null;
+}
+
+export function maskKey(key) {
+  if (!key) return "(not set)";
+  if (key.length <= 12) return key.slice(0, 2) + "…" + key.slice(-2);
+  return key.slice(0, 6) + "…" + key.slice(-4);
+}

package/src/cli.js

@@ -0,0 +1,78 @@
+import { Command } from "commander";
+import { registerApiCommands } from "./command-registry.js";
+import { authLogin, authStatus, authLogout } from "./commands/auth.js";
+import { configSet, configGet, configList } from "./commands/config.js";
+import { balanceCommand } from "./commands/balance.js";
+import { listCommand } from "./commands/list.js";
+import { agentAddCommand } from "./commands/agent.js";
+import { runInteractive } from "./interactive.js";
+
+export function run(argv) {
+  const program = new Command();
+
+  program
+    .name("scrapecreators")
+    .description("CLI for the ScrapeCreators API — scrape 27+ social media platforms")
+    .version("1.0.0")
+    .option("--api-key <key>", "API key (overrides env and config)")
+    .option("--format <format>", "output format: json, table, csv, markdown", "auto")
+    .option("--json", "shorthand for compact JSON (default)")
+    .option("--pretty", "pretty-print JSON with indentation")
+    .option("--output <path>", "save response to file, print path only")
+    .option("--clean", "strip noisy fields: booleans, empty values, settings (any format)")
+    .option("--no-color", "disable ANSI colors")
+    .option("--verbose", "show request URL, timing, credits used");
+
+  const globalOptsGetter = () => program.opts();
+
+  // --- API commands (dynamic from api-config) ---
+  registerApiCommands(program, globalOptsGetter);
+
+  // --- auth ---
+  const authCmd = program.command("auth").description("manage authentication");
+  authCmd.command("login").description("set your API key").action(() => authLogin());
+  authCmd.command("status").description("show current auth status").action(() => authStatus());
+  authCmd.command("logout").description("remove stored API key").action(() => authLogout());
+
+  // --- balance ---
+  program
+    .command("balance")
+    .description("check credit balance")
+    .action(() => balanceCommand(globalOptsGetter()));
+
+  // --- config ---
+  const configCmd = program.command("config").description("manage CLI configuration");
+  configCmd
+    .command("set <key> <value>")
+    .description("set a config value")
+    .action((key, value) => configSet(key, value));
+  configCmd
+    .command("get <key>")
+    .description("get a config value")
+    .action((key) => configGet(key));
+  configCmd
+    .command("list")
+    .description("show all config values")
+    .action(() => configList());
+
+  // --- list ---
+  program
+    .command("list [platform]")
+    .description("list available platforms and endpoints")
+    .action((platform) => listCommand(platform));
+
+  // --- agent-first: agent add ---
+  const agentCmd = program.command("agent").description("configure AI agent integrations");
+  agentCmd
+    .command("add <target>")
+    .description("write MCP config into an agent (cursor, claude, codex)")
+    .action((target) => agentAddCommand(target, globalOptsGetter()));
+
+  // if no args and stdin is a TTY, launch interactive mode
+  if (argv.length <= 2 && process.stdin.isTTY) {
+    runInteractive(globalOptsGetter());
+    return;
+  }
+
+  program.parse(argv);
+}

package/src/command-registry.js

@@ -0,0 +1,156 @@
+import { apis } from "../api-config/apis.js";
+import { handleApiCommand } from "./commands/api.js";
+
+function buildParamDescription(p) {
+  const base = p.description || "";
+  if (p.placeholder === undefined || p.placeholder === "") return base;
+  const val = String(p.placeholder);
+  if (p.type === "boolean" || val === "null" || val === "false" || val === "true") return base;
+  return `${base} (e.g. ${val})`;
+}
+
+const toolDefs = apis.flatMap((api) =>
+  api.endpoints.map((ep) => ({
+    name: ep.path.replace(/^\//, "").replace(/\//g, "_").replace(/-/g, "_"),
+    title: `${api.name} - ${ep.name}`,
+    description: ep.fullDescription || ep.description,
+    method: ep.method,
+    path: ep.path,
+    params: (ep.params || []).map((p) => ({
+      name: p.name,
+      type: p.type,
+      required: !!p.required,
+      description: buildParamDescription(p),
+      ...(p.type === "select" && p.options ? { options: p.options } : {}),
+      ...(p.default !== undefined ? { default: p.default } : {}),
+    })),
+    ...(ep.paginationField ? { paginationField: ep.paginationField } : {}),
+    ...(ep.credits ? { credits: ep.credits } : {}),
+  }))
+);
+
+// /v1/tiktok/profile -> { platform: "tiktok", action: "profile" }
+// /v3/tiktok/profile/videos -> { platform: "tiktok", action: "profile-videos" }
+// /v1/facebook/adLibrary/search/ads -> { platform: "facebook", action: "adlibrary-search-ads" }
+// /v1/linktree -> { platform: "linktree", action: "get" }
+// /v1/detect-age-gender -> { platform: "detect", action: "age-gender" }
+// /v1/credit-balance -> { platform: "credit", action: "balance" }
+function parseToolPath(path) {
+  // handle hyphenated single-segment paths like /v1/detect-age-gender, /v1/credit-balance
+  const stripped = path.replace(/^\//, "");
+  const parts = stripped.split("/");
+
+  if (parts.length === 2 && parts[1].includes("-")) {
+    const idx = parts[1].indexOf("-");
+    return {
+      platform: parts[1].slice(0, idx),
+      action: parts[1].slice(idx + 1),
+    };
+  }
+
+  const platform = parts[1];
+  const actionParts = parts.slice(2);
+  // if no action segments (e.g. /v1/linktree), use "get" as default
+  const action = actionParts.length > 0
+    ? actionParts.join("-").toLowerCase()
+    : "get";
+  return { platform, action };
+}
+
+// pick latest version when multiple exist for same platform+action
+function deduplicateTools(tools) {
+  const map = new Map();
+  for (const tool of tools) {
+    const { platform, action } = parseToolPath(tool.path);
+    const key = `${platform}/${action}`;
+    const existing = map.get(key);
+    if (!existing) {
+      map.set(key, tool);
+    } else {
+      const existingVersion = parseInt(existing.path.match(/^\/v(\d+)/)?.[1] || "1");
+      const newVersion = parseInt(tool.path.match(/^\/v(\d+)/)?.[1] || "1");
+      if (newVersion > existingVersion) {
+        map.set(key, tool);
+      }
+    }
+  }
+  return [...map.values()];
+}
+
+function snakeToCli(name) {
+  return name.replace(/_/g, "-");
+}
+
+function cliToCamel(str) {
+  return str.replace(/-([a-z])/g, (_, c) => c.toUpperCase());
+}
+
+export function registerApiCommands(program, globalOptsGetter) {
+  const tools = deduplicateTools(toolDefs);
+
+  // group by platform
+  const platforms = new Map();
+  for (const tool of tools) {
+    const { platform, action } = parseToolPath(tool.path);
+    if (!platforms.has(platform)) platforms.set(platform, []);
+    platforms.get(platform).push({ ...tool, _action: action });
+  }
+
+  for (const [platform, tools] of platforms) {
+    // skip credit/balance - handled as `scrapecreators balance`
+    if (platform === "credit") continue;
+
+    const platCmd = program
+      .command(platform)
+      .description(`${platform} endpoints`);
+
+    for (const tool of tools) {
+      const actionCmd = platCmd
+        .command(tool._action)
+        .description(tool.title);
+
+      for (const param of tool.params) {
+        const flag = snakeToCli(param.name);
+        const camel = cliToCamel(flag);
+
+        if (param.type === "boolean") {
+          actionCmd.option(`--${flag}`, param.description);
+        } else if (param.type === "select" && param.options?.length) {
+          actionCmd.option(
+            `--${flag} <value>`,
+            `${param.description} (${param.options.join("|")})`,
+          );
+        } else {
+          const bracket = param.required ? `<${flag}>` : `[${flag}]`;
+          actionCmd.option(`--${flag} ${bracket}`, param.description);
+        }
+      }
+
+      actionCmd.action(async (cmdOpts) => {
+        const globalOpts = globalOptsGetter();
+        await handleApiCommand(tool, cmdOpts, globalOpts);
+      });
+    }
+  }
+
+  return platforms;
+}
+
+export function getToolDefs() {
+  return toolDefs;
+}
+
+export function getDeduplicatedTools() {
+  return deduplicateTools(toolDefs);
+}
+
+export function getPlatformMap() {
+  const tools = deduplicateTools(toolDefs);
+  const platforms = new Map();
+  for (const tool of tools) {
+    const { platform, action } = parseToolPath(tool.path);
+    if (!platforms.has(platform)) platforms.set(platform, []);
+    platforms.get(platform).push({ ...tool, _action: action });
+  }
+  return platforms;
+}