@scrapecreators/cli 1.0.0

package/src/commands/agent.js

@@ -0,0 +1,311 @@
+import {
+  appendFileSync,
+  chmodSync,
+  closeSync,
+  constants,
+  existsSync,
+  lstatSync,
+  mkdirSync,
+  openSync,
+  readFileSync,
+  renameSync,
+  realpathSync,
+  unlinkSync,
+  writeFileSync,
+} from "fs";
+import { dirname, isAbsolute, join, relative, resolve } from "path";
+import { homedir } from "os";
+import chalk from "chalk";
+import * as prompts from "@clack/prompts";
+import { resolveApiKey, storeApiKey } from "../auth.js";
+import { callApi } from "../api-client.js";
+
+const MCP_URL = "https://api.scrapecreators.com/mcp";
+
+const TARGETS = {
+  cursor: {
+    name: "Cursor",
+    configPath: () => resolve(process.cwd(), ".cursor", "mcp.json"),
+    basePath: () => process.cwd(),
+    buildConfig: (apiKey) => ({
+      mcpServers: {
+        scrapecreators: {
+          url: MCP_URL,
+          headers: { "x-api-key": apiKey },
+        },
+      },
+    }),
+  },
+  claude: {
+    name: "Claude Desktop",
+    configPath: () => {
+      if (process.platform === "win32") {
+        const appData = process.env.APPDATA;
+        if (!appData) return null;
+        return resolve(appData, "Claude", "claude_desktop_config.json");
+      }
+      return resolve(homedir(), ".claude", "claude_desktop_config.json");
+    },
+    basePath: () => {
+      if (process.platform === "win32") return process.env.APPDATA || null;
+      return homedir();
+    },
+    buildConfig: (apiKey) => ({
+      mcpServers: {
+        scrapecreators: {
+          url: MCP_URL,
+          headers: { "x-api-key": apiKey },
+        },
+      },
+    }),
+  },
+  codex: {
+    name: "Codex",
+    configPath: () => resolve(homedir(), ".codex", "mcp.json"),
+    basePath: () => homedir(),
+    buildConfig: (apiKey) => ({
+      mcpServers: {
+        scrapecreators: {
+          url: MCP_URL,
+          headers: { "x-api-key": apiKey },
+        },
+      },
+    }),
+  },
+};
+
+function isPlainObject(value) {
+  return !!value && typeof value === "object" && !Array.isArray(value);
+}
+
+function isWithinBase(basePath, candidatePath) {
+  const rel = relative(basePath, candidatePath);
+  return rel === "" || (!isAbsolute(rel) && !rel.startsWith(".."));
+}
+
+export function resolveConfigPath(rawPath, rawBasePath, targetName) {
+  if (!rawPath || !rawBasePath) {
+    console.error(chalk.red(`Could not resolve ${targetName} config path on this system.`));
+    process.exitCode = 1;
+    return null;
+  }
+
+  const baseReal = realpathSync(rawBasePath);
+  const resolvedPath = resolve(rawPath);
+  const resolvedParent = dirname(resolvedPath);
+
+  let existingAncestor = resolvedParent;
+  while (!existsSync(existingAncestor) && dirname(existingAncestor) !== existingAncestor) {
+    existingAncestor = dirname(existingAncestor);
+  }
+
+  if (!existsSync(existingAncestor)) {
+    console.error(chalk.red(`Could not validate parent directory for ${targetName} config.`));
+    process.exitCode = 1;
+    return null;
+  }
+
+  const ancestorReal = realpathSync(existingAncestor);
+  if (!isWithinBase(baseReal, ancestorReal) || !isWithinBase(baseReal, resolvedPath)) {
+    console.error(chalk.red(`${targetName} config path resolves outside the expected base directory.`));
+    process.exitCode = 1;
+    return null;
+  }
+
+  if (existsSync(resolvedPath) && lstatSync(resolvedPath).isSymbolicLink()) {
+    console.error(chalk.red(`${targetName} config path is a symlink. Refusing to write secrets to symlinked files.`));
+    process.exitCode = 1;
+    return null;
+  }
+
+  return resolvedPath;
+}
+
+export function secureWriteJson(configPath, contents) {
+  const noFollow = typeof constants.O_NOFOLLOW === "number" ? constants.O_NOFOLLOW : 0;
+  const flags = constants.O_WRONLY | constants.O_CREAT | constants.O_TRUNC | noFollow;
+  let fd;
+
+  try {
+    fd = openSync(configPath, flags, 0o600);
+  } catch (err) {
+    const noFollowUnsupported = noFollow && (
+      err?.code === "EINVAL" ||
+      err?.code === "ENOTSUP" ||
+      err?.code === "EOPNOTSUPP"
+    );
+    if (noFollowUnsupported) {
+      const tmpPath = `${configPath}.${process.pid}.${Date.now()}.tmp`;
+      writeFileSync(tmpPath, contents, { encoding: "utf-8", mode: 0o600, flag: "wx" });
+      try {
+        if (existsSync(configPath) && lstatSync(configPath).isSymbolicLink()) {
+          throw new Error("refusing to overwrite symlinked config path");
+        }
+        renameSync(tmpPath, configPath);
+      } catch (fallbackErr) {
+        try {
+          if (existsSync(tmpPath)) unlinkSync(tmpPath);
+        } catch {
+          // ignore cleanup errors
+        }
+        throw fallbackErr;
+      }
+      return;
+    }
+    throw err;
+  }
+
+  try {
+    writeFileSync(fd, contents, "utf-8");
+  } finally {
+    closeSync(fd);
+  }
+}
+
+function ensureOwnerOnly(pathToFile) {
+  try {
+    chmodSync(pathToFile, 0o600);
+  } catch (err) {
+    console.error(chalk.yellow(`Warning: could not set 0600 on ${pathToFile}: ${err.message}`));
+  }
+}
+
+export function ensureGitignore(configPath) {
+  let dir = dirname(configPath);
+  let gitRoot = null;
+  while (dir !== dirname(dir)) {
+    if (existsSync(join(dir, ".git"))) {
+      gitRoot = dir;
+      break;
+    }
+    dir = dirname(dir);
+  }
+  if (!gitRoot) return null;
+
+  const rel = relative(gitRoot, configPath).replace(/\\/g, "/");
+  const gitignorePath = join(gitRoot, ".gitignore");
+
+  if (existsSync(gitignorePath)) {
+    let content;
+    try {
+      content = readFileSync(gitignorePath, "utf-8");
+    } catch {
+      return null;
+    }
+    const lines = content.split(/\r?\n/);
+    // already covered by an exact match or parent-dir glob
+    const configDir = dirname(rel) + "/";
+    if (lines.some((l) => {
+      const trimmed = l.trim();
+      return trimmed === rel || trimmed === `/${rel}` || trimmed === configDir || trimmed === `/${configDir}`;
+    })) {
+      return null;
+    }
+  }
+
+  try {
+    const prefix = existsSync(gitignorePath) && readFileSync(gitignorePath, "utf-8").endsWith("\n") ? "" : "\n";
+    appendFileSync(gitignorePath, `${prefix}${rel}\n`, "utf-8");
+    return rel;
+  } catch {
+    return null;
+  }
+}
+
+export async function agentAddCommand(target, globalOpts) {
+  const targetLower = target?.toLowerCase();
+  const targetDef = TARGETS[targetLower];
+
+  if (!targetDef) {
+    console.error(chalk.red("Unknown target."));
+    console.error(`Available targets: ${Object.keys(TARGETS).join(", ")}`);
+    process.exitCode = 1;
+    return;
+  }
+
+  let apiKey = resolveApiKey(globalOpts);
+  if (!apiKey) {
+    const key = await prompts.text({
+      message: "Enter your ScrapeCreators API key",
+      placeholder: "paste from https://app.scrapecreators.com",
+      validate: (v) => (v.length < 5 ? "Too short" : undefined),
+    });
+
+    if (prompts.isCancel(key)) {
+      prompts.cancel("Cancelled.");
+      return;
+    }
+
+    const result = await callApi(key, "GET", "/v1/credit-balance");
+    if (!result.ok) {
+      console.error(chalk.red("Invalid API key."));
+      process.exitCode = 1;
+      return;
+    }
+
+    storeApiKey(key);
+    apiKey = key;
+    console.log(chalk.green("API key validated and saved."));
+  }
+
+  const configPath = resolveConfigPath(targetDef.configPath(), targetDef.basePath(), targetDef.name);
+  if (!configPath) return;
+  const newConfig = targetDef.buildConfig(apiKey);
+
+  let existing = {};
+  if (existsSync(configPath)) {
+    let raw;
+    try {
+      raw = readFileSync(configPath, "utf-8");
+    } catch (err) {
+      console.error(chalk.red(`Could not read existing config at ${configPath}: ${err.message}`));
+      process.exitCode = 1;
+      return;
+    }
+
+    try {
+      existing = JSON.parse(raw);
+    } catch (err) {
+      console.error(chalk.red(`Existing config is not valid JSON: ${configPath}`));
+      console.error(chalk.yellow("Fix the file or back it up before running this command again."));
+      if (err?.message) console.error(chalk.dim(err.message));
+      process.exitCode = 1;
+      return;
+    }
+
+    if (!isPlainObject(existing)) {
+      console.error(chalk.red(`Expected a JSON object at root in ${configPath}.`));
+      process.exitCode = 1;
+      return;
+    }
+  }
+
+  if (existing.mcpServers !== undefined && !isPlainObject(existing.mcpServers)) {
+    console.error(chalk.red(`Expected "mcpServers" to be an object in ${configPath}.`));
+    process.exitCode = 1;
+    return;
+  }
+
+  // merge: preserve other MCP servers
+  existing.mcpServers = {
+    ...(existing.mcpServers || {}),
+    ...newConfig.mcpServers,
+  };
+
+  mkdirSync(dirname(configPath), { recursive: true, mode: 0o700 });
+  try {
+    secureWriteJson(configPath, JSON.stringify(existing, null, 2) + "\n");
+  } catch (err) {
+    console.error(chalk.red(`Could not write config file ${configPath}: ${err.message}`));
+    process.exitCode = 1;
+    return;
+  }
+  ensureOwnerOnly(configPath);
+
+  const addedEntry = ensureGitignore(configPath);
+  console.log(chalk.green(`\n${targetDef.name} MCP config written to ${configPath}`));
+  if (addedEntry) {
+    console.log(chalk.green(`Added ${addedEntry} to .gitignore to keep your API key out of version control.`));
+  }
+  console.log(chalk.dim("Make sure the scrapecreators MCP server is enabled in your editor settings."));
+}

package/src/commands/api.js

@@ -0,0 +1,49 @@
+import ora from "ora";
+import chalk from "chalk";
+import { resolveApiKey } from "../auth.js";
+import { callApi } from "../api-client.js";
+import { printResult } from "../output.js";
+
+export async function handleApiCommand(tool, args, globalOpts) {
+  const apiKey = resolveApiKey(globalOpts);
+  if (!apiKey) {
+    console.error(chalk.red("No API key found."));
+    console.error(chalk.yellow("Run 'scrapecreators auth login' or set SCRAPECREATORS_API_KEY"));
+    process.exitCode = 1;
+    return;
+  }
+
+  if (tool.credits && tool.credits > 1 && process.stdout.isTTY) {
+    console.error(chalk.yellow(`Note: this endpoint costs ${tool.credits} credits per request.`));
+  }
+
+  const params = {};
+  for (const param of tool.params) {
+    const cliFlag = param.name.replace(/_/g, "-");
+    const value = args[camelCase(cliFlag)];
+    if (value !== undefined) {
+      params[param.name] = value;
+    }
+  }
+
+  if (globalOpts.trim) params.trim = true;
+  if (globalOpts.region) params.region = globalOpts.region;
+
+  const showSpinner = process.stdout.isTTY && !globalOpts.raw && !globalOpts.json;
+  const spinner = showSpinner ? ora({ text: "Fetching...", stream: process.stderr }).start() : null;
+
+  try {
+    const result = await callApi(apiKey, tool.method, tool.path, params);
+    if (spinner) spinner.stop();
+    printResult(result, globalOpts);
+  } catch (err) {
+    if (spinner) spinner.stop();
+    console.error(chalk.red("Request failed."));
+    if (globalOpts.verbose) console.error(chalk.dim(err.message));
+    process.exitCode = 1;
+  }
+}
+
+function camelCase(str) {
+  return str.replace(/-([a-z])/g, (_, c) => c.toUpperCase());
+}

package/src/commands/auth.js

@@ -0,0 +1,69 @@
+import * as prompts from "@clack/prompts";
+import chalk from "chalk";
+import { resolveApiKey, storeApiKey, clearApiKey, getStoredApiKey, maskKey } from "../auth.js";
+import { callApi } from "../api-client.js";
+
+export async function authLogin() {
+  prompts.intro(chalk.bold("ScrapeCreators Authentication"));
+
+  const apiKey = await prompts.text({
+    message: "Enter your API key",
+    placeholder: "paste your key from https://app.scrapecreators.com",
+    validate: (v) => (v.length < 5 ? "API key seems too short" : undefined),
+  });
+
+  if (prompts.isCancel(apiKey)) {
+    prompts.cancel("Cancelled.");
+    return;
+  }
+
+  const spinner = prompts.spinner();
+  spinner.start("Validating API key...");
+
+  try {
+    const result = await callApi(apiKey, "GET", "/v1/credit-balance");
+    if (!result.ok) {
+      spinner.stop("Invalid API key.");
+      console.error(chalk.red("The API key was rejected. Check it and try again."));
+      process.exitCode = 1;
+      return;
+    }
+
+    storeApiKey(apiKey);
+    const credits = result.data?.creditCount ?? result.data?.credits ?? "unknown";
+    spinner.stop(`Authenticated. Balance: ${credits} credits.`);
+    prompts.outro(chalk.green("API key saved."));
+  } catch {
+    spinner.stop("Connection failed.");
+    console.error(chalk.red("Could not reach API. Check your network and try again."));
+    process.exitCode = 1;
+  }
+}
+
+export async function authStatus() {
+  const key = resolveApiKey();
+  if (!key) {
+    console.log(chalk.yellow("Not authenticated. Run 'scrapecreators auth login'."));
+    return;
+  }
+
+  const stored = getStoredApiKey();
+  const source = stored ? "stored config" : process.env.SCRAPECREATORS_API_KEY ? "environment variable" : "unknown";
+
+  console.log(`API key: ${chalk.cyan(maskKey(key))} (from ${source})`);
+
+  try {
+    const result = await callApi(key, "GET", "/v1/credit-balance");
+    if (result.ok) {
+      const credits = result.data?.creditCount ?? result.data?.credits ?? "unknown";
+      console.log(`Credits: ${chalk.green(credits)}`);
+    }
+  } catch {
+    // non-critical, just skip
+  }
+}
+
+export async function authLogout() {
+  clearApiKey();
+  console.log(chalk.green("API key removed from stored config."));
+}

package/src/commands/balance.js

@@ -0,0 +1,40 @@
+import chalk from "chalk";
+import ora from "ora";
+import { resolveApiKey } from "../auth.js";
+import { callApi } from "../api-client.js";
+import { printResult } from "../output.js";
+
+export async function balanceCommand(globalOpts) {
+  const apiKey = resolveApiKey(globalOpts);
+  if (!apiKey) {
+    console.error(chalk.red("No API key found."));
+    console.error(chalk.yellow("Run 'scrapecreators auth login' or set SCRAPECREATORS_API_KEY"));
+    process.exitCode = 1;
+    return;
+  }
+
+  const showSpinner = process.stdout.isTTY && !globalOpts.raw && !globalOpts.json;
+  const spinner = showSpinner ? ora({ text: "Checking balance...", stream: process.stderr }).start() : null;
+
+  try {
+    const result = await callApi(apiKey, "GET", "/v1/credit-balance");
+    if (spinner) spinner.stop();
+
+    if (globalOpts.raw || globalOpts.json || !process.stdout.isTTY) {
+      printResult(result, globalOpts);
+    } else if (result.ok) {
+      const credits = result.data?.creditCount ?? result.data?.credits ?? "unknown";
+      console.log(`Credits remaining: ${chalk.green(chalk.bold(credits))}`);
+    } else {
+      printResult(result, globalOpts);
+    }
+  } catch (err) {
+    if (spinner) spinner.stop();
+    console.error(chalk.red("Request failed."));
+    if (globalOpts.verbose) {
+      console.error(chalk.dim(err.message));
+      if (err.cause) console.error(chalk.dim(`Cause: ${err.cause.message || err.cause}`));
+    }
+    process.exitCode = 1;
+  }
+}

package/src/commands/config.js

@@ -0,0 +1,65 @@
+import chalk from "chalk";
+import config from "../config.js";
+import { maskKey } from "../auth.js";
+
+const ALLOWED_KEYS = ["apiKey", "defaultFormat"];
+const VALID_FORMATS = ["auto", "json", "table", "csv", "markdown"];
+
+export function configSet(key, value) {
+  if (!ALLOWED_KEYS.includes(key)) {
+    console.error(chalk.red("Unknown config key."));
+    console.error(`Valid keys: ${ALLOWED_KEYS.join(", ")}`);
+    process.exitCode = 1;
+    return;
+  }
+  if (key === "defaultFormat" && !VALID_FORMATS.includes(value)) {
+    console.error(chalk.red(`Invalid format: ${value}`));
+    console.error(`Valid formats: ${VALID_FORMATS.join(", ")}`);
+    process.exitCode = 1;
+    return;
+  }
+  config.set(key, value);
+  const display = key === "apiKey" ? maskKey(value) : value;
+  console.log(chalk.green(`${key} = ${display}`));
+}
+
+export function configGet(key) {
+  if (!ALLOWED_KEYS.includes(key)) {
+    console.error(chalk.red("Unknown config key."));
+    console.error(`Valid keys: ${ALLOWED_KEYS.join(", ")}`);
+    process.exitCode = 1;
+    return;
+  }
+  if (key === "apiKey") {
+    printApiKeyStatus();
+  } else {
+    console.log(config.get(key) || chalk.dim("(not set)"));
+  }
+}
+
+export function configList() {
+  console.log(chalk.bold("Current configuration:\n"));
+
+  process.stdout.write(`  ${chalk.cyan("apiKey")}: `);
+  printApiKeyStatus();
+
+  for (const key of ALLOWED_KEYS) {
+    if (key === "apiKey") continue;
+    console.log(`  ${chalk.cyan(key)}: ${config.get(key) || chalk.dim("(not set)")}`);
+  }
+
+  console.log(chalk.dim(`\nConfig file: ${config.path}`));
+}
+
+function printApiKeyStatus() {
+  const stored = config.get("apiKey");
+  const envKey = process.env.SCRAPECREATORS_API_KEY;
+
+  if (stored) {
+    console.log(`${maskKey(stored)} ${chalk.dim("(from stored config)")}`);
+  } else if (envKey) {
+    console.log(`${maskKey(envKey)} ${chalk.dim("(from SCRAPECREATORS_API_KEY env)")}`);
+  } else {
+    console.log(chalk.dim("(not set)"));
+  }
+}

package/src/commands/list.js

@@ -0,0 +1,52 @@
+import chalk from "chalk";
+import Table from "cli-table3";
+import { getPlatformMap } from "../command-registry.js";
+
+export function listCommand(platform) {
+  const platforms = getPlatformMap();
+
+  if (platform) {
+    const tools = platforms.get(platform);
+    if (!tools) {
+      console.error(chalk.red(`Unknown platform: ${platform}`));
+      console.error(`Available: ${[...platforms.keys()].join(", ")}`);
+      process.exitCode = 1;
+      return;
+    }
+
+    console.log(chalk.bold(`\n${platform} endpoints:\n`));
+    const table = new Table({
+      head: [chalk.cyan("Command"), chalk.cyan("Description"), chalk.cyan("Required Params")],
+    });
+
+    for (const tool of tools) {
+      const required = tool.params
+        .filter((p) => p.required)
+        .map((p) => `--${p.name.replace(/_/g, "-")}`)
+        .join(", ");
+
+      table.push([
+        `scrapecreators ${platform} ${tool._action}`,
+        tool.title,
+        required || chalk.dim("none"),
+      ]);
+    }
+    console.log(table.toString());
+    return;
+  }
+
+  // list all platforms (skip credit — handled by `scrapecreators balance`)
+  console.log(chalk.bold("\nAvailable platforms:\n"));
+  const table = new Table({
+    head: [chalk.cyan("Platform"), chalk.cyan("Endpoints")],
+  });
+
+  const sorted = [...platforms.entries()]
+    .filter(([name]) => name !== "credit")
+    .sort((a, b) => a[0].localeCompare(b[0]));
+  for (const [name, tools] of sorted) {
+    table.push([name, tools.length]);
+  }
+  console.log(table.toString());
+  console.log(chalk.dim("\nRun 'scrapecreators list <platform>' for endpoint details."));
+}

package/src/config.js

@@ -0,0 +1,23 @@
+import Conf from "conf";
+import { chmodSync } from "fs";
+
+const config = new Conf({
+  projectName: "scrapecreators",
+  schema: {
+    apiKey: { type: "string", default: "" },
+    defaultFormat: {
+      type: "string",
+      default: "auto",
+      enum: ["auto", "json", "table", "csv", "markdown"],
+    },
+  },
+});
+
+// restrict config file to owner-only (contains API key)
+try {
+  chmodSync(config.path, 0o600);
+} catch (err) {
+  console.error(`Warning: could not set 0600 on ${config.path}: ${err.message}`);
+}
+
+export default config;