@scrapeatlas/cli 0.1.0

package/package.json

@@ -0,0 +1,36 @@
+{
+  "name": "@scrapeatlas/cli",
+  "version": "0.1.0",
+  "description": "Official ScrapeAtlas CLI — discover and call social data API endpoints from your terminal.",
+  "type": "module",
+  "bin": {
+    "scrapeatlas": "bin/scrapeatlas.mjs"
+  },
+  "files": [
+    "bin/",
+    "src/",
+    "catalog.json",
+    "README.md"
+  ],
+  "engines": {
+    "node": ">=22"
+  },
+  "license": "UNLICENSED",
+  "homepage": "https://scrapeatlas.com/docs/",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/OnlineDopamine/social-scraper.git",
+    "directory": "packages/cli"
+  },
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org/"
+  },
+  "keywords": [
+    "scrapeatlas",
+    "cli",
+    "social-media",
+    "scraping",
+    "api"
+  ]
+}

package/src/cli.mjs

@@ -0,0 +1,320 @@
+import { lstat, readFile, writeFile } from 'node:fs/promises';
+import { resolve } from 'node:path';
+import { parseArgs } from 'node:util';
+import { readConfig, saveConfig, secretInput } from './config.mjs';
+import { CliError, origin, request } from './http.mjs';
+import { convert, kebab, valid } from './schema.mjs';
+
+const catalog = JSON.parse(await readFile(new URL('../catalog.json', import.meta.url), 'utf8'));
+const pkg = JSON.parse(await readFile(new URL('../package.json', import.meta.url), 'utf8'));
+const globalOptions = {
+  help: { type: 'boolean', short: 'h' },
+  version: { type: 'boolean', short: 'v' },
+  pretty: { type: 'boolean' },
+  json: { type: 'boolean' },
+  envelope: { type: 'boolean' },
+  output: { type: 'string' },
+  'api-key': { type: 'string' },
+  'base-url': { type: 'string' },
+  timeout: { type: 'string' },
+  input: { type: 'string' },
+  stdin: { type: 'boolean' },
+};
+const generalHelp = `ScrapeAtlas CLI ${pkg.version}
+
+Usage: scrapeatlas <platform> <action> [--parameters]
+       scrapeatlas call <operationId> --input '{"parameter":"value"}'
+       scrapeatlas list [platform] [--json]
+       scrapeatlas auth login [--stdin]
+       scrapeatlas auth status | auth logout
+
+Explore: scrapeatlas bluesky profile --help
+Global options:
+  --api-key <key>      Override SCRAPEATLAS_API_KEY / stored login
+  --base-url <origin> Override SCRAPEATLAS_BASE_URL / stored origin
+  --input <json>      Request object; @file reads JSON from a file
+  --pretty            Indented JSON (default: compact JSON)
+  --envelope          Include httpStatus, requestId and retryAfter around body
+  --output <file>     Save JSON to a new file (never overwrite)
+  --timeout <ms>      Total deadline, 1..120000 ms (default: 90000)
+  --help, -h          Show help for a command
+  --version, -v       Print version
+
+One request per command. Pagination is explicit. Inspect status and coverage.
+Data goes to stdout; diagnostics go to stderr. Exit: 0 usable, 1 API/network, 2 usage.
+Get a customer API key from https://scrapeatlas.com/dashboard/keys/.
+`;
+function parse(args, operation) {
+  const options = { ...globalOptions };
+  const names = new Map();
+  for (const [name, schema] of Object.entries(operation?.inputSchema.properties ?? {})) {
+    for (const flag of new Set([name, kebab(name)])) {
+      if (Object.hasOwn(globalOptions, flag) || (names.has(flag) && names.get(flag) !== name))
+        continue;
+      names.set(flag, name);
+      options[flag] = { type: schema.type === 'boolean' ? 'boolean' : 'string' };
+      if (schema.type === 'boolean') {
+        options[`no-${flag}`] = { type: 'boolean' };
+        names.set(`no-${flag}`, name);
+      }
+    }
+  }
+  let parsed;
+  try {
+    parsed = parseArgs({ args, options, allowPositionals: true, tokens: true });
+  } catch {
+    throw new CliError(
+      'invalid_arguments',
+      'Unknown option or missing value. Use --help; use --input for JSON and boolean values.'
+    );
+  }
+  const seen = new Set();
+  for (const token of parsed.tokens.filter((item) => item.kind === 'option')) {
+    const key = names.get(token.name) ?? token.name;
+    if (seen.has(key)) throw new CliError('duplicate_option', 'Duplicate options are not allowed.');
+    seen.add(key);
+  }
+  return { ...parsed, names };
+}
+function help(operation) {
+  return (
+    `scrapeatlas ${operation.platform} ${operation.action}\n${operation.method} ${operation.path}\n\n${operation.description}\n\nParameters:\n` +
+    Object.entries(operation.inputSchema.properties)
+      .map(
+        ([name, schema]) =>
+          `  --${kebab(name)} <${schema.type ?? 'json'}>${operation.inputSchema.required.includes(name) ? ' (required)' : ''}${schema.enum ? ` [${schema.enum.join(', ')}]` : ''}\n    ${schema.description ?? ''}`
+      )
+      .join('\n') +
+    '\n\nCamelCase and snake_case API parameter names also work. Use --input for nested objects, arrays, or explicit booleans. Runtime refinements are validated by the gateway.\n\n' +
+    generalHelp
+  );
+}
+async function inputObject(value) {
+  if (!value) return {};
+  let text = value;
+  if (value.startsWith('@')) {
+    const handle = await import('node:fs/promises').then((fs) => fs.open(value.slice(1), 'r'));
+    try {
+      if ((await handle.stat()).size > 65536)
+        throw new CliError('input_too_large', 'Request input exceeds 64 KiB.');
+      const buffer = Buffer.alloc(65537);
+      let size = 0;
+      while (size < buffer.length) {
+        const { bytesRead } = await handle.read(buffer, size, buffer.length - size, null);
+        if (!bytesRead) break;
+        size += bytesRead;
+      }
+      if (size > 65536) throw new CliError('input_too_large', 'Request input exceeds 64 KiB.');
+      text = buffer.subarray(0, size).toString('utf8');
+    } finally {
+      await handle.close();
+    }
+  }
+  if (Buffer.byteLength(text) > 65536)
+    throw new CliError('input_too_large', 'Request input exceeds 64 KiB.');
+  try {
+    const value = JSON.parse(text);
+    if (!value || typeof value !== 'object' || Array.isArray(value)) throw new Error();
+    return value;
+  } catch {
+    throw new CliError('invalid_input', '--input must contain a JSON object.');
+  }
+}
+function apiKey(value) {
+  if (typeof value !== 'string' || !value.trim())
+    throw new CliError('missing_key', 'Run scrapeatlas auth login or set SCRAPEATLAS_API_KEY.');
+  if (!/^[\x21-\x7e]{1,8192}$/.test(value))
+    throw new CliError('invalid_key', 'API key must be a nonempty token without whitespace.');
+  return value;
+}
+async function emit(value, options) {
+  const text = JSON.stringify(value, null, options.pretty ? 2 : undefined) + '\n';
+  if (options.output) {
+    await writeFile(resolve(options.output), text, { flag: 'wx', mode: 0o600 });
+    process.stdout.write(resolve(options.output) + '\n');
+  } else process.stdout.write(text);
+}
+export async function run(args) {
+  try {
+    // Identify positionals with a permissive first pass; strict parsing follows with endpoint options.
+    const rough = parseArgs({
+      args,
+      strict: false,
+      allowPositionals: true,
+      options: globalOptions,
+    });
+    const words = rough.positionals;
+    const operation =
+      words[0] === 'call'
+        ? catalog.find((entry) => entry.operation === words[1])
+        : catalog.find((entry) => entry.platform === words[0] && entry.action === words[1]);
+    const { values: options, positionals, names } = parse(args, operation);
+    if (options.output) {
+      try {
+        await lstat(resolve(options.output));
+        throw new CliError('output_exists', 'Output file already exists; choose a new path.');
+      } catch (error) {
+        if (error.code !== 'ENOENT') throw error;
+      }
+    }
+    if (options.version) {
+      process.stdout.write(pkg.version + '\n');
+      return 0;
+    }
+    if (options.help || !positionals.length) {
+      process.stdout.write(operation ? help(operation) : generalHelp);
+      return 0;
+    }
+    if (positionals[0] === 'list') {
+      if (positionals.length > 2)
+        throw new CliError('invalid_arguments', 'Usage: scrapeatlas list [platform]');
+      const entries = catalog.filter(
+        (entry) => !positionals[1] || entry.platform === positionals[1]
+      );
+      if (!entries.length)
+        throw new CliError('unknown_platform', 'Unknown platform. Run scrapeatlas list.');
+      if (options.json || options.output || options.pretty) await emit(entries, options);
+      else
+        process.stdout.write(
+          positionals[1]
+            ? entries
+                .map((entry) => `${entry.platform} ${entry.action}  ${entry.method} ${entry.path}`)
+                .join('\n') + '\n'
+            : [...new Set(entries.map((entry) => entry.platform))]
+                .map(
+                  (platform) =>
+                    `${platform}  (${entries.filter((entry) => entry.platform === platform).length} commands)`
+                )
+                .join('\n') + '\n'
+        );
+      return 0;
+    }
+    if (!operation && positionals[0] !== 'auth')
+      throw new CliError('unknown_command', 'Unknown command. Run scrapeatlas list or --help.');
+    if (positionals.length !== 2)
+      throw new CliError(
+        'invalid_arguments',
+        'Expected exactly a platform and action, or call and operation ID.'
+      );
+    const config = await readConfig();
+    const base = origin(
+      options['base-url'] ||
+        process.env.SCRAPEATLAS_BASE_URL ||
+        config.baseUrl ||
+        'https://api.scrapeatlas.com'
+    );
+    const timeout = Number(options.timeout ?? 90000);
+    if (!Number.isInteger(timeout) || timeout < 1 || timeout > 120000)
+      throw new CliError('invalid_timeout', 'Timeout must be 1..120000 milliseconds.');
+    const explicitKey = options['api-key'] || process.env.SCRAPEATLAS_API_KEY;
+    // Saved credentials belong to the origin at which they were verified.
+    const savedKey =
+      (config.baseUrl || 'https://api.scrapeatlas.com') === base ? config.apiKey : undefined;
+    if (positionals[0] === 'auth') {
+      if (positionals[1] === 'logout') {
+        await saveConfig({});
+        await emit(
+          {
+            authenticated: false,
+            storedKeyRemoved: true,
+            environmentKeyPresent: Boolean(process.env.SCRAPEATLAS_API_KEY),
+          },
+          options
+        );
+        return 0;
+      }
+      if (positionals[1] === 'status') {
+        await emit(
+          {
+            authenticated: Boolean(explicitKey || savedKey),
+            source: options['api-key']
+              ? 'flag'
+              : process.env.SCRAPEATLAS_API_KEY
+                ? 'environment'
+                : savedKey
+                  ? 'config'
+                  : null,
+            baseUrl: base,
+            verified: false,
+          },
+          options
+        );
+        return 0;
+      }
+      if (positionals[1] !== 'login')
+        throw new CliError('unknown_command', 'Use auth login, auth status, or auth logout.');
+      const key = apiKey(
+        options.stdin ? await secretInput(true) : explicitKey || (await secretInput(false))
+      );
+      const result = await request(
+        base,
+        key,
+        { method: 'GET', path: '/v1/auth/verify' },
+        {},
+        Math.min(timeout, 10000)
+      );
+      if (result.status !== 200 || result.body?.valid !== true)
+        throw new CliError(
+          'authentication_failed',
+          'Gateway did not verify the key; nothing was saved.',
+          1
+        );
+      await saveConfig({ apiKey: key, baseUrl: base });
+      await emit({ authenticated: true, verified: true, baseUrl: base }, options);
+      return 0;
+    }
+    const input = await inputObject(options.input);
+    for (const [flag, name] of names)
+      if (options[flag] !== undefined) {
+        if (Object.hasOwn(input, name))
+          throw new CliError('duplicate_parameter', 'A parameter was supplied more than once.');
+        input[name] = flag.startsWith('no-')
+          ? false
+          : convert(operation.inputSchema.properties[name], String(options[flag]));
+      }
+    if (!valid(operation.inputSchema, input))
+      throw new CliError(
+        'invalid_parameters',
+        'Parameters do not match this endpoint. Use --help or list <platform> --json for its schema.'
+      );
+    if (Buffer.byteLength(JSON.stringify(input)) > 65536)
+      throw new CliError('input_too_large', 'Request input exceeds 64 KiB.');
+    const result = await request(base, apiKey(explicitKey || savedKey), operation, input, timeout);
+    const failed =
+      result.status >= 400 ||
+      result.body?.success === false ||
+      ['failed', 'challenged'].includes(result.body?.status);
+    await emit(
+      options.envelope
+        ? {
+            httpStatus: result.status,
+            body: result.body,
+            ...(result.requestId ? { requestId: result.requestId } : {}),
+            ...(result.retryAfter ? { retryAfter: result.retryAfter } : {}),
+          }
+        : result.body,
+      options
+    );
+    if (failed)
+      process.stderr.write(
+        JSON.stringify({
+          error: true,
+          code: `HTTP_${result.status}`,
+          message: 'Gateway returned a failure; inspect the response body.',
+        }) + '\n'
+      );
+    return failed ? 1 : 0;
+  } catch (error) {
+    const known = error instanceof CliError;
+    process.stderr.write(
+      JSON.stringify({
+        error: true,
+        code: known ? error.code : 'local_error',
+        message: known
+          ? error.message
+          : 'CLI operation failed. Check file permissions and configuration; output files must not already exist.',
+      }) + '\n'
+    );
+    return known ? error.exitCode : 2;
+  }
+}

package/src/config.mjs

@@ -0,0 +1,91 @@
+import { chmod, lstat, mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises';
+import { homedir } from 'node:os';
+import { join, resolve } from 'node:path';
+import { randomUUID } from 'node:crypto';
+import { CliError } from './http.mjs';
+
+export function configDirectory() {
+  return resolve(
+    process.env.SCRAPEATLAS_CONFIG_DIR ||
+      join(process.env.XDG_CONFIG_HOME || join(homedir(), '.config'), 'scrapeatlas')
+  );
+}
+export async function readConfig() {
+  try {
+    const value = JSON.parse(await readFile(join(configDirectory(), 'config.json'), 'utf8'));
+    if (!value || typeof value !== 'object' || Array.isArray(value)) throw new Error();
+    return value;
+  } catch (error) {
+    if (error.code === 'ENOENT') return {};
+    throw new CliError(
+      'config_error',
+      'Cannot read CLI config. Repair or remove config.json in the ScrapeAtlas config directory.'
+    );
+  }
+}
+export async function saveConfig(config) {
+  const directory = configDirectory();
+  await mkdir(directory, { recursive: true, mode: 0o700 });
+  if (!(await lstat(directory)).isDirectory())
+    throw new CliError('config_error', 'Config directory must be a real directory.');
+  await chmod(directory, 0o700);
+  const temporary = join(directory, `.config-${randomUUID()}.tmp`);
+  try {
+    await writeFile(temporary, JSON.stringify(config) + '\n', { mode: 0o600, flag: 'wx' });
+    await rename(temporary, join(directory, 'config.json'));
+  } finally {
+    await rm(temporary, { force: true });
+  }
+}
+export async function secretInput(fromStdin) {
+  if (fromStdin) {
+    const chunks = [];
+    let size = 0;
+    for await (const chunk of process.stdin) {
+      size += chunk.length;
+      if (size > 8192) throw new CliError('invalid_key', 'API key input is too large.');
+      chunks.push(chunk);
+    }
+    return Buffer.concat(chunks).toString('utf8').trim();
+  }
+  if (!process.stdin.isTTY)
+    throw new CliError('missing_key', 'Use SCRAPEATLAS_API_KEY or auth login --stdin.');
+  process.stderr.write('ScrapeAtlas API key (hidden): ');
+  const previous = process.stdin.isRaw;
+  process.stdin.setRawMode(true);
+  process.stdin.resume();
+  try {
+    return await new Promise((resolve, reject) => {
+      let value = '';
+      function onData(chunk) {
+        for (const character of chunk.toString('utf8')) {
+          if (character === '\r' || character === '\n') {
+            cleanup();
+            resolve(value.trim());
+            return;
+          }
+          if (character === '\u0003' || character === '\u0004') {
+            cleanup();
+            reject(new CliError('cancelled', 'Login cancelled.', 130));
+            return;
+          }
+          if (character === '\u007f' || character === '\b') value = value.slice(0, -1);
+          else if (character >= ' ') value += character;
+          if (value.length > 8192) {
+            cleanup();
+            reject(new CliError('invalid_key', 'API key input is too large.'));
+            return;
+          }
+        }
+      }
+      function cleanup() {
+        process.stdin.off('data', onData);
+      }
+      process.stdin.on('data', onData);
+    });
+  } finally {
+    process.stdin.setRawMode(previous);
+    process.stdin.pause();
+    process.stderr.write('\n');
+  }
+}

package/src/http.mjs

@@ -0,0 +1,104 @@
+export class CliError extends Error {
+  constructor(code, message, exitCode = 2) {
+    super(message);
+    this.code = code;
+    this.exitCode = exitCode;
+  }
+}
+export function origin(value) {
+  let url;
+  try {
+    url = new URL(value);
+  } catch {
+    throw new CliError(
+      'invalid_origin',
+      'Base URL must be an HTTPS origin (or HTTP loopback for development).'
+    );
+  }
+  if (
+    url.username ||
+    url.password ||
+    url.pathname !== '/' ||
+    url.search ||
+    url.hash ||
+    !(
+      url.protocol === 'https:' ||
+      (url.protocol === 'http:' && ['localhost', '127.0.0.1', '[::1]'].includes(url.hostname))
+    )
+  )
+    throw new CliError(
+      'invalid_origin',
+      'Base URL must be an HTTPS origin (or HTTP loopback for development).'
+    );
+  return url.origin;
+}
+export async function request(base, key, operation, input, timeout) {
+  const url = new URL(operation.path, base);
+  if (operation.method === 'GET')
+    for (const [name, value] of Object.entries(input)) {
+      url.searchParams.set(name, typeof value === 'object' ? JSON.stringify(value) : String(value));
+    }
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), timeout);
+  let reader;
+  try {
+    const response = await fetch(url, {
+      method: operation.method,
+      redirect: 'manual',
+      credentials: 'omit',
+      signal: controller.signal,
+      headers: {
+        authorization: `Bearer ${key}`,
+        accept: 'application/json',
+        'user-agent': 'scrapeatlas-cli/0.1.0',
+        ...(operation.method === 'POST' ? { 'content-type': 'application/json' } : {}),
+      },
+      ...(operation.method === 'POST' ? { body: JSON.stringify(input) } : {}),
+    });
+    if (response.status >= 300 && response.status < 400) {
+      await response.body?.cancel();
+      throw new CliError('redirect_refused', 'Gateway redirect refused; verify your base URL.', 1);
+    }
+    if (!/^application\/json(?:\s*;|$)/i.test(response.headers.get('content-type') ?? '')) {
+      await response.body?.cancel();
+      throw new CliError('invalid_response', 'Gateway did not return JSON.', 1);
+    }
+    reader = response.body?.getReader();
+    let size = 0;
+    const chunks = [];
+    while (reader) {
+      const chunk = await reader.read();
+      if (chunk.done) break;
+      size += chunk.value.byteLength;
+      if (size > 16 * 1024 * 1024)
+        throw new CliError(
+          'response_too_large',
+          'Response exceeds 16 MiB; request a smaller page.',
+          1
+        );
+      chunks.push(chunk.value);
+    }
+    let body;
+    try {
+      body = JSON.parse(Buffer.concat(chunks).toString('utf8'));
+    } catch {
+      throw new CliError('invalid_response', 'Gateway returned invalid JSON.', 1);
+    }
+    return {
+      body,
+      status: response.status,
+      requestId: response.headers.get('x-request-id'),
+      retryAfter: response.headers.get('retry-after'),
+    };
+  } catch (error) {
+    if (error instanceof CliError) throw error;
+    throw new CliError(
+      controller.signal.aborted ? 'timeout' : 'network_error',
+      controller.signal.aborted ? 'Request deadline exceeded.' : 'Gateway request failed.',
+      1
+    );
+  } finally {
+    clearTimeout(timer);
+    await reader?.cancel().catch(() => {});
+  }
+}

package/src/schema.mjs

@@ -0,0 +1,66 @@
+export const kebab = (value) =>
+  value
+    .replace(/([a-z0-9])([A-Z])/g, '$1-$2')
+    .replaceAll('_', '-')
+    .toLowerCase();
+export function valid(schema, value) {
+  if (schema.anyOf) return schema.anyOf.some((option) => valid(option, value));
+  if ('const' in schema && value !== schema.const) return false;
+  if (schema.enum && !schema.enum.includes(value)) return false;
+  switch (schema.type) {
+    case 'null':
+      return value === null;
+    case 'string':
+      return (
+        typeof value === 'string' &&
+        (schema.minLength === undefined || value.length >= schema.minLength) &&
+        (schema.maxLength === undefined || value.length <= schema.maxLength) &&
+        (!schema.pattern || new RegExp(schema.pattern).test(value))
+      );
+    case 'number':
+    case 'integer':
+      return (
+        typeof value === 'number' &&
+        Number.isFinite(value) &&
+        (schema.type !== 'integer' || Number.isInteger(value)) &&
+        (schema.minimum === undefined || value >= schema.minimum) &&
+        (schema.maximum === undefined || value <= schema.maximum)
+      );
+    case 'boolean':
+      return typeof value === 'boolean';
+    case 'array':
+      return (
+        Array.isArray(value) &&
+        (schema.minItems === undefined || value.length >= schema.minItems) &&
+        (schema.maxItems === undefined || value.length <= schema.maxItems) &&
+        value.every((item) => valid(schema.items, item))
+      );
+    case 'object':
+      return (
+        value !== null &&
+        typeof value === 'object' &&
+        !Array.isArray(value) &&
+        (schema.required ?? []).every((key) => Object.hasOwn(value, key)) &&
+        Object.entries(value).every(([key, item]) => {
+          if (schema.propertyNames && !valid(schema.propertyNames, key)) return false;
+          if (Object.hasOwn(schema.properties ?? {}, key))
+            return valid(schema.properties[key], item);
+          return (
+            schema.additionalProperties !== false &&
+            (typeof schema.additionalProperties !== 'object' ||
+              valid(schema.additionalProperties, item))
+          );
+        })
+      );
+    default:
+      return true;
+  }
+}
+export function convert(schema, raw) {
+  if (schema.type === 'string') return raw;
+  try {
+    return JSON.parse(raw);
+  } catch {
+    return raw;
+  }
+}