@datafuel/sdk 0.1.0 → 0.3.0

package/src/client.ts

@@ -1,19 +1,34 @@
 /** The client. It only moves bytes; `core` decides what goes on the wire. */
 
 import * as core from "./core.js";
-import type { AskOptions, CrawlOptions, MapOptions } from "./core.js";
+import type { AskOptions, CrawlOptions, MapOptions, SearchOptions } from "./core.js";
 import { apiError, DataFuelError, NoApiKey, TransportError, WaitTimeout } from "./errors.js";
 import type {
+  Analytics,
+  AnalyticsOptions,
+  BalanceSplit,
   CallOptions,
+  CancelResult,
   Capability,
   CrawlResult,
   CrawlResultsPage,
   CrawlStatus,
   JobResults,
+  JobsPage,
   JobStatus,
+  JobSummary,
+  JsInstruction,
+  ListOptions,
+  ListTasksOptions,
   Profile,
+  ProxyCountry,
+  ProxyLocation,
   ScrapeOptions,
   SiteMap,
+  TasksPage,
+  TaskSummary,
+  TransactionsOptions,
+  TransactionsPage,
 } from "./models.js";
 import { Capabilities, CrawlPage, isDone, Result } from "./models.js";
 
@@ -46,7 +61,8 @@ type Send = CallOptions & { timeoutMs?: number; signal?: AbortSignal };
  *
  * Pick the call by the shape of the work: one URL is {@link scrape}, a site's
  * URL list is {@link map}, many pages from a start URL is {@link crawl}, a list
- * of known URLs is {@link runJob}, a question for an AI engine is {@link ask}.
+ * of known URLs is {@link runJob}, a question for an AI engine is {@link ask}, a
+ * Google search is {@link search}.
  *
  * Every write carries an `Idempotency-Key`, generated per request, so a retry
  * attaches to the task already running instead of charging twice.
@@ -84,7 +100,7 @@ export class DataFuel {
   }
 
   private async send(request: core.Request, opts: Send = {}): Promise<unknown> {
-    if (!this.apiKey) {
+    if (!this.apiKey && request.auth) {
       throw new NoApiKey("no API key: pass one to the client or set DATAFUEL_API_KEY");
     }
     const url = new URL(this.baseUrl + request.path);
@@ -205,6 +221,12 @@ export class DataFuel {
     return this.runTask(request, options);
   }
 
+  /** Run a Google search and return the results page, parsed to JSON by default. */
+  async search(query: string, options: SearchOptions & CallOptions = {}): Promise<Result> {
+    const request = core.buildSearch(query, options, core.key(options.idempotencyKey));
+    return this.runTask(request, options);
+  }
+
   /**
    * List the URLs of a site without scraping them.
    *
@@ -232,6 +254,19 @@ export class DataFuel {
     return strField(await this.send(request, options), "job_id");
   }
 
+  /**
+   * Stop a crawl. Queued pages are refunded, pages in flight finish and bill.
+   * Throws {@link JobNotCancellable} when it already finished.
+   */
+  async cancelCrawl(crawlId: string, options: CallOptions = {}): Promise<CancelResult> {
+    return cancelResult(
+      await this.send(
+        new core.Request("POST", `/crawl/${core.pathSegment(crawlId)}/cancel`),
+        options,
+      ),
+    );
+  }
+
   /** Return the progress of a crawl. */
   async getCrawl(crawlId: string, options: CallOptions = {}): Promise<CrawlStatus> {
     const body = record(
@@ -363,20 +398,35 @@ export class DataFuel {
     return strField(await this.send(request, options), "id");
   }
 
+  /** Queue a batch of Google searches and return the job id. */
+  async createSearchJob(
+    queries: string[],
+    options: SearchOptions & CallOptions & { sequential?: boolean } = {},
+  ): Promise<string> {
+    const request = core.buildSearchJob(
+      queries,
+      options,
+      options.sequential ?? false,
+      core.key(options.idempotencyKey),
+    );
+    return strField(await this.send(request, options), "id");
+  }
+
   /** Return the progress of a job. */
   async getJob(jobId: string, options: CallOptions = {}): Promise<JobStatus> {
-    const body = record(
+    return jobStatus(
       await this.send(new core.Request("GET", `/job/${core.pathSegment(jobId)}`), options),
     );
-    const raw = body as unknown as Partial<JobStatus>;
-    return {
-      status: raw.status as JobStatus["status"],
-      tasks_count: raw.tasks_count ?? 0,
-      tasks_done: raw.tasks_done ?? 0,
-      tasks_remaining: raw.tasks_remaining ?? 0,
-      total_cost: raw.total_cost ?? 0,
-      done: isDone(raw.status),
-    };
+  }
+
+  /**
+   * Stop a job. Queued tasks are refunded, tasks in flight finish and bill.
+   * Throws {@link JobNotCancellable} when it already finished.
+   */
+  async cancelJob(jobId: string, options: CallOptions = {}): Promise<CancelResult> {
+    return cancelResult(
+      await this.send(new core.Request("POST", `/job/${core.pathSegment(jobId)}/cancel`), options),
+    );
   }
 
   /**
@@ -399,6 +449,35 @@ export class DataFuel {
     };
   }
 
+  /** One page of your jobs and crawls, newest first. Pass `nextCursor` back as `cursor`. */
+  async listJobs(options: ListOptions & CallOptions = {}): Promise<JobsPage> {
+    const body = record(await this.send(core.listJobsRequest(options), options));
+    const next = typeof body.next_cursor === "string" ? body.next_cursor : undefined;
+    return {
+      jobs: array(body.jobs).map((raw) => {
+        const job = record(raw) as unknown as JobSummary;
+        return { ...job, ...jobStatus(job) };
+      }),
+      ...(next ? { nextCursor: next } : {}),
+    };
+  }
+
+  /**
+   * One page of your tasks, newest first, including those of jobs and crawls.
+   * Items carry no result: fetch it with {@link getTask}.
+   */
+  async listTasks(options: ListTasksOptions & CallOptions = {}): Promise<TasksPage> {
+    const body = record(await this.send(core.listTasksRequest(options), options));
+    const next = typeof body.next_cursor === "string" ? body.next_cursor : undefined;
+    return {
+      tasks: array(body.tasks).map((raw) => {
+        const task = record(raw) as unknown as TaskSummary;
+        return { ...task, job_id: task.job_id ?? null };
+      }),
+      ...(next ? { nextCursor: next } : {}),
+    };
+  }
+
   /** Poll until the job is done. Without `timeoutMs` it waits indefinitely. */
   async waitJob(jobId: string, options: CallOptions = {}): Promise<JobStatus> {
     // The timeout bounds the whole wait, not each poll.
@@ -434,21 +513,108 @@ export class DataFuel {
     return this.jobResults(id, options);
   }
 
+  /** Create a search job, wait for it, and return its results. */
+  async runSearchJob(
+    queries: string[],
+    options: SearchOptions & CallOptions & { sequential?: boolean } = {},
+  ): Promise<JobResults> {
+    const id = await this.createSearchJob(queries, options);
+    await this.waitJob(id, options);
+    return this.jobResults(id, options);
+  }
+
   // --- account -----------------------------------------------------------
 
-  /** Which task types and LLM engines are switched on right now. */
+  /** Which task types and LLM engines are switched on right now. Needs no key. */
   async capabilities(options: CallOptions = {}): Promise<Capabilities> {
-    return new Capabilities(
-      record(await this.send(new core.Request("GET", "/capabilities"), options)),
+    const request = new core.Request(
+      "GET",
+      "/config/capabilities",
+      undefined,
+      undefined,
+      undefined,
+      false,
+    );
+    return new Capabilities(record(await this.send(request, options)));
+  }
+
+  /** The browser actions `jsInstructions` accepts, with their arguments. Needs no key. */
+  async jsInstructions(options: CallOptions = {}): Promise<JsInstruction[]> {
+    const request = new core.Request(
+      "GET",
+      "/config/js-instructions",
+      undefined,
+      undefined,
+      undefined,
+      false,
     );
+    const body = record(await this.send(request, options));
+    return Array.isArray(body.instructions) ? (body.instructions as JsInstruction[]) : [];
+  }
+
+  /** Countries, regions and cities a proxy type can exit from. */
+  async proxyLocations(
+    options: CallOptions & { proxyType?: string } = {},
+  ): Promise<ProxyCountry[]> {
+    const params = options.proxyType ? { proxy_type: options.proxyType } : undefined;
+    return list(
+      await this.send(new core.Request("GET", "/config/proxy/locations", params), options),
+    ) as ProxyCountry[];
   }
 
-  /** Remaining credits. */
+  /** ASNs a proxy type can exit from in one country (ISO 3166-1 alpha-2). */
+  async proxyAsns(
+    country: string,
+    options: CallOptions & { proxyType?: string } = {},
+  ): Promise<ProxyLocation[]> {
+    const params: Record<string, string> = { country };
+    if (options.proxyType) params.proxy_type = options.proxyType;
+    return list(
+      await this.send(new core.Request("GET", "/config/proxy/asn", params), options),
+    ) as ProxyLocation[];
+  }
+
+  /** Remaining credits: plan and pay-as-you-go together. */
   async balance(options: CallOptions = {}): Promise<number> {
     const body = await this.send(new core.Request("GET", "/users/@me/balance"), options);
     return intField(body, "balance");
   }
 
+  /** Remaining credits by pool: plan credits (spent first) and pay-as-you-go credits. */
+  async balanceSplit(options: CallOptions = {}): Promise<BalanceSplit> {
+    const body = await this.send(new core.Request("GET", "/users/@me/balance"), options);
+    return {
+      balance: intField(body, "balance"),
+      plan_balance: intField(body, "plan_balance"),
+      payg_balance: intField(body, "payg_balance"),
+    };
+  }
+
+  /**
+   * Credit movements, newest first: plan assignments, credit pack purchases, usage, refunds, expiry.
+   * `sums` totals each operation over the whole range, not just this page.
+   */
+  async transactions(options: TransactionsOptions & CallOptions = {}): Promise<TransactionsPage> {
+    const body = record(await this.send(core.transactionsRequest(options), options));
+    return {
+      transactions: array(body.transactions) as TransactionsPage["transactions"],
+      total_count: Number(body.total_count ?? 0),
+      sums: array(body.sums) as TransactionsPage["sums"],
+    };
+  }
+
+  /** Usage over a date range: totals, a time series and breakdowns. Default: the last 30 days. */
+  async analytics(options: AnalyticsOptions & CallOptions = {}): Promise<Analytics> {
+    const body = record(await this.send(core.analyticsRequest(options), options));
+    return {
+      ...(body as unknown as Analytics),
+      timeseries: array(body.timeseries) as Analytics["timeseries"],
+      by_module: array(body.by_module) as Analytics["by_module"],
+      top_targets: array(body.top_targets) as Analytics["top_targets"],
+      by_status_code: array(body.by_status_code) as Analytics["by_status_code"],
+    };
+  }
+
   /** The account behind the API key. */
   async me(options: CallOptions = {}): Promise<Profile> {
     const body = await this.send(new core.Request("GET", "/users/@me"), options);
@@ -504,6 +670,38 @@ function record(body: unknown): Record<string, unknown> {
   return body as Record<string, unknown>;
 }
 
+function list(body: unknown): unknown[] {
+  if (!Array.isArray(body)) {
+    throw new DataFuelError(`unexpected answer from the API: ${JSON.stringify(body) ?? "empty"}`);
+  }
+  return body;
+}
+
+function array(value: unknown): unknown[] {
+  return Array.isArray(value) ? value : [];
+}
+
+function jobStatus(body: unknown): JobStatus {
+  const raw = record(body) as unknown as Partial<JobStatus>;
+  return {
+    status: raw.status as JobStatus["status"],
+    tasks_count: raw.tasks_count ?? 0,
+    tasks_done: raw.tasks_done ?? 0,
+    tasks_remaining: raw.tasks_remaining ?? 0,
+    total_cost: raw.total_cost ?? 0,
+    done: isDone(raw.status),
+  };
+}
+
+function cancelResult(body: unknown): CancelResult {
+  const raw = record(body) as unknown as Partial<CancelResult>;
+  return {
+    ...jobStatus(body),
+    refunded_tasks: raw.refunded_tasks ?? 0,
+    refunded_credits: raw.refunded_credits ?? 0,
+  };
+}
+
 function field(body: unknown, name: string): unknown {
   const value = record(body)[name];
   if (value === undefined || value === null) {

package/src/core.ts

@@ -7,12 +7,21 @@
  */
 
 import { APIError, Unavailable } from "./errors.js";
-import type { AI, Engine, Proxy, ScrapeOptions } from "./models.js";
+import type {
+  AI,
+  AnalyticsOptions,
+  Engine,
+  ListOptions,
+  ListTasksOptions,
+  Proxy,
+  ScrapeOptions,
+  TransactionsOptions,
+} from "./models.js";
 
 export const DEFAULT_BASE_URL = "https://scraping-api.datafuel.ai/api/v1";
 
 /** Kept in step with package.json by a test; see test/hardening.test.ts. */
-export const VERSION = "0.1.0";
+export const VERSION = "0.3.0";
 
 /** How long a synchronous call waits for a result before giving up. */
 export const DEFAULT_TIMEOUT_MS = 180_000;
@@ -30,6 +39,7 @@ export class Request {
     readonly params?: Record<string, string>,
     readonly body?: Record<string, unknown>,
     readonly idempotencyKey?: string,
+    readonly auth: boolean = true,
   ) {}
 
   /** GETs are safe by nature, writes because they carry an idempotency key. */
@@ -75,10 +85,10 @@ export function headers(
   request: Request,
 ): Record<string, string> {
   const out: Record<string, string> = {
-    "X-API-Key": apiKey,
     Accept: "application/json",
     "User-Agent": userAgent,
   };
+  if (apiKey) out["X-API-Key"] = apiKey;
   if (request.body !== undefined) out["Content-Type"] = "application/json";
   if (request.idempotencyKey !== undefined) out["Idempotency-Key"] = request.idempotencyKey;
   return out;
@@ -215,7 +225,7 @@ export function buildJob(
   sequential: boolean,
   idempotencyKey: string,
 ): Request {
-  const attrs = { ...scrapeAttributes(options), urls: [...urls] };
+  const attrs = { ...scrapeAttributes(options), urls: [...urls], ...proxySession(proxy) };
   return new Request(
     "POST",
     "/job",
@@ -231,6 +241,7 @@ export interface AskOptions {
   websearch?: boolean;
   followUp?: string;
   country?: string;
+  location?: string;
   format?: string;
 }
 
@@ -243,6 +254,7 @@ export function askAttributes(
   if (options.websearch) attrs.websearch = true;
   if (options.followUp) attrs.follow_up_prompt = options.followUp;
   if (options.country) attrs.proxy_country = options.country;
+  if (options.location) attrs.location = options.location;
   if (options.format) attrs.result_format = options.format;
   return attrs;
 }
@@ -335,6 +347,7 @@ export function buildCrawl(
   if (options.excludePaths?.length) attrs.exclude_paths = [...options.excludePaths];
   if (options.includeSubdomains) attrs.include_subdomains = true;
   if (options.allowBackwardLinks) attrs.allow_backward_links = true;
+  Object.assign(attrs, proxySession(proxy));
   return new Request(
     "POST",
     "/crawl",
@@ -344,6 +357,103 @@ export function buildCrawl(
   );
 }
 
+/** Options for `search`, the Google SERP module. Proxy type is not used. */
+export interface SearchOptions {
+  /** Google `gl`, e.g. "us". */
+  country?: string;
+  /** Google `hl`, e.g. "en". */
+  language?: string;
+  /** Canonical location name. Give at most one of location, uule, lat/lon. */
+  location?: string;
+  /** 1-based, default 1. */
+  page?: number;
+  /** e.g. "google.de". */
+  googleDomain?: string;
+  uule?: string;
+  lat?: number;
+  lon?: number;
+  /** Metres around lat/lon or location, max 1000. */
+  radius?: number;
+  cr?: string;
+  lr?: string;
+  tbs?: string;
+  safe?: "active" | "off";
+  nfpr?: boolean;
+  filter?: boolean;
+  uds?: string;
+  kgmid?: string;
+  si?: string;
+  ludocid?: string;
+  lsig?: string;
+  ibp?: string;
+  /** Exit country of the request. */
+  proxyCountry?: string;
+  /** Default json. */
+  format?: "json" | "html" | "markdown";
+}
+
+export function searchAttributes(
+  queryField: string,
+  query: string | string[],
+  options: SearchOptions,
+): Record<string, unknown> {
+  const attrs: Record<string, unknown> = { [queryField]: query };
+  const strings: [keyof SearchOptions, string][] = [
+    ["country", "country"],
+    ["language", "language"],
+    ["location", "location"],
+    ["googleDomain", "google_domain"],
+    ["uule", "uule"],
+    ["cr", "cr"],
+    ["lr", "lr"],
+    ["tbs", "tbs"],
+    ["safe", "safe"],
+    ["uds", "uds"],
+    ["kgmid", "kgmid"],
+    ["si", "si"],
+    ["ludocid", "ludocid"],
+    ["lsig", "lsig"],
+    ["ibp", "ibp"],
+    ["proxyCountry", "proxy_country"],
+    ["format", "result_format"],
+  ];
+  for (const [option, attribute] of strings) {
+    if (options[option]) attrs[attribute] = options[option];
+  }
+  if (options.page) attrs.page = options.page;
+  if (options.radius) attrs.radius = options.radius;
+  if (options.lat !== undefined) attrs.lat = options.lat;
+  if (options.lon !== undefined) attrs.lon = options.lon;
+  if (options.nfpr !== undefined) attrs.nfpr = options.nfpr;
+  if (options.filter !== undefined) attrs.filter = options.filter;
+  return attrs;
+}
+
+export function buildSearch(
+  query: string,
+  options: SearchOptions,
+  idempotencyKey: string,
+): Request {
+  const attrs = searchAttributes("query", query, options);
+  return new Request("POST", "/task", undefined, envelope("serp", attrs), idempotencyKey);
+}
+
+export function buildSearchJob(
+  queries: string[],
+  options: SearchOptions,
+  sequential: boolean,
+  idempotencyKey: string,
+): Request {
+  const attrs = searchAttributes("queries", [...queries], options);
+  return new Request(
+    "POST",
+    "/job",
+    undefined,
+    envelope("serp", attrs, { multithreaded: !sequential }),
+    idempotencyKey,
+  );
+}
+
 export function crawlResultsRequest(crawlId: string, cursor?: string, limit?: number): Request {
   const params: Record<string, string> = {};
   if (cursor) params.cursor = cursor;
@@ -355,6 +465,64 @@ export function crawlResultsRequest(crawlId: string, cursor?: string, limit?: nu
   );
 }
 
+/** A Date as its UTC day, which is what the API's date filters take. */
+export function day(value: string | Date): string {
+  return typeof value === "string" ? value : value.toISOString().slice(0, 10);
+}
+
+function query(
+  entries: [string, string | number | Date | undefined][],
+): Record<string, string> | undefined {
+  const params: Record<string, string> = {};
+  for (const [name, value] of entries) {
+    if (value === undefined || value === "") continue;
+    params[name] = value instanceof Date ? day(value) : String(value);
+  }
+  return Object.keys(params).length > 0 ? params : undefined;
+}
+
+function listRequest(path: string, options: ListOptions, jobId?: string): Request {
+  const params = query([
+    ["status", options.status],
+    ["type", options.type],
+    ["job_id", jobId],
+    ["start_date", options.startDate],
+    ["end_date", options.endDate],
+    ["limit", options.limit],
+    ["cursor", options.cursor],
+  ]);
+  return new Request("GET", path, params);
+}
+
+export function listJobsRequest(options: ListOptions): Request {
+  return listRequest("/job", options);
+}
+
+export function listTasksRequest(options: ListTasksOptions): Request {
+  return listRequest("/task", options, options.jobId);
+}
+
+export function transactionsRequest(options: TransactionsOptions): Request {
+  const params = query([
+    ["operation", options.operation],
+    ["start_date", options.startDate],
+    ["end_date", options.endDate],
+    ["page", options.page],
+    ["limit", options.limit],
+  ]);
+  return new Request("GET", "/users/@me/transactions", params);
+}
+
+export function analyticsRequest(options: AnalyticsOptions): Request {
+  const params = query([
+    ["start_date", options.startDate],
+    ["end_date", options.endDate],
+    ["interval", options.interval],
+    ["module", options.module],
+  ]);
+  return new Request("GET", "/task/analytics/dashboard", params);
+}
+
 /** Whether the API answered "the task is still running" instead of a result. */
 export function stillProcessing(body: unknown): boolean {
   return (

package/src/errors.ts

@@ -24,10 +24,47 @@ export class NoApiKey extends DataFuelError {}
 /** The request never got an answer: DNS, connection, abort, read timeout. */
 export class TransportError extends DataFuelError {}
 
+/** The `code` of an API error. The API may add codes; unknown ones pass through. */
+export type ErrorCode =
+  | "UNAUTHORIZED"
+  | "INVALID_API_KEY"
+  | "FORBIDDEN"
+  | "INSUFFICIENT_CREDITS"
+  | "RATE_LIMIT_EXCEEDED"
+  | "CONCURRENCY_LIMIT_REACHED"
+  | "INVALID_REQUEST_BODY"
+  | "INVALID_ATTRIBUTES"
+  | "MISSING_TARGET"
+  | "UNSUPPORTED_TASK_TYPE"
+  | "JOB_REQUIRES_MULTIPLE_TARGETS"
+  | "INVALID_IDEMPOTENCY_KEY"
+  | "IDEMPOTENCY_KEY_REUSED"
+  | "INVALID_TASK_ID"
+  | "INVALID_JOB_ID"
+  | "TASK_NOT_FOUND"
+  | "JOB_NOT_FOUND"
+  | "CRAWL_NOT_FOUND"
+  | "JOB_NOT_CANCELLABLE"
+  | "INVALID_CRAWL_PATTERN"
+  | "CRAWL_UNSUPPORTED_OPTION"
+  | "INVALID_CURSOR"
+  | "INVALID_QUERY_PARAM"
+  | "INVALID_PROXY_TYPE"
+  | "INVALID_COUNTRY"
+  | "INVALID_DATE_FORMAT"
+  | "INVALID_DATE_RANGE"
+  | "INVALID_INTERVAL"
+  | "MODULE_UNAVAILABLE"
+  | "ENGINE_UNAVAILABLE"
+  | "API_KEY_RESET_FAILED"
+  | "TASK_RESULT_TIMEOUT"
+  | "INTERNAL_ERROR"
+  | (string & {});
+
 /** A non-2xx answer from the API itself. */
 export class APIError extends DataFuelError {
   readonly status: number;
-  readonly code: string;
+  readonly code: ErrorCode;
   /** Seconds the API asked us to wait, from Retry-After. 0 when absent. */
   readonly retryAfter: number;
 
@@ -41,6 +78,8 @@ export class APIError extends DataFuelError {
 
 /** 401: the API key is missing or invalid. */
 export class Unauthorized extends APIError {}
+/** 403 FORBIDDEN: the account is inactive. */
+export class Forbidden extends APIError {}
 /** 404: unknown id, or one that belongs to another account. */
 export class NotFound extends APIError {}
 /** 429: the account's request rate or concurrency limit was reached. */
@@ -51,7 +90,9 @@ export class InsufficientCredits extends APIError {}
 export class InvalidAttributes extends APIError {}
 /** 422: the key was already used for a different request. */
 export class IdempotencyKeyReused extends APIError {}
-/** 503: an operator switched something off. The message carries the reason. */
+/** 409 JOB_NOT_CANCELLABLE: the job or crawl already finished. */
+export class JobNotCancellable extends APIError {}
+/** 503: an operator switched something off, or a dependency is down. The message carries the reason. */
 export class Unavailable extends APIError {}
 /** 503 MODULE_UNAVAILABLE: this task type is switched off. Nothing was charged. */
 export class ModuleUnavailable extends Unavailable {}
@@ -109,12 +150,16 @@ const BY_CODE: Record<string, new (s: number, c: string, m: string, r?: number)
   INVALID_ATTRIBUTES: InvalidAttributes,
   IDEMPOTENCY_KEY_REUSED: IdempotencyKeyReused,
   INVALID_API_KEY: Unauthorized,
+  FORBIDDEN: Forbidden,
+  JOB_NOT_CANCELLABLE: JobNotCancellable,
 };
 
 const BY_STATUS: Record<number, new (s: number, c: string, m: string, r?: number) => APIError> = {
   401: Unauthorized,
   402: InsufficientCredits,
+  403: Forbidden,
   404: NotFound,
+  409: JobNotCancellable,
   422: IdempotencyKeyReused,
   429: RateLimited,
   503: Unavailable,