@datafuel/sdk 0.1.0 → 0.3.0

package/dist/index.js

@@ -23,6 +23,8 @@ var APIError = class extends DataFuelError {
 };
 var Unauthorized = class extends APIError {
 };
+var Forbidden = class extends APIError {
+};
 var NotFound = class extends APIError {
 };
 var RateLimited = class extends APIError {
@@ -33,6 +35,8 @@ var InvalidAttributes = class extends APIError {
 };
 var IdempotencyKeyReused = class extends APIError {
 };
+var JobNotCancellable = class extends APIError {
+};
 var Unavailable = class extends APIError {
 };
 var ModuleUnavailable = class extends Unavailable {
@@ -68,12 +72,16 @@ var BY_CODE = {
   INSUFFICIENT_CREDITS: InsufficientCredits,
   INVALID_ATTRIBUTES: InvalidAttributes,
   IDEMPOTENCY_KEY_REUSED: IdempotencyKeyReused,
-  INVALID_API_KEY: Unauthorized
+  INVALID_API_KEY: Unauthorized,
+  FORBIDDEN: Forbidden,
+  JOB_NOT_CANCELLABLE: JobNotCancellable
 };
 var BY_STATUS = {
   401: Unauthorized,
   402: InsufficientCredits,
+  403: Forbidden,
   404: NotFound,
+  409: JobNotCancellable,
   422: IdempotencyKeyReused,
   429: RateLimited,
   503: Unavailable
@@ -94,24 +102,26 @@ function apiError(status, body, retryAfter = 0) {
 
 // src/core.ts
 var DEFAULT_BASE_URL = "https://scraping-api.datafuel.ai/api/v1";
-var VERSION = "0.1.0";
+var VERSION = "0.3.0";
 var DEFAULT_TIMEOUT_MS = 18e4;
 var STILL_PROCESSING_DELAY_MS = 2e3;
 var STILL_PROCESSING = "TASK_STILL_PROCESSING";
 var MAX_IDEMPOTENCY_KEY = 255;
 var Request = class {
-  constructor(method, path, params, body, idempotencyKey) {
+  constructor(method, path, params, body, idempotencyKey, auth = true) {
     this.method = method;
     this.path = path;
     this.params = params;
     this.body = body;
     this.idempotencyKey = idempotencyKey;
+    this.auth = auth;
   }
   method;
   path;
   params;
   body;
   idempotencyKey;
+  auth;
   /** GETs are safe by nature, writes because they carry an idempotency key. */
   get retryable() {
     return this.method === "GET" || this.idempotencyKey !== void 0;
@@ -138,10 +148,10 @@ function newIdempotencyKey() {
 }
 function headers(apiKey, userAgent, request) {
   const out = {
-    "X-API-Key": apiKey,
     Accept: "application/json",
     "User-Agent": userAgent
   };
+  if (apiKey) out["X-API-Key"] = apiKey;
   if (request.body !== void 0) out["Content-Type"] = "application/json";
   if (request.idempotencyKey !== void 0) out["Idempotency-Key"] = request.idempotencyKey;
   return out;
@@ -237,7 +247,7 @@ function buildScrape(url, options, proxy, idempotencyKey) {
   );
 }
 function buildJob(urls, options, proxy, sequential, idempotencyKey) {
-  const attrs = { ...scrapeAttributes(options), urls: [...urls] };
+  const attrs = { ...scrapeAttributes(options), urls: [...urls], ...proxySession(proxy) };
   return new Request(
     "POST",
     "/job",
@@ -251,6 +261,7 @@ function askAttributes(promptField, prompt, options) {
   if (options.websearch) attrs.websearch = true;
   if (options.followUp) attrs.follow_up_prompt = options.followUp;
   if (options.country) attrs.proxy_country = options.country;
+  if (options.location) attrs.location = options.location;
   if (options.format) attrs.result_format = options.format;
   return attrs;
 }
@@ -293,6 +304,7 @@ function buildCrawl(url, options, proxy, idempotencyKey) {
   if (options.excludePaths?.length) attrs.exclude_paths = [...options.excludePaths];
   if (options.includeSubdomains) attrs.include_subdomains = true;
   if (options.allowBackwardLinks) attrs.allow_backward_links = true;
+  Object.assign(attrs, proxySession(proxy));
   return new Request(
     "POST",
     "/crawl",
@@ -301,6 +313,52 @@ function buildCrawl(url, options, proxy, idempotencyKey) {
     idempotencyKey
   );
 }
+function searchAttributes(queryField, query2, options) {
+  const attrs = { [queryField]: query2 };
+  const strings = [
+    ["country", "country"],
+    ["language", "language"],
+    ["location", "location"],
+    ["googleDomain", "google_domain"],
+    ["uule", "uule"],
+    ["cr", "cr"],
+    ["lr", "lr"],
+    ["tbs", "tbs"],
+    ["safe", "safe"],
+    ["uds", "uds"],
+    ["kgmid", "kgmid"],
+    ["si", "si"],
+    ["ludocid", "ludocid"],
+    ["lsig", "lsig"],
+    ["ibp", "ibp"],
+    ["proxyCountry", "proxy_country"],
+    ["format", "result_format"]
+  ];
+  for (const [option, attribute] of strings) {
+    if (options[option]) attrs[attribute] = options[option];
+  }
+  if (options.page) attrs.page = options.page;
+  if (options.radius) attrs.radius = options.radius;
+  if (options.lat !== void 0) attrs.lat = options.lat;
+  if (options.lon !== void 0) attrs.lon = options.lon;
+  if (options.nfpr !== void 0) attrs.nfpr = options.nfpr;
+  if (options.filter !== void 0) attrs.filter = options.filter;
+  return attrs;
+}
+function buildSearch(query2, options, idempotencyKey) {
+  const attrs = searchAttributes("query", query2, options);
+  return new Request("POST", "/task", void 0, envelope("serp", attrs), idempotencyKey);
+}
+function buildSearchJob(queries, options, sequential, idempotencyKey) {
+  const attrs = searchAttributes("queries", [...queries], options);
+  return new Request(
+    "POST",
+    "/job",
+    void 0,
+    envelope("serp", attrs, { multithreaded: !sequential }),
+    idempotencyKey
+  );
+}
 function crawlResultsRequest(crawlId, cursor, limit) {
   const params = {};
   if (cursor) params.cursor = cursor;
@@ -311,6 +369,54 @@ function crawlResultsRequest(crawlId, cursor, limit) {
     Object.keys(params).length > 0 ? params : void 0
   );
 }
+function day(value) {
+  return typeof value === "string" ? value : value.toISOString().slice(0, 10);
+}
+function query(entries) {
+  const params = {};
+  for (const [name, value] of entries) {
+    if (value === void 0 || value === "") continue;
+    params[name] = value instanceof Date ? day(value) : String(value);
+  }
+  return Object.keys(params).length > 0 ? params : void 0;
+}
+function listRequest(path, options, jobId) {
+  const params = query([
+    ["status", options.status],
+    ["type", options.type],
+    ["job_id", jobId],
+    ["start_date", options.startDate],
+    ["end_date", options.endDate],
+    ["limit", options.limit],
+    ["cursor", options.cursor]
+  ]);
+  return new Request("GET", path, params);
+}
+function listJobsRequest(options) {
+  return listRequest("/job", options);
+}
+function listTasksRequest(options) {
+  return listRequest("/task", options, options.jobId);
+}
+function transactionsRequest(options) {
+  const params = query([
+    ["operation", options.operation],
+    ["start_date", options.startDate],
+    ["end_date", options.endDate],
+    ["page", options.page],
+    ["limit", options.limit]
+  ]);
+  return new Request("GET", "/users/@me/transactions", params);
+}
+function analyticsRequest(options) {
+  const params = query([
+    ["start_date", options.startDate],
+    ["end_date", options.endDate],
+    ["interval", options.interval],
+    ["module", options.module]
+  ]);
+  return new Request("GET", "/task/analytics/dashboard", params);
+}
 function stillProcessing(body) {
   return body !== null && typeof body === "object" && body.code === STILL_PROCESSING;
 }
@@ -474,7 +580,7 @@ var DataFuel = class {
     return new Promise((resolve) => setTimeout(resolve, ms));
   }
   async send(request, opts = {}) {
-    if (!this.apiKey) {
+    if (!this.apiKey && request.auth) {
       throw new NoApiKey("no API key: pass one to the client or set DATAFUEL_API_KEY");
     }
     const url = new URL(this.baseUrl + request.path);
@@ -578,6 +684,11 @@ var DataFuel = class {
     const request = buildAsk(prompt, options, key(options.idempotencyKey));
     return this.runTask(request, options);
   }
+  /** Run a Google search and return the results page, parsed to JSON by default. */
+  async search(query2, options = {}) {
+    const request = buildSearch(query2, options, key(options.idempotencyKey));
+    return this.runTask(request, options);
+  }
   /**
    * List the URLs of a site without scraping them.
    *
@@ -602,6 +713,18 @@ var DataFuel = class {
     const request = buildCrawl(url, options, options.proxy, key(options.idempotencyKey));
     return strField(await this.send(request, options), "job_id");
   }
+  /**
+   * Stop a crawl. Queued pages are refunded, pages in flight finish and bill.
+   * Throws {@link JobNotCancellable} when it already finished.
+   */
+  async cancelCrawl(crawlId, options = {}) {
+    return cancelResult(
+      await this.send(
+        new Request("POST", `/crawl/${pathSegment(crawlId)}/cancel`),
+        options
+      )
+    );
+  }
   /** Return the progress of a crawl. */
   async getCrawl(crawlId, options = {}) {
     const body = record(
@@ -712,20 +835,30 @@ var DataFuel = class {
     );
     return strField(await this.send(request, options), "id");
   }
+  /** Queue a batch of Google searches and return the job id. */
+  async createSearchJob(queries, options = {}) {
+    const request = buildSearchJob(
+      queries,
+      options,
+      options.sequential ?? false,
+      key(options.idempotencyKey)
+    );
+    return strField(await this.send(request, options), "id");
+  }
   /** Return the progress of a job. */
   async getJob(jobId, options = {}) {
-    const body = record(
+    return jobStatus(
       await this.send(new Request("GET", `/job/${pathSegment(jobId)}`), options)
     );
-    const raw = body;
-    return {
-      status: raw.status,
-      tasks_count: raw.tasks_count ?? 0,
-      tasks_done: raw.tasks_done ?? 0,
-      tasks_remaining: raw.tasks_remaining ?? 0,
-      total_cost: raw.total_cost ?? 0,
-      done: isDone(raw.status)
-    };
+  }
+  /**
+   * Stop a job. Queued tasks are refunded, tasks in flight finish and bill.
+   * Throws {@link JobNotCancellable} when it already finished.
+   */
+  async cancelJob(jobId, options = {}) {
+    return cancelResult(
+      await this.send(new Request("POST", `/job/${pathSegment(jobId)}/cancel`), options)
+    );
   }
   /**
    * Return the per-task results of a job.
@@ -746,6 +879,33 @@ var DataFuel = class {
       tasks: tasks.map((task) => new Result(record(task)))
     };
   }
+  /** One page of your jobs and crawls, newest first. Pass `nextCursor` back as `cursor`. */
+  async listJobs(options = {}) {
+    const body = record(await this.send(listJobsRequest(options), options));
+    const next = typeof body.next_cursor === "string" ? body.next_cursor : void 0;
+    return {
+      jobs: array(body.jobs).map((raw) => {
+        const job = record(raw);
+        return { ...job, ...jobStatus(job) };
+      }),
+      ...next ? { nextCursor: next } : {}
+    };
+  }
+  /**
+   * One page of your tasks, newest first, including those of jobs and crawls.
+   * Items carry no result: fetch it with {@link getTask}.
+   */
+  async listTasks(options = {}) {
+    const body = record(await this.send(listTasksRequest(options), options));
+    const next = typeof body.next_cursor === "string" ? body.next_cursor : void 0;
+    return {
+      tasks: array(body.tasks).map((raw) => {
+        const task = record(raw);
+        return { ...task, job_id: task.job_id ?? null };
+      }),
+      ...next ? { nextCursor: next } : {}
+    };
+  }
   /** Poll until the job is done. Without `timeoutMs` it waits indefinitely. */
   async waitJob(jobId, options = {}) {
     const { timeoutMs, ...perPoll } = options;
@@ -771,18 +931,90 @@ var DataFuel = class {
     await this.waitJob(id, options);
     return this.jobResults(id, options);
   }
+  /** Create a search job, wait for it, and return its results. */
+  async runSearchJob(queries, options = {}) {
+    const id = await this.createSearchJob(queries, options);
+    await this.waitJob(id, options);
+    return this.jobResults(id, options);
+  }
   // --- account -----------------------------------------------------------
-  /** Which task types and LLM engines are switched on right now. */
+  /** Which task types and LLM engines are switched on right now. Needs no key. */
   async capabilities(options = {}) {
-    return new Capabilities(
-      record(await this.send(new Request("GET", "/capabilities"), options))
+    const request = new Request(
+      "GET",
+      "/config/capabilities",
+      void 0,
+      void 0,
+      void 0,
+      false
+    );
+    return new Capabilities(record(await this.send(request, options)));
+  }
+  /** The browser actions `jsInstructions` accepts, with their arguments. Needs no key. */
+  async jsInstructions(options = {}) {
+    const request = new Request(
+      "GET",
+      "/config/js-instructions",
+      void 0,
+      void 0,
+      void 0,
+      false
+    );
+    const body = record(await this.send(request, options));
+    return Array.isArray(body.instructions) ? body.instructions : [];
+  }
+  /** Countries, regions and cities a proxy type can exit from. */
+  async proxyLocations(options = {}) {
+    const params = options.proxyType ? { proxy_type: options.proxyType } : void 0;
+    return list(
+      await this.send(new Request("GET", "/config/proxy/locations", params), options)
+    );
+  }
+  /** ASNs a proxy type can exit from in one country (ISO 3166-1 alpha-2). */
+  async proxyAsns(country, options = {}) {
+    const params = { country };
+    if (options.proxyType) params.proxy_type = options.proxyType;
+    return list(
+      await this.send(new Request("GET", "/config/proxy/asn", params), options)
     );
   }
-  /** Remaining credits. */
+  /** Remaining credits: plan and pay-as-you-go together. */
   async balance(options = {}) {
     const body = await this.send(new Request("GET", "/users/@me/balance"), options);
     return intField(body, "balance");
   }
+  /** Remaining credits by pool: plan credits (spent first) and pay-as-you-go credits. */
+  async balanceSplit(options = {}) {
+    const body = await this.send(new Request("GET", "/users/@me/balance"), options);
+    return {
+      balance: intField(body, "balance"),
+      plan_balance: intField(body, "plan_balance"),
+      payg_balance: intField(body, "payg_balance")
+    };
+  }
+  /**
+   * Credit movements, newest first: plan assignments, credit pack purchases, usage, refunds, expiry.
+   * `sums` totals each operation over the whole range, not just this page.
+   */
+  async transactions(options = {}) {
+    const body = record(await this.send(transactionsRequest(options), options));
+    return {
+      transactions: array(body.transactions),
+      total_count: Number(body.total_count ?? 0),
+      sums: array(body.sums)
+    };
+  }
+  /** Usage over a date range: totals, a time series and breakdowns. Default: the last 30 days. */
+  async analytics(options = {}) {
+    const body = record(await this.send(analyticsRequest(options), options));
+    return {
+      ...body,
+      timeseries: array(body.timeseries),
+      by_module: array(body.by_module),
+      top_targets: array(body.top_targets),
+      by_status_code: array(body.by_status_code)
+    };
+  }
   /** The account behind the API key. */
   async me(options = {}) {
     const body = await this.send(new Request("GET", "/users/@me"), options);
@@ -828,6 +1060,34 @@ function record(body) {
   }
   return body;
 }
+function list(body) {
+  if (!Array.isArray(body)) {
+    throw new DataFuelError(`unexpected answer from the API: ${JSON.stringify(body) ?? "empty"}`);
+  }
+  return body;
+}
+function array(value) {
+  return Array.isArray(value) ? value : [];
+}
+function jobStatus(body) {
+  const raw = record(body);
+  return {
+    status: raw.status,
+    tasks_count: raw.tasks_count ?? 0,
+    tasks_done: raw.tasks_done ?? 0,
+    tasks_remaining: raw.tasks_remaining ?? 0,
+    total_cost: raw.total_cost ?? 0,
+    done: isDone(raw.status)
+  };
+}
+function cancelResult(body) {
+  const raw = record(body);
+  return {
+    ...jobStatus(body),
+    refunded_tasks: raw.refunded_tasks ?? 0,
+    refunded_credits: raw.refunded_credits ?? 0
+  };
+}
 function field(body, name) {
   const value = record(body)[name];
   if (value === void 0 || value === null) {
@@ -854,9 +1114,11 @@ export {
   DataFuel,
   DataFuelError,
   EngineUnavailable,
+  Forbidden,
   IdempotencyKeyReused,
   InsufficientCredits,
   InvalidAttributes,
+  JobNotCancellable,
   ModuleUnavailable,
   NoApiKey,
   NotFound,