@datafuel/sdk 0.2.0 → 0.4.0

package/src/client.ts

@@ -4,14 +4,17 @@ import * as core from "./core.js";
 import type { AskOptions, CrawlOptions, MapOptions, SearchOptions } from "./core.js";
 import { apiError, DataFuelError, NoApiKey, TransportError, WaitTimeout } from "./errors.js";
 import type {
+  AIProvider,
   Analytics,
   AnalyticsOptions,
+  BalanceSplit,
   CallOptions,
   CancelResult,
   Capability,
   CrawlResult,
   CrawlResultsPage,
   CrawlStatus,
+  Health,
   JobResults,
   JobsPage,
   JobStatus,
@@ -20,6 +23,7 @@ import type {
   ListOptions,
   ListTasksOptions,
   Profile,
+  ProtectionCheck,
   ProxyCountry,
   ProxyLocation,
   ScrapeOptions,
@@ -124,7 +128,7 @@ export class DataFuel {
         // none) throws "Illegal invocation".
         const send = this.fetchImpl;
         const response = await send(url, signal ? { ...init, signal } : init);
-        return await parse(response);
+        return await parse(response, request.degradedOk);
       } catch (caught) {
         if (isAbort(caught)) {
           throw new TransportError("the request was aborted or timed out", { cause: caught });
@@ -551,6 +555,40 @@ export class DataFuel {
     return Array.isArray(body.instructions) ? (body.instructions as JsInstruction[]) : [];
   }
 
+  /** The LLM providers and models `ai` accepts. Needs no key. */
+  async aiProviders(options: CallOptions = {}): Promise<AIProvider[]> {
+    const request = new core.Request(
+      "GET",
+      "/config/ai-providers",
+      undefined,
+      undefined,
+      undefined,
+      false,
+    );
+    const body = record(await this.send(request, options));
+    return Array.isArray(body.providers) ? (body.providers as AIProvider[]) : [];
+  }
+
+  /**
+   * Whether the API is up. `deep` also checks the dependencies it needs to
+   * serve scrapes. A degraded report is returned, not thrown: read `ok`.
+   * Needs no key.
+   */
+  async health(options: CallOptions & { deep?: boolean } = {}): Promise<Health> {
+    const body = record(await this.send(core.healthRequest(options.deep ?? false), options));
+    return { ...(body as unknown as Health), ok: body.status === "ok" };
+  }
+
+  /**
+   * Which anti-bot protection sits in front of each URL. Nothing is scraped
+   * and nothing is charged; invalid URLs are skipped.
+   */
+  async checkProtection(urls: string[], options: CallOptions = {}): Promise<ProtectionCheck[]> {
+    return list(
+      await this.send(new core.Request("POST", "/filter/check", undefined, [...urls]), options),
+    ) as ProtectionCheck[];
+  }
+
   /** Countries, regions and cities a proxy type can exit from. */
   async proxyLocations(
     options: CallOptions & { proxyType?: string } = {},
@@ -573,14 +611,24 @@ export class DataFuel {
     ) as ProxyLocation[];
   }
 
-  /** Remaining credits. */
+  /** Remaining credits: plan and pay-as-you-go together. */
   async balance(options: CallOptions = {}): Promise<number> {
     const body = await this.send(new core.Request("GET", "/users/@me/balance"), options);
     return intField(body, "balance");
   }
 
+  /** Remaining credits by pool: plan credits (spent first) and pay-as-you-go credits. */
+  async balanceSplit(options: CallOptions = {}): Promise<BalanceSplit> {
+    const body = await this.send(new core.Request("GET", "/users/@me/balance"), options);
+    return {
+      balance: intField(body, "balance"),
+      plan_balance: intField(body, "plan_balance"),
+      payg_balance: intField(body, "payg_balance"),
+    };
+  }
+
   /**
-   * Credit movements, newest first: purchases, usage, refunds, expiry.
+   * Credit movements, newest first: plan assignments, credit pack purchases, usage, refunds, expiry.
    * `sums` totals each operation over the whole range, not just this page.
    */
   async transactions(options: TransactionsOptions & CallOptions = {}): Promise<TransactionsPage> {
@@ -629,7 +677,7 @@ function envApiKey(): string | undefined {
   return typeof process !== "undefined" ? process.env?.DATAFUEL_API_KEY : undefined;
 }
 
-async function parse(response: Response): Promise<unknown> {
+async function parse(response: Response, degradedOk = false): Promise<unknown> {
   const text = await response.text();
   let body: unknown;
   if (text.length > 0) {
@@ -639,6 +687,7 @@ async function parse(response: Response): Promise<unknown> {
       body = text;
     }
   }
+  if (degradedOk && response.status === 503 && isHealthReport(body)) return body;
   if (!response.ok) {
     const header = response.headers.get("Retry-After");
     const retryAfter = header !== null && !Number.isNaN(Number(header)) ? Number(header) : 0;
@@ -647,6 +696,10 @@ async function parse(response: Response): Promise<unknown> {
   return body;
 }
 
+function isHealthReport(body: unknown): boolean {
+  return body !== null && typeof body === "object" && "status" in body;
+}
+
 function isAbort(error: unknown): boolean {
   return error instanceof Error && (error.name === "AbortError" || error.name === "TimeoutError");
 }

package/src/core.ts

@@ -21,7 +21,7 @@ import type {
 export const DEFAULT_BASE_URL = "https://scraping-api.datafuel.ai/api/v1";
 
 /** Kept in step with package.json by a test; see test/hardening.test.ts. */
-export const VERSION = "0.2.0";
+export const VERSION = "0.4.0";
 
 /** How long a synchronous call waits for a result before giving up. */
 export const DEFAULT_TIMEOUT_MS = 180_000;
@@ -37,9 +37,11 @@ export class Request {
     readonly method: string,
     readonly path: string,
     readonly params?: Record<string, string>,
-    readonly body?: Record<string, unknown>,
+    readonly body?: Record<string, unknown> | unknown[],
     readonly idempotencyKey?: string,
     readonly auth: boolean = true,
+    /** A 503 that carries a health report is an answer, not an error. */
+    readonly degradedOk: boolean = false,
   ) {}
 
   /** GETs are safe by nature, writes because they carry an idempotency key. */
@@ -109,23 +111,10 @@ function aiAttributes(ai: AI): Record<string, unknown> {
   return out;
 }
 
-/**
- * Catch a half-filled AI key before it costs a request. The API does not check
- * these, so a missing one would be spent on a scrape whose AI step then fails.
- */
+/** The API answers 400 INVALID_ATTRIBUTES without a provider; say so before sending. */
 export function validateAI(ai: AI): void {
-  const given = {
-    provider: ai.provider !== undefined,
-    model: ai.model !== undefined,
-    apiKey: ai.apiKey !== undefined,
-  };
-  const present = Object.values(given).filter(Boolean).length;
-  if (present > 0 && present < 3) {
-    const missing = Object.entries(given)
-      .filter(([, ok]) => !ok)
-      .map(([name]) => name)
-      .join(", ");
-    throw new TypeError(`ai needs provider, model and apiKey together; missing: ${missing}`);
+  if (!ai.provider) {
+    throw new TypeError("ai needs a provider; aiProviders() lists the ones the API supports");
   }
 }
 
@@ -141,7 +130,12 @@ export function scrapeAttributes(options: ScrapeOptions = {}): Record<string, un
   if (options.waitFor) attrs.wait_for_selector = options.waitFor;
   if (options.waitForTimeoutMs) attrs.wait_for_selector_timeout_ms = options.waitForTimeoutMs;
   if (options.jsInstructions) attrs.js_instructions = options.jsInstructions;
-  if (options.blockResource) attrs.block_resource = options.blockResource;
+  if (options.blockResource?.length) {
+    attrs.block_resource =
+      typeof options.blockResource === "string"
+        ? options.blockResource
+        : [...options.blockResource];
+  }
   if (options.mainContentOnly) attrs.main_content_only = true;
   if (options.includeImages !== undefined) attrs.include_images = options.includeImages;
   if (options.extract) attrs.extract_selector = selector(options.extract);
@@ -386,7 +380,10 @@ export interface SearchOptions {
   ludocid?: string;
   lsig?: string;
   ibp?: string;
-  /** Exit country of the request. */
+  /**
+   * Accepted but not used yet: searches leave through DataFuel's own pool.
+   * Target a market with `country` and `language`.
+   */
   proxyCountry?: string;
   /** Default json. */
   format?: "json" | "html" | "markdown";
@@ -465,6 +462,18 @@ export function crawlResultsRequest(crawlId: string, cursor?: string, limit?: nu
   );
 }
 
+export function healthRequest(deep: boolean): Request {
+  return new Request(
+    "GET",
+    "/healthz",
+    deep ? { deep: "1" } : undefined,
+    undefined,
+    undefined,
+    false,
+    true,
+  );
+}
+
 /** A Date as its UTC day, which is what the API's date filters take. */
 export function day(value: string | Date): string {
   return typeof value === "string" ? value : value.toISOString().slice(0, 10);

package/src/errors.ts

@@ -45,6 +45,10 @@ export type ErrorCode =
   | "JOB_NOT_FOUND"
   | "CRAWL_NOT_FOUND"
   | "JOB_NOT_CANCELLABLE"
+  | "TASK_ALREADY_EXISTS"
+  | "JOB_ALREADY_EXISTS"
+  | "URL_LENGTH_CANNOT_BE_ZERO"
+  | "ANALYTICS_FETCH_FAILED"
   | "INVALID_CRAWL_PATTERN"
   | "CRAWL_UNSUPPORTED_OPTION"
   | "INVALID_CURSOR"
@@ -92,6 +96,12 @@ export class InvalidAttributes extends APIError {}
 export class IdempotencyKeyReused extends APIError {}
 /** 409 JOB_NOT_CANCELLABLE: the job or crawl already finished. */
 export class JobNotCancellable extends APIError {}
+/**
+ * 409 TASK_ALREADY_EXISTS / JOB_ALREADY_EXISTS: the create collided with an
+ * existing task or job and is not an idempotent replay. Nothing was charged;
+ * send the request again.
+ */
+export class AlreadyExists extends APIError {}
 /** 503: an operator switched something off, or a dependency is down. The message carries the reason. */
 export class Unavailable extends APIError {}
 /** 503 MODULE_UNAVAILABLE: this task type is switched off. Nothing was charged. */
@@ -152,6 +162,8 @@ const BY_CODE: Record<string, new (s: number, c: string, m: string, r?: number)
   INVALID_API_KEY: Unauthorized,
   FORBIDDEN: Forbidden,
   JOB_NOT_CANCELLABLE: JobNotCancellable,
+  TASK_ALREADY_EXISTS: AlreadyExists,
+  JOB_ALREADY_EXISTS: AlreadyExists,
 };
 
 const BY_STATUS: Record<number, new (s: number, c: string, m: string, r?: number) => APIError> = {
@@ -159,7 +171,6 @@ const BY_STATUS: Record<number, new (s: number, c: string, m: string, r?: number
   402: InsufficientCredits,
   403: Forbidden,
   404: NotFound,
-  409: JobNotCancellable,
   422: IdempotencyKeyReused,
   429: RateLimited,
   503: Unavailable,

package/src/index.ts

@@ -32,6 +32,7 @@ export type { ClientOptions } from "./client.js";
 export { DEFAULT_BASE_URL, VERSION } from "./core.js";
 export type { AskOptions, CrawlOptions, MapOptions, SearchOptions } from "./core.js";
 export {
+  AlreadyExists,
   APIError,
   Blocked,
   DataFuelError,
@@ -55,9 +56,11 @@ export type { ErrorCode } from "./errors.js";
 export { Capabilities, CrawlPage, isDone, Result } from "./models.js";
 export type {
   AI,
+  AIProvider,
   Analytics,
   AnalyticsCounts,
   AnalyticsOptions,
+  BalanceSplit,
   CallOptions,
   CancelResult,
   Capability,
@@ -66,6 +69,7 @@ export type {
   CrawlStatus,
   Engine,
   Format,
+  Health,
   JobResults,
   JobsPage,
   JobStatus,
@@ -77,6 +81,7 @@ export type {
   ListTasksOptions,
   Payload,
   Profile,
+  ProtectionCheck,
   Proxy,
   ProxyCountry,
   ProxyLocation,

package/src/models.ts

@@ -60,7 +60,9 @@ export interface AI {
   prompt?: string;
   /** Example JSON object the output must follow. */
   format?: unknown;
-  provider?: "openai" | "anthropic" | "google";
+  /** Required. One of the providers `df.aiProviders()` lists; the API answers 400 INVALID_ATTRIBUTES without it. */
+  provider?: string;
+  /** One of the models `df.aiProviders()` lists for `provider`. Omit for the provider's default. */
   model?: string;
   apiKey?: string;
 }
@@ -75,9 +77,15 @@ export interface ScrapeOptions {
   jsRendering?: boolean;
   waitFor?: string;
   waitForTimeoutMs?: number;
-  /** An object keyed by action, e.g. `{ click: "#more" }`. `df.jsInstructions()` lists them. */
+  /**
+   * Browser actions run after load: an array of single-action objects, run in
+   * order, e.g. `[{ click: "#more" }, { wait_ms: 1000 }, { click: "#more" }]`.
+   * An object keyed by action is still accepted, but its order is not
+   * guaranteed and an action cannot repeat. `df.jsInstructions()` lists them.
+   */
   jsInstructions?: unknown;
-  blockResource?: string;
+  /** Resource types the browser must not load: one, e.g. `"Image"`, or several. */
+  blockResource?: string | string[];
   /** Markdown only: always render just the `<main>` / `<article>` container. */
   mainContentOnly?: boolean;
   /** Markdown only: `false` drops images and saves tokens. */
@@ -414,6 +422,8 @@ export interface TransactionsOptions {
 export interface Transaction {
   id: number;
   amount: number;
+  /** The part of `amount` that moved plan credits, same sign; the rest moved pay-as-you-go credits. */
+  plan_amount?: number;
   operation: TransactionOperation;
   /** What `reference_id` points to, e.g. `task_id` or `job_id`. */
   reference_type: string;
@@ -508,13 +518,30 @@ export interface Analytics {
   by_status_code: StatusCodeBreakdown[];
 }
 
+/**
+ * Credits by pool. Plan credits are spent first, roll over when the plan renews and
+ * expire if it is not renewed. Pay-as-you-go credits come from credit packs, are spent
+ * after plan credits and never expire.
+ */
+export interface BalanceSplit {
+  /** Total spendable credits: `plan_balance` plus `payg_balance`. */
+  balance: number;
+  plan_balance: number;
+  payg_balance: number;
+}
+
 /** The account behind the API key. */
 export interface Profile {
   email: string;
   username: string;
   current_concurrency: number;
   concurrency_limit: number;
+  /** Total spendable credits: `plan_credit_balance` plus `payg_credit_balance`. */
   credit_balance: number;
+  /** Spent first; expire if the plan is not renewed. */
+  plan_credit_balance?: number;
+  /** Spent after plan credits; never expire. */
+  payg_credit_balance?: number;
   monthly_credit_limit: number;
 }
 
@@ -538,6 +565,28 @@ export interface JsInstruction {
   example: unknown;
 }
 
+/** An LLM provider `ai.provider` accepts, with the models `ai.model` accepts for it. */
+export interface AIProvider {
+  name: string;
+  models: string[];
+}
+
+/** The anti-bot protection in front of one host and path, from `checkProtection`. */
+export interface ProtectionCheck {
+  host: string;
+  path: string;
+  /** cloudflare, cloudflare_5sec, akamai, imperva, perimeterx or unprotected. */
+  protection_type: string;
+}
+
+/** The state of the API. `checks` is filled by a deep check only, keyed by dependency. */
+export interface Health {
+  status: "ok" | "degraded" | (string & {});
+  checks?: Record<string, { status: "ok" | "fail" | (string & {}); latency_ms: number }>;
+  /** Whether the API can serve requests. */
+  ok: boolean;
+}
+
 /** A named proxy location: a city, or an ASN. */
 export interface ProxyLocation {
   code: string;