@datafuel/sdk 0.3.0 → 0.4.0

package/src/core.ts

@@ -21,7 +21,7 @@ import type {
 export const DEFAULT_BASE_URL = "https://scraping-api.datafuel.ai/api/v1";
 
 /** Kept in step with package.json by a test; see test/hardening.test.ts. */
-export const VERSION = "0.3.0";
+export const VERSION = "0.4.0";
 
 /** How long a synchronous call waits for a result before giving up. */
 export const DEFAULT_TIMEOUT_MS = 180_000;
@@ -37,9 +37,11 @@ export class Request {
     readonly method: string,
     readonly path: string,
     readonly params?: Record<string, string>,
-    readonly body?: Record<string, unknown>,
+    readonly body?: Record<string, unknown> | unknown[],
     readonly idempotencyKey?: string,
     readonly auth: boolean = true,
+    /** A 503 that carries a health report is an answer, not an error. */
+    readonly degradedOk: boolean = false,
   ) {}
 
   /** GETs are safe by nature, writes because they carry an idempotency key. */
@@ -109,23 +111,10 @@ function aiAttributes(ai: AI): Record<string, unknown> {
   return out;
 }
 
-/**
- * Catch a half-filled AI key before it costs a request. The API does not check
- * these, so a missing one would be spent on a scrape whose AI step then fails.
- */
+/** The API answers 400 INVALID_ATTRIBUTES without a provider; say so before sending. */
 export function validateAI(ai: AI): void {
-  const given = {
-    provider: ai.provider !== undefined,
-    model: ai.model !== undefined,
-    apiKey: ai.apiKey !== undefined,
-  };
-  const present = Object.values(given).filter(Boolean).length;
-  if (present > 0 && present < 3) {
-    const missing = Object.entries(given)
-      .filter(([, ok]) => !ok)
-      .map(([name]) => name)
-      .join(", ");
-    throw new TypeError(`ai needs provider, model and apiKey together; missing: ${missing}`);
+  if (!ai.provider) {
+    throw new TypeError("ai needs a provider; aiProviders() lists the ones the API supports");
   }
 }
 
@@ -141,7 +130,12 @@ export function scrapeAttributes(options: ScrapeOptions = {}): Record<string, un
   if (options.waitFor) attrs.wait_for_selector = options.waitFor;
   if (options.waitForTimeoutMs) attrs.wait_for_selector_timeout_ms = options.waitForTimeoutMs;
   if (options.jsInstructions) attrs.js_instructions = options.jsInstructions;
-  if (options.blockResource) attrs.block_resource = options.blockResource;
+  if (options.blockResource?.length) {
+    attrs.block_resource =
+      typeof options.blockResource === "string"
+        ? options.blockResource
+        : [...options.blockResource];
+  }
   if (options.mainContentOnly) attrs.main_content_only = true;
   if (options.includeImages !== undefined) attrs.include_images = options.includeImages;
   if (options.extract) attrs.extract_selector = selector(options.extract);
@@ -386,7 +380,10 @@ export interface SearchOptions {
   ludocid?: string;
   lsig?: string;
   ibp?: string;
-  /** Exit country of the request. */
+  /**
+   * Accepted but not used yet: searches leave through DataFuel's own pool.
+   * Target a market with `country` and `language`.
+   */
   proxyCountry?: string;
   /** Default json. */
   format?: "json" | "html" | "markdown";
@@ -465,6 +462,18 @@ export function crawlResultsRequest(crawlId: string, cursor?: string, limit?: nu
   );
 }
 
+export function healthRequest(deep: boolean): Request {
+  return new Request(
+    "GET",
+    "/healthz",
+    deep ? { deep: "1" } : undefined,
+    undefined,
+    undefined,
+    false,
+    true,
+  );
+}
+
 /** A Date as its UTC day, which is what the API's date filters take. */
 export function day(value: string | Date): string {
   return typeof value === "string" ? value : value.toISOString().slice(0, 10);

package/src/errors.ts

@@ -45,6 +45,10 @@ export type ErrorCode =
   | "JOB_NOT_FOUND"
   | "CRAWL_NOT_FOUND"
   | "JOB_NOT_CANCELLABLE"
+  | "TASK_ALREADY_EXISTS"
+  | "JOB_ALREADY_EXISTS"
+  | "URL_LENGTH_CANNOT_BE_ZERO"
+  | "ANALYTICS_FETCH_FAILED"
   | "INVALID_CRAWL_PATTERN"
   | "CRAWL_UNSUPPORTED_OPTION"
   | "INVALID_CURSOR"
@@ -92,6 +96,12 @@ export class InvalidAttributes extends APIError {}
 export class IdempotencyKeyReused extends APIError {}
 /** 409 JOB_NOT_CANCELLABLE: the job or crawl already finished. */
 export class JobNotCancellable extends APIError {}
+/**
+ * 409 TASK_ALREADY_EXISTS / JOB_ALREADY_EXISTS: the create collided with an
+ * existing task or job and is not an idempotent replay. Nothing was charged;
+ * send the request again.
+ */
+export class AlreadyExists extends APIError {}
 /** 503: an operator switched something off, or a dependency is down. The message carries the reason. */
 export class Unavailable extends APIError {}
 /** 503 MODULE_UNAVAILABLE: this task type is switched off. Nothing was charged. */
@@ -152,6 +162,8 @@ const BY_CODE: Record<string, new (s: number, c: string, m: string, r?: number)
   INVALID_API_KEY: Unauthorized,
   FORBIDDEN: Forbidden,
   JOB_NOT_CANCELLABLE: JobNotCancellable,
+  TASK_ALREADY_EXISTS: AlreadyExists,
+  JOB_ALREADY_EXISTS: AlreadyExists,
 };
 
 const BY_STATUS: Record<number, new (s: number, c: string, m: string, r?: number) => APIError> = {
@@ -159,7 +171,6 @@ const BY_STATUS: Record<number, new (s: number, c: string, m: string, r?: number
   402: InsufficientCredits,
   403: Forbidden,
   404: NotFound,
-  409: JobNotCancellable,
   422: IdempotencyKeyReused,
   429: RateLimited,
   503: Unavailable,

package/src/index.ts

@@ -32,6 +32,7 @@ export type { ClientOptions } from "./client.js";
 export { DEFAULT_BASE_URL, VERSION } from "./core.js";
 export type { AskOptions, CrawlOptions, MapOptions, SearchOptions } from "./core.js";
 export {
+  AlreadyExists,
   APIError,
   Blocked,
   DataFuelError,
@@ -55,6 +56,7 @@ export type { ErrorCode } from "./errors.js";
 export { Capabilities, CrawlPage, isDone, Result } from "./models.js";
 export type {
   AI,
+  AIProvider,
   Analytics,
   AnalyticsCounts,
   AnalyticsOptions,
@@ -67,6 +69,7 @@ export type {
   CrawlStatus,
   Engine,
   Format,
+  Health,
   JobResults,
   JobsPage,
   JobStatus,
@@ -78,6 +81,7 @@ export type {
   ListTasksOptions,
   Payload,
   Profile,
+  ProtectionCheck,
   Proxy,
   ProxyCountry,
   ProxyLocation,

package/src/models.ts

@@ -60,7 +60,9 @@ export interface AI {
   prompt?: string;
   /** Example JSON object the output must follow. */
   format?: unknown;
-  provider?: "openai" | "anthropic" | "google";
+  /** Required. One of the providers `df.aiProviders()` lists; the API answers 400 INVALID_ATTRIBUTES without it. */
+  provider?: string;
+  /** One of the models `df.aiProviders()` lists for `provider`. Omit for the provider's default. */
   model?: string;
   apiKey?: string;
 }
@@ -75,9 +77,15 @@ export interface ScrapeOptions {
   jsRendering?: boolean;
   waitFor?: string;
   waitForTimeoutMs?: number;
-  /** An object keyed by action, e.g. `{ click: "#more" }`. `df.jsInstructions()` lists them. */
+  /**
+   * Browser actions run after load: an array of single-action objects, run in
+   * order, e.g. `[{ click: "#more" }, { wait_ms: 1000 }, { click: "#more" }]`.
+   * An object keyed by action is still accepted, but its order is not
+   * guaranteed and an action cannot repeat. `df.jsInstructions()` lists them.
+   */
   jsInstructions?: unknown;
-  blockResource?: string;
+  /** Resource types the browser must not load: one, e.g. `"Image"`, or several. */
+  blockResource?: string | string[];
   /** Markdown only: always render just the `<main>` / `<article>` container. */
   mainContentOnly?: boolean;
   /** Markdown only: `false` drops images and saves tokens. */
@@ -557,6 +565,28 @@ export interface JsInstruction {
   example: unknown;
 }
 
+/** An LLM provider `ai.provider` accepts, with the models `ai.model` accepts for it. */
+export interface AIProvider {
+  name: string;
+  models: string[];
+}
+
+/** The anti-bot protection in front of one host and path, from `checkProtection`. */
+export interface ProtectionCheck {
+  host: string;
+  path: string;
+  /** cloudflare, cloudflare_5sec, akamai, imperva, perimeterx or unprotected. */
+  protection_type: string;
+}
+
+/** The state of the API. `checks` is filled by a deep check only, keyed by dependency. */
+export interface Health {
+  status: "ok" | "degraded" | (string & {});
+  checks?: Record<string, { status: "ok" | "fail" | (string & {}); latency_ms: number }>;
+  /** Whether the API can serve requests. */
+  ok: boolean;
+}
+
 /** A named proxy location: a city, or an ASN. */
 export interface ProxyLocation {
   code: string;